package main // carlos system geoip publish|status|off (geoip spec §4.2, §5.5). The // strings here await Paul's copy review (plan 1, Task 13) before merge. import ( "context" "encoding/json" "errors" "flag" "fmt" "io" "log" "net/http" "os" "strings" "time" ) func runSystemGeoipVerb(rest []string) { if len(rest) == 0 { fmt.Fprint(os.Stderr, "system geoip: no verb given\n\n") systemUsage() os.Exit(2) } switch rest[0] { case "publish": runSystemGeoipPublish(rest[1:]) case "status": runSystemGeoipStatus(rest[1:]) case "off": runSystemGeoipOff(rest[1:]) default: fmt.Fprintf(os.Stderr, "system geoip: unknown verb %q\n\n", rest[0]) systemUsage() os.Exit(2) } } func runSystemGeoipPublish(args []string) { fs := flag.NewFlagSet("system geoip publish", flag.ExitOnError) consoleFlag := consoleOption(fs, "console origin (default: the only logged-in one)") force := fs.Bool("force", false, "publish even if the database has under half the current one's nodes") fs.Usage = flagUsage(fs, "") fs.Parse(args) if fs.NArg() != 1 { fs.Usage() os.Exit(2) } sess := requireDeployAPISession(*consoleFlag) printUsing(stdout, sess) ptr, err := sess.publishGeoip(context.Background(), fs.Arg(0), *force, stdout) if err != nil { log.Fatalf("carlos system geoip publish: %v", err) } fmt.Fprintf(stdout, "%s %s: %s, built %s. every edge picks it up within 12 minutes\n", colorFor(stdout).ok("published"), short8cli(ptr.SHA256), ptr.DatabaseType, time.Unix(ptr.BuildEpoch, 0).UTC().Format("2006-01-02")) } // geoipPointerView is the pointer as the CLI reads it back from a job. type geoipPointerView struct { SHA256 string `json:"sha256"` DatabaseType string `json:"database_type"` BuildEpoch int64 `json:"build_epoch"` } // publishGeoip streams the file in one raw PUT (the console stages it and // answers 202 with a job), then follows the job to its result. func (s *apiSession) publishGeoip(ctx context.Context, path string, force bool, w io.Writer) (geoipPointerView, error) { f, err := os.Open(path) if err != nil { return geoipPointerView{}, err } defer f.Close() fi, err := f.Stat() if err != nil { return geoipPointerView{}, err } q := "" if force { q = "?force=1" } req, err := http.NewRequestWithContext(ctx, http.MethodPut, s.c.base+"/api/cli/system/geoip"+q, f) if err != nil { return geoipPointerView{}, err } req.ContentLength = fi.Size() req.Header.Set("Content-Type", "application/octet-stream") if s.c.token != "" { req.Header.Set("Authorization", "Bearer "+s.c.token) } // The upload's own client, like doMultipartMethod's: c.httpc's 30s // absolute timeout is sized for JSON, not a 130 MB database. client := &http.Client{Timeout: shipHTTPClientTimeout, CheckRedirect: neverFollow} res, err := client.Do(req) if err != nil { return geoipPointerView{}, err } defer res.Body.Close() if merr := consoleMoved(s.c.base, res.StatusCode, res.Header.Get("Location")); merr != nil { return geoipPointerView{}, offerConsoleMoved(merr) } body, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20)) if res.StatusCode != http.StatusAccepted { return geoipPointerView{}, apiErrorFromBody(res.StatusCode, body) } var started struct { Job string `json:"job"` } if err := json.Unmarshal(body, &started); err != nil || started.Job == "" { return geoipPointerView{}, errors.New("the console accepted the upload but named no job") } return s.watchGeoipJob(ctx, started.Job, w) } func (s *apiSession) watchGeoipJob(ctx context.Context, id string, w io.Writer) (geoipPointerView, error) { printed := 0 deadline := time.Now().Add(16 * time.Minute) for { st, body, err := s.getJSON(ctx, "/api/cli/system/jobs/"+id) if err != nil { return geoipPointerView{}, fmt.Errorf("job %s: %v", id, err) } if st != http.StatusOK { return geoipPointerView{}, fmt.Errorf("job %s: %s", id, apiErrorFromBody(st, body)) } var j struct { Steps []string `json:"steps"` Done bool `json:"done"` Error string `json:"error"` Result *geoipPointerView `json:"result"` } _ = json.Unmarshal(body, &j) for ; printed < len(j.Steps); printed++ { fmt.Fprintf(w, " %s\n", j.Steps[printed]) } if j.Done { if j.Error != "" { return geoipPointerView{}, errors.New(j.Error) } if j.Result == nil { return geoipPointerView{}, errors.New("the job finished without naming what it published") } return *j.Result, nil } if time.Now().After(deadline) { return geoipPointerView{}, errors.New("the console's job did not finish in 16 minutes") } time.Sleep(geoipPollInterval) } } // geoipPollInterval is a var so a test need not wait seconds per poll. var geoipPollInterval = 2 * time.Second type geoipStatusResponse struct { Pointer *struct { SHA256 string `json:"sha256"` DatabaseType string `json:"database_type"` Build time.Time `json:"build"` Published time.Time `json:"published"` Origin string `json:"origin"` Expired bool `json:"expired"` ExpiresAt time.Time `json:"expires_at"` Attribution string `json:"attribution"` } `json:"pointer"` PointerError string `json:"pointer_error"` Feed *struct { Name string `json:"name"` LastSuccess time.Time `json:"last_success"` Failures int `json:"failures"` NotFound int `json:"not_found"` LastFailure string `json:"last_failure"` Failing bool `json:"failing"` } `json:"feed"` Boxes []struct { Label string `json:"label"` Fleet string `json:"fleet"` State string `json:"state"` } `json:"boxes"` Ready bool `json:"ready"` Blocking []string `json:"blocking"` FeedSet bool `json:"feed_set"` } func runSystemGeoipStatus(args []string) { fs := flag.NewFlagSet("system geoip status", flag.ExitOnError) consoleFlag := consoleOption(fs, "console origin (default: the only logged-in one)") fs.Usage = flagUsage(fs, "") fs.Parse(args) sess := requireDeployAPISession(*consoleFlag) st, body, err := sess.getJSON(context.Background(), "/api/cli/system/geoip") if err != nil { log.Fatalf("carlos system geoip status: %v", err) } if st != http.StatusOK { log.Fatalf("carlos system geoip status: %s", apiErrorFromBody(st, body)) } var resp geoipStatusResponse if err := json.Unmarshal(body, &resp); err != nil { log.Fatalf("carlos system geoip status: unreadable response: %v", err) } printGeoipStatus(stdout, resp, time.Now()) } // printGeoipStatus is the verb minus the fetch, so it is testable. func printGeoipStatus(w io.Writer, resp geoipStatusResponse, now time.Time) { c := colorFor(w) switch p := resp.Pointer; { case resp.PointerError != "": fmt.Fprintf(w, "database: %s\n", c.bad(resp.PointerError)) case p == nil: fmt.Fprintln(w, "database: none published. geoip is off") case p.Expired: fmt.Fprintf(w, "database: %s %s expired. publish a newer one or turn geoip off\n", short8cli(p.SHA256), p.DatabaseType) default: age := int(now.Sub(p.Build).Hours() / 24) fmt.Fprintf(w, "database: %s %s, built %s (%d days ago), %s\n", short8cli(p.SHA256), p.DatabaseType, p.Build.Format("2006-01-02"), age, p.Origin) if !p.ExpiresAt.IsZero() { fmt.Fprintf(w, " expires %s (GeoLite2 must be replaced within 30 days of its build)\n", p.ExpiresAt.Format("2006-01-02")) } if p.Attribution != "" { fmt.Fprintf(w, " pages that show this data must credit: %s\n", p.Attribution) } } switch f := resp.Feed; { case f != nil && (f.Failures > 0 || f.NotFound > 0): n := f.Failures if f.NotFound > n { n = f.NotFound } line := fmt.Sprintf("feed: %s, failing (%d in a row, last: %s)", f.Name, n, f.LastFailure) if f.Failing { line = c.bad(line) } fmt.Fprintln(w, line) case f != nil && !f.LastSuccess.IsZero(): fmt.Fprintf(w, "feed: %s, last success %s\n", f.Name, f.LastSuccess.Format("2006-01-02")) case resp.FeedSet: fmt.Fprintln(w, "feed: set (CONSOLE_GEOIP_FEED), not run yet") } fmt.Fprintln(w) for _, b := range resp.Boxes { name := b.Label if b.Fleet != "" { name = b.Fleet + "/" + b.Label } fmt.Fprintf(w, " %-28s %s\n", name, b.State) } fmt.Fprintln(w) if resp.Ready { fmt.Fprintf(w, "%s: every box strips forged headers and has the current database\n", c.ok("ready")) return } fmt.Fprintf(w, "%s: %d in the way\n", c.bad("not ready"), len(resp.Blocking)) for _, b := range resp.Blocking { fmt.Fprintf(w, " %s\n", b) } } func runSystemGeoipOff(args []string) { fs := flag.NewFlagSet("system geoip off", flag.ExitOnError) consoleFlag := consoleOption(fs, "console origin (default: the only logged-in one)") fs.Usage = flagUsage(fs, "") fs.Parse(args) sess := requireDeployAPISession(*consoleFlag) st, body, err := sess.deleteJSON(context.Background(), "/api/cli/system/geoip") if err != nil { log.Fatalf("carlos system geoip off: %v", err) } if st != http.StatusOK { log.Fatalf("carlos system geoip off: %s", apiErrorFromBody(st, body)) } var out struct { FeedSet bool `json:"feed_set"` } _ = json.Unmarshal(body, &out) printGeoipOff(stdout, out.FeedSet) } func printGeoipOff(w io.Writer, feedSet bool) { fmt.Fprintln(w, "geoip is off. every edge drops the database within 12 minutes") if feedSet { fmt.Fprintln(w, "the feed is still set and will publish again within a day. unset CONSOLE_GEOIP_FEED to keep it off") } } func short8cli(sha string) string { if len(sha) > 8 { return sha[:8] } return strings.TrimSpace(sha) }