package main import ( "context" "crypto/sha256" "encoding/hex" "errors" "flag" "fmt" "io" "log" "net/http" "net/url" "os" "os/signal" "slices" "strconv" "strings" "syscall" "time" ) const ( // importCheckWait bounds the wait for "uploaded" after the 202: the // console's integrity check (30 minutes) and its put (20), and a CAS. importCheckWait = 55 * time.Minute // importSwapWait and importPlaceWait are spec §5's waits for the box. importSwapWait = 10 * time.Minute importPlaceWait = 5 * time.Minute // importVerifyTries is how many times a verify answered 502, 503 or 504 // (the gate while the instance wakes, or a backhaul) is asked again, // waiting Retry-After, or importVerifyRetry without one. importVerifyTries = 20 importVerifyRetry = 3 * time.Second ) // The usage block of carlos instances import, one line to a literal. The // continuation lines are indented when the block is put together. const ( importUsageSynopsis = "usage: carlos instances import --app --host --file --replaces " importUsageOptions = "[--verify ] [--environment ]" importUsageCommit = "carlos instances import commit --app --host " importUsageAbort = "carlos instances import abort --app --host " importUsageDescription = "carlos instances import moves an instance the platform never ran onto the platform." ) // importUsage is the block flagUsage prints above the flags of import, // import commit and import abort. func importUsage() string { optionsIndent := strings.Repeat(" ", 30) verbIndent := strings.Repeat(" ", 7) return strings.Join([]string{ importUsageSynopsis, optionsIndent + importUsageOptions, verbIndent + importUsageCommit, verbIndent + importUsageAbort, "", importUsageDescription, }, "\n") } // importCheckWaitOverLine is what the CLI says when the console has not // kept the upload by the end of importCheckWait. func importCheckWaitOverLine(app, host string) string { return fmt.Sprintf("the console has not kept the upload of %s after %d minutes. Check carlos instances list. To start over, run carlos instances import abort --app %s --host %s, then run the import again.", host, int(importCheckWait/time.Minute), app, host) } // importEndedLine is what the CLI says when the record it was following // has gone, or now carries another import. func importEndedLine(host string) string { return fmt.Sprintf("the import of %s has ended. It was deleted, or another import of %s began. Check carlos instances list.", host, host) } // importNothingToAbortError is import abort's error for a host with no // record. The CLI cannot tell a typo from an import that was aborted and // cleaned up, so the sentence says both. func importNothingToAbortError(host, app string) error { return fmt.Errorf("%s is not an instance of %s, so there is no import to abort. An import that was aborted and cleaned up leaves no instance behind.", host, app) } // errImportFailed is an import that ended badly once its own line was // printed: the command exits 1 and adds nothing. var errImportFailed = errors.New("instances import failed") // importOver is an import that ended on the console or the box, not by // this command: refused, aborted, rolled back, or no longer on the record. // line is what to print. The token is not shown: there is nothing left to // verify. type importOver struct{ line string } func (e importOver) Error() string { return e.line } // importLive is an exit that leaves the import live and uncommitted // (spec §5, ruling M17): the token is shown first, then lines. type importLive struct{ lines []string } func (e importLive) Error() string { return strings.Join(e.lines, "\n") } // importStopSignals end an import early, through its live exit: Ctrl-C, a // supervisor's SIGTERM, and a closed terminal (SIGHUP). Not SIGPIPE, unlike // carlos db's set: a write to a reset connection (the upload, a poll, the // verify) raises it, and that must stay the request's own error rather than // cancel the whole import. syscall.SIGHUP and SIGTERM exist on every // platform the CLI builds for, as carlos db's set relies on. var importStopSignals = []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGHUP} // importJob is one `carlos instances import`. type importJob struct { acct apiAcct app, host, file, replaces, environment, verify, message string verifyClient *http.Client // nil: a client with a 2-minute timeout } // runInstancesImport is `carlos instances import` (seeded-instances spec §5). func runInstancesImport(args []string) { if len(args) > 0 && (args[0] == "commit" || args[0] == "abort") { runInstancesImportVerb(args[0], args[1:]) return } fs := flag.NewFlagSet("instances import", flag.ExitOnError) messageFlags := addMutationMessageFlags(fs) app := appFlag(fs, "app the instance belongs to") host := fs.String("host", "", "instance host — one DNS label + \".\" + the app's instance domain") file := fs.String("file", "", "the snapshot to import: a consistent SQLite file taken after the old process stopped") replaces := fs.String("replaces", "", "the address of the route this import takes over, as host:port") environment := fs.String("environment", "", "named config bundle this route binds to (default: the app's default bundle) — carlos env environments lists them") verify := fs.String("verify", "", "a path to request with the verify token once swapped. A 2xx commits the import, anything else aborts it") consoleFlag := consoleOption(fs, "console to create through (default: the only logged-in one)") acctRaw := fs.String("account", "", "account (sqid or name) the app belongs to (default: your only account, or the configured/chosen one)") acctIDRaw := fs.String("account-id", "", "account sqid, matched as a sqid ONLY — supersedes --account") fs.Usage = flagUsage(fs, importUsage()) fs.Parse(args) if *host == "" || *file == "" || *replaces == "" { fs.Usage() os.Exit(2) } // A --verify that cannot make a request would fail only after the swap, // and abort a good import over a typo. if *verify != "" { if _, err := url.Parse(importVerifyTarget(strings.TrimSpace(*host), strings.TrimSpace(*verify))); err != nil { fs.Usage() os.Exit(2) } } sess := requireAPISession("import", *consoleFlag) printUsing(stdout, sess) acct := resolvedAccount(sess, acctRaw, acctIDRaw) app2 := resolvedApp(sess, acct, *app, instancesRecordsUsage) message, merr := resolveMutationMessage(fs, messageFlags) if merr != nil { log.Fatalf("instances import: %v", merr) } // Ctrl-C, SIGTERM or a closed terminal cancels the import's context // rather than killing the process, so an import left live still shows // its token and the commands to finish it (M17). ctx, stop := signal.NotifyContext(context.Background(), importStopSignals...) defer stop() err := sess.runImport(ctx, stdout, importJob{acct: acct, app: app2, host: strings.TrimSpace(*host), file: *file, replaces: strings.TrimSpace(*replaces), environment: strings.TrimSpace(*environment), verify: strings.TrimSpace(*verify), message: message}) switch { case errors.Is(err, errImportFailed): stop() os.Exit(1) case err != nil: log.Fatalf("instances import: %v", err) } } // runImport hashes the file and begins, then drives the import (driveImport) // and settles what its ending prints. The verify token stays in memory. It // is printed (SD-N72) right after begin without --verify, and with --verify // only before an exit that leaves the import live and uncommitted. A // cancelled ctx after begin is such an exit: the token and the commands to // finish by hand, and no commit or abort is sent. Cancelled before begin // answers, it is begin's error. func (s *apiSession) runImport(ctx context.Context, w io.Writer, j importJob) error { f, err := os.Open(j.file) if err != nil { return err } defer f.Close() h := sha256.New() size, err := io.Copy(h, f) if err != nil { return err } fmt.Fprintln(w, fmt.Sprintf("uploading %s (%s)", j.file, humanBytes(&size))) rec, token, err := s.importBegin(ctx, j.acct, j.app, j.host, j.environment, hex.EncodeToString(h.Sum(nil)), size, j.replaces, j.message) if err != nil { return err } id := "" if rec.Seed != nil { id = rec.Seed.ID } shown := false showToken := func() { if !shown { fmt.Fprintln(w, fmt.Sprintf("verify token: %s. It is shown only this once.", token)) shown = true } } if j.verify == "" { showToken() } err = s.driveImport(ctx, w, j, f, size, id, token) var over importOver var live importLive switch { case err == nil, errors.Is(err, errImportFailed): return err case errors.As(err, &over): fmt.Fprintln(w, over.line) return errImportFailed } showToken() if ctx.Err() != nil { fmt.Fprintln(w, importFinishLine(j.app, j.host)) return errImportFailed } if errors.As(err, &live) { for _, l := range live.lines { fmt.Fprintln(w, l) } return errImportFailed } return err } // driveImport is everything after begin: upload, wait for the check, // stopped, wait for the swap, and then either the hand-finish commands or // the verify and its commit or abort. func (s *apiSession) driveImport(ctx context.Context, w io.Writer, j importJob, f *os.File, size int64, id, token string) error { if _, err := f.Seek(0, io.SeekStart); err != nil { return err } if err := s.importUpload(ctx, j.acct, j.app, j.host, id, f, size); err != nil { return err } // The console answered 202 and checks the file on its own; the record // stays awaiting-upload until it is kept. if _, err := s.pollImport(ctx, j, importCheckWait, importLook(j.host, id, "uploaded", "ready", "swapped", "placed", "committed")); err != nil { if errors.Is(err, errSeedWaitOver) { return importLive{lines: []string{importCheckWaitOverLine(j.app, j.host)}} } return err } if err := s.importStep(ctx, j.acct, j.app, j.host, id, "stopped"); err != nil { return err } fmt.Fprintln(w, fmt.Sprintf("uploaded. The old process of %s is marked stopped, and the box swaps it on its next pass.", j.host)) fmt.Fprintln(w, fmt.Sprintf("waiting for the box to swap %s", j.host)) swapped, err := s.pollImport(ctx, j, importSwapWait, importLook(j.host, id, "swapped", "placed", "committed")) if err != nil { if errors.Is(err, errSeedWaitOver) { return importLive{lines: []string{fmt.Sprintf("the box has not swapped %s after %d minutes. Check carlos instances list, then commit or abort.", j.host, int(importSwapWait/time.Minute))}} } return err } if swapped.Seed.State == "committed" { // Committed by hand (the token was shown at begin) between two looks. fmt.Fprintln(w, importCommittedLine(j.host)) return nil } fmt.Fprintln(w, fmt.Sprintf("swapped. Until it is committed, %s answers only requests that carry the verify token.", j.host)) if j.verify == "" { fmt.Fprintln(w, importFinishLine(j.app, j.host)) return nil } client := j.verifyClient if client == nil { client = &http.Client{Timeout: 2 * time.Minute} } code, verr := verifyImport(ctx, client, j.host, j.verify, token) if ctx.Err() != nil { // Cancelled is not a verdict: neither commit nor abort. return ctx.Err() } if verr == nil && code >= 200 && code < 300 { fmt.Fprintln(w, fmt.Sprintf("verify passed (%d). Committing.", code)) fmt.Fprintln(w, "waiting for the box to finish placing the snapshot") if _, err := s.pollImport(ctx, j, importPlaceWait, importLook(j.host, id, "placed", "committed")); err != nil { if errors.Is(err, errSeedWaitOver) { return importLive{lines: []string{importFinishLine(j.app, j.host)}} } return err } if err := s.importStep(ctx, j.acct, j.app, j.host, id, "commit"); err != nil { return err } fmt.Fprintln(w, importCommittedLine(j.host)) return nil } status := strconv.Itoa(code) if verr != nil { status = verr.Error() } fmt.Fprintln(w, fmt.Sprintf("verify failed (%s). Aborting the import.", status)) if err := s.importStep(ctx, j.acct, j.app, j.host, id, "abort"); err != nil { return err } fmt.Fprintln(w, importAbortedLine(j.host)) return errImportFailed } // pollImport is pollSeed for an import: a record that has gone ends the // import with the line to print, importEndedLine, rather than the // membership sentence. func (s *apiSession) pollImport(ctx context.Context, j importJob, limit time.Duration, look func(apiInstanceRecord) (bool, error)) (apiInstanceRecord, error) { rec, err := s.pollSeed(ctx, j.acct, j.app, j.host, limit, look) var gone seedGone if errors.As(err, &gone) { return rec, importOver{line: importEndedLine(j.host)} } return rec, err } // importLook is pollSeed's look for an import: done once the seed is in // one of states. A refused, aborted or rolled-back seed, or a record that // no longer carries this import (id, when known), ends the import with the // line to print: SD-N78, the seed's reason, seedCell's "aborting", or, for // a record that carries no such import, importEndedLine. func importLook(host, id string, states ...string) func(apiInstanceRecord) (bool, error) { return func(rec apiInstanceRecord) (bool, error) { sd := rec.Seed if sd == nil || (id != "" && sd.ID != id) { return false, importOver{line: importEndedLine(host)} } switch sd.State { case "refused": return false, importOver{line: fmt.Sprintf("the box refused the import: %s", sd.Reason)} case "aborted", "rolled-back": if sd.Reason != "" { return false, importOver{line: sd.Reason} } return false, importOver{line: seedCell(rec)} } return slices.Contains(states, sd.State), nil } } // importLanded is, per verb, the states that prove it landed. var importLanded = map[string][]string{ "stopped": {"ready", "swapped", "placed", "committed"}, "commit": {"committed"}, "abort": {"aborted", "rolled-back"}, } // importStep runs one of the import's verbs on seed id, which the console // requires (final review F1). When the console refuses it, the record says // whether it landed anyway: a stopped whose first answer was lost is // refused naming a later state (SD-N54), and a commit or abort that // overlapped another is refused as changed (SD-N60). An abort whose record // has then gone is done, since the abort finishes a rolled-back import and // deletes its record. func (s *apiSession) importStep(ctx context.Context, a apiAcct, app, host, id, verb string) error { err := s.importVerb(ctx, a, app, host, id, verb) if err == nil { return nil } rec, found, rerr := s.instanceGet(ctx, a, app, host) switch { case rerr != nil: return err case !found: if verb == "abort" && id != "" { return nil } return err case rec.Seed == nil || (id != "" && rec.Seed.ID != id): return err case slices.Contains(importLanded[verb], rec.Seed.State): return nil } return err } func importFinishLine(app, host string) string { return fmt.Sprintf("commit with: carlos instances import commit --app %s --host %s. Abort with: carlos instances import abort --app %s --host %s.", app, host, app, host) } func importCommittedLine(host string) string { return fmt.Sprintf("committed. %s is served by the platform now.", host) } func importAbortedLine(host string) string { return fmt.Sprintf("aborted. The box puts the old route of %s back on its next pass.", host) } // runInstancesImportVerb is `carlos instances import commit|abort`: the // verb alone, for an import left to be finished by hand. func runInstancesImportVerb(verb string, args []string) { fs := flag.NewFlagSet("instances import "+verb, flag.ExitOnError) app := appFlag(fs, "app the instance belongs to") host := fs.String("host", "", "instance host — one DNS label + \".\" + the app's instance domain") consoleFlag := consoleOption(fs, "console to create through (default: the only logged-in one)") acctRaw := fs.String("account", "", "account (sqid or name) the app belongs to (default: your only account, or the configured/chosen one)") acctIDRaw := fs.String("account-id", "", "account sqid, matched as a sqid ONLY — supersedes --account") fs.Usage = flagUsage(fs, importUsage()) fs.Parse(args) if *host == "" { fs.Usage() os.Exit(2) } sess := requireAPISession("import", *consoleFlag) printUsing(stdout, sess) acct := resolvedAccount(sess, acctRaw, acctIDRaw) app2 := resolvedApp(sess, acct, *app, instancesRecordsUsage) if err := sess.runImportVerb(context.Background(), stdout, acct, app2, strings.TrimSpace(*host), verb); err != nil { log.Fatalf("instances import %s: %v", verb, err) } } // runImportVerb acts on the import the host holds now: it reads the record // and sends the seed id it read (final review F1), so a re-begun import is // only ever acted on by a command run after it began. A host with no record, // or no seed, sends no id, and the console's own answer is the error. func (s *apiSession) runImportVerb(ctx context.Context, w io.Writer, a apiAcct, app, host, verb string) error { rec, found, err := s.instanceGet(ctx, a, app, host) if err != nil { return err } id := "" if found && rec.Seed != nil { id = rec.Seed.ID } if err := s.importStep(ctx, a, app, host, id, verb); err != nil { if verb == "abort" && !found && err.Error() == fmt.Sprintf("%s is not an instance of %s", host, app) { return importNothingToAbortError(host, app) } return err } if verb == "commit" { fmt.Fprintln(w, importCommittedLine(host)) return nil } fmt.Fprintln(w, importAbortedLine(host)) return nil } // verifyImport is the verify request through the edge (spec §5.5) to // https://. func verifyImport(ctx context.Context, client *http.Client, host, path, token string) (int, error) { return verifyImportURL(ctx, client, importVerifyTarget(host, path), token) } // importVerifyTarget is the verify URL, https://. func importVerifyTarget(host, path string) string { if !strings.HasPrefix(path, "/") { path = "/" + path } return "https://" + host + path } // verifyImportURL asks target with the verify token and returns the // status that decides the import. 502, 503 and 504 are asked again (the // gate answers 503 with Retry-After while the instance wakes, and a // backhaul can answer 502 or 504 meanwhile), up to importVerifyTries // times. A redirect is never followed: only a 2xx of the path itself // commits. func verifyImportURL(ctx context.Context, client *http.Client, target, token string) (int, error) { c := *client c.CheckRedirect = neverFollow for try := 0; ; try++ { req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) if err != nil { return 0, err } req.Header.Set("Carlos-Import-Verify", token) res, err := c.Do(req) if err != nil { return 0, err } io.Copy(io.Discard, io.LimitReader(res.Body, 1<<20)) res.Body.Close() switch res.StatusCode { case http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: default: return res.StatusCode, nil } if try >= importVerifyTries { return res.StatusCode, nil } wait := importVerifyRetry if n, err := strconv.Atoi(res.Header.Get("Retry-After")); err == nil && n >= 0 && n <= 30 { wait = time.Duration(n) * time.Second } if err := seedWait(ctx, wait); err != nil { return 0, err } } }