package main import ( "bytes" "context" "encoding/json" "errors" "flag" "github.com/aws/aws-sdk-go-v2/aws" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/carlosframework/platform/internal/awsinv" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfence" "github.com/carlosframework/platform/internal/hostresolve" "github.com/carlosframework/platform/internal/provision" "io" "net/http" "net/http/httptest" "os" "strings" "sync" "testing" "time" ) // The fake console below answers the move doors in Task 2's shapes: the flat // view (state, target, updated_at, id, last, protocol) and, for detail=1, // the same plus the detail object. Each flat GET returns the next scripted // frame; a detail GET returns the frame the last flat GET returned. const ( tMoveID = "mv_abcdefghijklmnopqrstuvwxyz" tHost = "acme.go.tito.io" tMovePth = "/api/cli/apps/bab/titogo/instances/acme.go.tito.io/move" ) type frame map[string]any func flatOf(f frame) frame { out := frame{} for k, v := range f { if k != "detail" { out[k] = v } } return out } type fakeMoveConsole struct { mu sync.Mutex frames []frame i int last frame tooMany int // the next flat GETs answered 429 flatGets int details int posts []string // method path body mutated bool // a mutation since the last flat GET answer func(method, path, body string) (int, any) // startDetail, when set, is the detail answer right after a mutation // (the move's own start), whatever the next scripted frame is. startDetail frame // detailStatus, when set, answers every detail GET with that status. detailStatus int } func (c *fakeMoveConsole) ServeHTTP(w http.ResponseWriter, r *http.Request) { c.mu.Lock() defer c.mu.Unlock() path := r.URL.Path if !strings.HasPrefix(path, tMovePth) { http.NotFound(w, r) return } write := func(code int, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) _ = json.NewEncoder(w).Encode(v) } if r.Method == http.MethodGet && path == tMovePth { if r.URL.Query().Get("detail") == "1" { c.details++ if c.detailStatus != 0 { write(c.detailStatus, map[string]string{"error": "detail refused by the test"}) return } cur := c.last if c.mutated && c.startDetail != nil { cur = c.startDetail } else if cur == nil || c.mutated { // Right after a mutation the console shows its result: the // next scripted frame. n := c.i if n >= len(c.frames) { n = len(c.frames) - 1 } cur = c.frames[n] } write(http.StatusOK, cur) return } c.flatGets++ if c.tooMany > 0 { c.tooMany-- write(http.StatusTooManyRequests, map[string]string{"error": "too many instance requests. wait a minute and try again"}) return } n := c.i if n >= len(c.frames) { n = len(c.frames) - 1 } c.last = c.frames[n] c.mutated = false c.i++ write(http.StatusOK, flatOf(c.last)) return } body, _ := io.ReadAll(r.Body) c.posts = append(c.posts, r.Method+" "+path+" "+strings.TrimSpace(string(body))) c.mutated = true if c.answer == nil { write(http.StatusAccepted, frame{"state": "preparing", "target": "uk", "id": tMoveID, "protocol": "phase1"}) return } code, v := c.answer(r.Method, path, string(body)) write(code, v) } func (c *fakeMoveConsole) mutations() []string { c.mu.Lock() defer c.mu.Unlock() return append([]string(nil), c.posts...) } // runMoveAgainst runs the verb against the fake, as an operator who is not // root, polling every millisecond. It returns the exit code and the output. func runMoveAgainst(t *testing.T, c *fakeMoveConsole, args ...string) (int, string) { t.Helper() return runMoveWith(t, c, nil, args...) } // runMoveWith is runMoveAgainst with deps adjusted first (stdin, the EC2 // reader). func runMoveWith(t *testing.T, c *fakeMoveConsole, adjust func(*moveDeps), args ...string) (int, string) { t.Helper() ts := httptest.NewServer(c) t.Cleanup(ts.Close) var out bytes.Buffer deps := moveDeps{ session: func(string) (*apiSession, error) { return &apiSession{c: newConsoleClient(ts.URL, "cli-token"), console: ts.URL, address: "op@keymail.dev"}, nil }, euid: func() int { return 1000 }, poll: time.Millisecond, stdin: strings.NewReader(""), ec2: func(context.Context, string) (awsinv.ExactEC2API, error) { t.Error("the EC2 reader was asked for") return nil, errors.New("no reader in this test") }, } if adjust != nil { adjust(&deps) } args = append(args, "--account", "bab", "--app", "titogo") ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() code := moveMain(ctx, args, &out, deps) return code, out.String() } // The frames a move goes through, as the console shows them. func none(last frame) frame { f := frame{"state": "none", "protocol": "phase1"} if last != nil { f["last"] = last } f["detail"] = frame{"region": "ie", "log": []string{}, "evidence": frame{}} return f } func inFlight(state string, move frame, extra frame) frame { m := frame{"id": tMoveID, "from": "ie", "to": "uk", "cut": "auto", "state": state} for k, v := range move { m[k] = v } d := frame{"move": m, "from": "ie", "region": "ie", "log": []string{}, "evidence": frame{}, "claim": frame{"kind": "moving", "exec": "ie-box", "target": "uk-box"}} for k, v := range extra { d[k] = v } f := frame{"state": state, "target": "uk", "updated_at": time.Now().Unix(), "id": tMoveID, "protocol": "phase1", "detail": d} if p := flatPhase(state, m); p != "" { f["phase"] = p } return f } // flatPhase is the flat view's phase, as the console derives it from the // journal (moveViewOf). func flatPhase(state string, m frame) string { str := func(k string) string { v, _ := m[k].(string); return v } switch state { case "cutting": return str("cut_phase") case "landing": return str("landing_phase") case "rolling_back": if rb, ok := m["rollback"].(frame); ok { tp, _ := rb["target_phase"].(string) sp, _ := rb["source_phase"].(string) return tp + "/" + sp } } return "" } func outcome(id, what string, offline float64) frame { return frame{"id": id, "outcome": what, "at": time.Now().UTC().Format(time.RFC3339), "offline_s": offline} } const ( mc1 = "Moving acme.go.tito.io from ie to uk." mc2 = "Copying data. acme.go.tito.io keeps serving while this runs." mc3 = "Checking that uk can run acme.go.tito.io." mc5 = "uk is ready. Starting the cut. acme.go.tito.io will be offline for about 45 seconds." mc6 = "Stopped in ie. Starting in uk." mc7 = "Moved. acme.go.tito.io is served from uk. It was offline for 41 seconds." mc8 = "uk could not start acme.go.tito.io: the landing ran past its deadline. Moving it back to ie." mc9 = "acme.go.tito.io is back in ie and serving. Nothing was lost." mc11 = "acme.go.tito.io could not start in uk or in ie. It is offline. Run carlos move resolve --host acme.go.tito.io --retry once the cause is fixed." mc12 = "Ready to cut. Run carlos move cut --host acme.go.tito.io when you are ready." ) func wantLines(t *testing.T, out string, lines ...string) { t.Helper() for _, l := range lines { if !strings.Contains(out, l+"\n") { t.Errorf("output lacks %q:\n%s", l, out) } } } func wantNoLine(t *testing.T, out string, lines ...string) { t.Helper() for _, l := range lines { if strings.Contains(out, l) { t.Errorf("output holds %q:\n%s", l, out) } } } // Exit codes (preflight L24): 0 done, 3 rolled back or aborted, 4 stuck or // waiting, 5 refused. Each sentence comes from the state it describes, and // the outcome only from last, bound to this move's id. func TestMoveWaitExitCodes(t *testing.T) { rb := frame{"id": "rb_x", "reason": "the landing ran past its deadline", "target_phase": "cleaning"} report := frame{"checks": frame{"t1_protocol": frame{"ok": true, "reason": ""}}} t.Run("rolled back", func(t *testing.T) { // The detail is read on each change of state or phase (L19): the // target's report is there when the move is first seen preparing. frames := []frame{none(nil), inFlight("preparing", nil, frame{"target_report": report})} c := &fakeMoveConsole{frames: append(frames, inFlight("ready", nil, nil), inFlight("cutting", frame{"cut_phase": "admitted", "cut_budget_s": 45}, nil), inFlight("landing", frame{"cut_phase": "flipped", "landing_phase": "probing"}, nil), inFlight("rolling_back", frame{"rollback": rb}, nil), none(outcome(tMoveID, "rolled_back", 0)), )} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 3 { t.Fatalf("exit %d, want 3:\n%s", code, out) } wantLines(t, out, mc1, mc2, mc3, mc5, mc6, mc8, mc9) if m := c.mutations(); len(m) != 1 || m[0] != "POST "+tMovePth+` {"cut":"auto","region":"uk"}` { t.Fatalf("mutations %q", m) } }) t.Run("done", func(t *testing.T) { // After the close the record's region is the target's: M-C7 names it. done := none(outcome(tMoveID, "done", 41.2)) done["detail"].(frame)["region"] = "uk" c := &fakeMoveConsole{frames: []frame{ none(nil), inFlight("preparing", nil, nil), inFlight("cutting", frame{"cut_phase": "stopping", "cut_budget_s": 45}, nil), inFlight("committed", frame{"cut_phase": "flipped"}, nil), done, }} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 0 { t.Fatalf("exit %d, want 0:\n%s", code, out) } wantLines(t, out, mc1, mc5, mc6, mc7) }) t.Run("stuck", func(t *testing.T) { c := &fakeMoveConsole{frames: []frame{ none(nil), inFlight("stuck", frame{"stuck_reason": "the recovery probe ran out of attempts"}, nil), }} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 4 { t.Fatalf("exit %d, want 4:\n%s", code, out) } wantLines(t, out, mc11, "the recovery probe ran out of attempts") }) t.Run("rolling back past the release deadline", func(t *testing.T) { past := time.Now().Add(-time.Minute).UTC().Format(time.RFC3339) late := frame{"id": "rb_x", "reason": "the landing ran past its deadline", "target_phase": "releasing", "release_deadline": past} c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("rolling_back", frame{"rollback": late}, nil)}} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 4 { t.Fatalf("exit %d, want 4:\n%s", code, out) } wantLines(t, out, mc8, "Waiting for uk-box in uk to confirm it has stopped acme.go.tito.io. acme.go.tito.io is offline.") }) t.Run("refused at the request", func(t *testing.T) { c := &fakeMoveConsole{frames: []frame{none(nil)}, answer: func(string, string, string) (int, any) { return http.StatusConflict, map[string]string{"error": "the instance is already in uk"} }} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 5 { t.Fatalf("exit %d, want 5:\n%s", code, out) } wantLines(t, out, "the instance is already in uk") }) t.Run("abort", func(t *testing.T) { c := &fakeMoveConsole{frames: []frame{ inFlight("preparing", nil, nil), inFlight("aborting", nil, nil), none(outcome(tMoveID, "aborted", 0)), }, answer: func(string, string, string) (int, any) { return http.StatusAccepted, frame{"state": "aborting", "target": "uk", "id": tMoveID, "protocol": "phase1"} }} code, out := runMoveAgainst(t, c, "abort", "--host", tHost) if code != 3 { t.Fatalf("exit %d, want 3:\n%s", code, out) } wantLines(t, out, "The move was aborted. acme.go.tito.io stays in ie.") if m := c.mutations(); len(m) != 1 || !strings.HasPrefix(m[0], "DELETE "+tMovePth+" ") { t.Fatalf("mutations %q", m) } }) t.Run("abort after the cut started", func(t *testing.T) { const mc13 = "This move has started its cut. It will finish or move back on its own." c := &fakeMoveConsole{frames: []frame{inFlight("landing", nil, nil)}, answer: func(string, string, string) (int, any) { return http.StatusConflict, map[string]string{"error": mc13} }} code, out := runMoveAgainst(t, c, "abort", "--host", tHost) if code != 5 { t.Fatalf("exit %d, want 5:\n%s", code, out) } wantLines(t, out, mc13) }) t.Run("cut and resolve post to their doors", func(t *testing.T) { for _, v := range []struct { args []string want string }{ {[]string{"cut", "--host", tHost, "--no-wait"}, "POST " + tMovePth + "/cut {}"}, {[]string{"resolve", "--host", tHost, "--retry", "--no-wait"}, "POST " + tMovePth + `/resolve {"retry":true}`}, } { c := &fakeMoveConsole{frames: []frame{inFlight("ready", nil, nil)}} code, out := runMoveAgainst(t, c, v.args...) if code != 0 { t.Fatalf("%v: exit %d:\n%s", v.args, code, out) } if m := c.mutations(); len(m) != 1 || m[0] != v.want { t.Fatalf("%v: mutations %q, want %q", v.args, m, v.want) } } }) } // Review Focus 1: the outcome is never inferred from the last state seen. A // wait that saw only preparing still reads rolled_back from last, bound to // this move's id; a last of another move is "not recorded". func TestMoveWaitSurvivesMissedStates(t *testing.T) { c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", nil, nil), none(outcome(tMoveID, "rolled_back", 0))}} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 3 { t.Fatalf("exit %d, want 3:\n%s", code, out) } wantLines(t, out, mc9) // The rollback's reason was never seen, so M-C8 (a present-tense line // with a reason) is not invented after the fact. wantNoLine(t, out, "could not start") for _, last := range []frame{outcome("mv_another", "done", 3), nil} { c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", nil, nil), none(last)}} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 4 { t.Fatalf("last %v: exit %d, want 4:\n%s", last, code, out) } wantLines(t, out, "This move's outcome is not recorded.") wantNoLine(t, out, "Moved.", "back in") } // Another move in flight, and ours closed: ours is read from last. other := inFlight("preparing", nil, nil) other["id"] = "mv_another" other["last"] = outcome(tMoveID, "done", 41.2) other["detail"].(frame)["region"] = "uk" c = &fakeMoveConsole{frames: []frame{none(nil), other}} code, out = runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 0 { t.Fatalf("exit %d, want 0:\n%s", code, out) } wantLines(t, out, mc7) } func TestMoveManualCutPrintsM12AndStops(t *testing.T) { c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", frame{"cut": "manual"}, nil), inFlight("ready", frame{"cut": "manual"}, nil)}} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk", "--manual-cut") if code != 4 { t.Fatalf("exit %d, want 4:\n%s", code, out) } wantLines(t, out, mc12) if m := c.mutations(); len(m) != 1 || m[0] != "POST "+tMovePth+` {"cut":"manual","region":"uk"}` { t.Fatalf("mutations %q: want the request and no cut", m) } } // L4: a console whose answer lacks the Phase 1 marker gets no mutation. func TestMoveRefusesAConsoleWithoutPhase1(t *testing.T) { legacy := frame{"state": "none"} for _, args := range [][]string{ {"--host", tHost, "--to", "uk"}, {"cut", "--host", tHost}, {"abort", "--host", tHost}, {"resolve", "--host", tHost, "--retry"}, {"status", "--host", tHost}, } { c := &fakeMoveConsole{frames: []frame{legacy}} code, out := runMoveAgainst(t, c, args...) if code != 5 { t.Fatalf("%v: exit %d, want 5:\n%s", args, code, out) } wantLines(t, out, "Moves are not enabled on this console.") if m := c.mutations(); len(m) != 0 { t.Fatalf("%v: mutations %q, want none", args, m) } } } // L18: relocating with waiting_for_box prints M-C22 and exits 4, never // waiting for the box. func TestMoveWaitReportsWaitingForABox(t *testing.T) { c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("relocating", nil, frame{"waiting_for_box": "ie"})}} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 4 { t.Fatalf("exit %d, want 4:\n%s", code, out) } wantLines(t, out, "acme.go.tito.io is waiting for a box in ie. Its last box was terminated before the move finished. Its data up to the last completed upload is kept in ie. It will start when a box joins the region, and any writes after that upload will be reported then.") c = &fakeMoveConsole{frames: []frame{none(nil), inFlight("relocating", nil, nil)}} code, out = runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 4 { t.Fatalf("relocating with a box: exit %d, want 4:\n%s", code, out) } wantLines(t, out, "acme.go.tito.io is relocating after its box was terminated. This move will not finish.") } // L19: the wait polls the flat view and reads detail only when the state // changes (or now and then). A 429 backs off, says so once, and never ends // the wait. func TestMoveWaitBacksOffAndReadsDetailOnChange(t *testing.T) { frames := []frame{none(nil)} for i := 0; i < 10; i++ { frames = append(frames, inFlight("preparing", nil, nil)) } done := none(outcome(tMoveID, "done", 41.2)) done["detail"].(frame)["region"] = "uk" frames = append(frames, done) c := &fakeMoveConsole{frames: frames, tooMany: 2} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 0 { t.Fatalf("exit %d, want 0:\n%s", code, out) } if n := strings.Count(out, "The console is limiting requests. Checking every 10 seconds."); n != 1 { t.Fatalf("the back-off line appears %d times, want once:\n%s", n, out) } if c.details > 4 { t.Fatalf("%d detail reads over %d flat polls; detail is read on a change", c.details, c.flatGets) } } // L25: carlos move refuses to run as root, before any request. func TestMoveRefusesRoot(t *testing.T) { c := &fakeMoveConsole{frames: []frame{none(nil)}} ts := httptest.NewServer(c) defer ts.Close() var out bytes.Buffer called := false deps := moveDeps{session: func(string) (*apiSession, error) { called = true; return nil, nil }, euid: func() int { return 0 }, poll: time.Millisecond} if code := moveMain(context.Background(), []string{"--host", tHost, "--to", "uk", "--app", "titogo"}, &out, deps); code != 5 { t.Fatalf("exit %d, want 5:\n%s", code, out.String()) } if called || c.flatGets != 0 { t.Fatal("a refused root run reached the console") } wantLines(t, out.String(), "Run carlos move as yourself, not as root. It uses your own credentials.") } // --no-wait answers from the mutation's own answer, and reads nothing after // it (fix round 2, ruling 2): a detail read that fails or is rate limited // cannot change the exit of a command the console accepted. func TestMoveNoWait(t *testing.T) { c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", nil, nil)}} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk", "--no-wait") if code != 0 { t.Fatalf("exit %d:\n%s", code, out) } wantLines(t, out, "Move "+tMoveID+" of acme.go.tito.io is preparing.") wantNoLine(t, out, mc2) } func TestMoveNoWaitReadsNothingAfterAcceptance(t *testing.T) { for _, status := range []int{http.StatusInternalServerError, http.StatusTooManyRequests} { for _, args := range [][]string{ {"--host", tHost, "--to", "uk", "--no-wait"}, {"cut", "--host", tHost, "--no-wait"}, {"abort", "--host", tHost, "--no-wait"}, {"resolve", "--host", tHost, "--retry", "--no-wait"}, } { c := &fakeMoveConsole{frames: []frame{inFlight("ready", nil, nil)}, detailStatus: status} if args[0] == "--host" { c.frames = []frame{none(nil)} } code, out := runMoveAgainst(t, c, args...) if code != 0 || c.details != 0 { t.Fatalf("%v with detail answering %d: exit %d after %d detail reads, want 0 and none:\n%s", args, status, code, c.details, out) } } } } // status: one read, the state in a line, exit 0; --wait waits for this // move's outcome; with nothing in flight and nothing closed, it says so. func TestMoveStatus(t *testing.T) { c := &fakeMoveConsole{frames: []frame{inFlight("preparing", nil, nil)}} code, out := runMoveAgainst(t, c, "status", "--host", tHost) if code != 0 { t.Fatalf("exit %d:\n%s", code, out) } wantLines(t, out, "Move "+tMoveID+" of acme.go.tito.io is preparing.", mc2) if m := c.mutations(); len(m) != 0 { t.Fatalf("status mutated: %q", m) } c = &fakeMoveConsole{frames: []frame{none(nil)}} if code, out := runMoveAgainst(t, c, "status", "--host", tHost, "--wait"); code != 0 || !strings.Contains(out, "No move is in flight for acme.go.tito.io.\n") { t.Fatalf("status --wait with nothing = %d:\n%s", code, out) } // Resumed after a disconnect: the first read is none, and last is the // move this operator was waiting on. closed := none(outcome(tMoveID, "rolled_back", 0)) c = &fakeMoveConsole{frames: []frame{closed}} // The wait never saw the move, so it never learnt its regions, and the // record's region now is not taken for them (fix round 2, ruling 3). if code, out := runMoveAgainst(t, c, "status", "--host", tHost, "--wait"); code != 3 || !strings.Contains(out, "Move "+tMoveID+" of acme.go.tito.io is rolled_back.\n") || strings.Contains(out, "back in") { t.Fatalf("status --wait after a rollback = %d:\n%s", code, out) } } func TestParseMoveArgs(t *testing.T) { for _, c := range []struct { args []string ok bool verb string }{ {[]string{"--host", tHost, "--to", "uk"}, true, "request"}, {[]string{"--host", tHost, "--to", "uk", "--manual-cut", "--no-wait"}, true, "request"}, {[]string{"status", "--host", tHost}, true, "status"}, {[]string{"status", "--host", tHost, "--wait"}, true, "status"}, {[]string{"cut", "--host", tHost}, true, "cut"}, {[]string{"abort", "--host", tHost}, true, "abort"}, {[]string{"resolve", "--host", tHost, "--retry"}, true, "resolve"}, {[]string{"--to", "uk"}, false, ""}, {[]string{"--host", tHost}, false, ""}, {[]string{"resolve", "--host", tHost}, false, ""}, {[]string{"cut", "--host", tHost, "--to", "uk"}, false, ""}, {[]string{"resolve", "--host", tHost, "--declare-lost", "uk-box"}, true, "resolve"}, {[]string{"resolve", "--host", tHost, "--declare-lost", "uk-box", "--retry"}, false, ""}, } { got, err := parseMoveArgs(c.args, io.Discard) if (err == nil) != c.ok || (c.ok && got.verb != c.verb) { t.Errorf("parseMoveArgs(%q) = %+v, %v; want ok %v verb %q", c.args, got, err, c.ok, c.verb) } } } // L16 (Task 6 folded in): every fenced `carlos move` line in the CLI page's // section parses through the CLI's own parser, the section names every verb, // and it states the four exit codes. func TestMoveDocsNameEveryVerb(t *testing.T) { raw, err := os.ReadFile("docs/site/cli.md") if err != nil { t.Fatal(err) } page := string(raw) start := strings.Index(page, "\n### carlos move\n") if start < 0 { t.Fatal("docs/site/cli.md has no ### carlos move section") } section := page[start+1:] if end := strings.Index(section[4:], "\n## "); end >= 0 { section = section[:end+4] } if end := strings.Index(section[4:], "\n### "); end >= 0 { section = section[:end+4] } verbs := map[string]bool{} inFence := false for _, line := range strings.Split(section, "\n") { if strings.HasPrefix(line, "```") { inFence = !inFence continue } fields := strings.Fields(line) if !inFence || len(fields) < 2 || fields[0] != "carlos" || fields[1] != "move" { continue } args := fields[2:] for i, a := range args { // [--flag] is an optional flag, and a placeholder stands for a // value. a = strings.TrimSuffix(strings.TrimPrefix(a, "["), "]") if strings.HasPrefix(a, "<") { a = "x" } args[i] = a } got, err := parseMoveArgs(args, io.Discard) if err != nil { t.Errorf("the page's example %q does not parse: %v", line, err) continue } verbs[got.verb] = true } for _, v := range []string{"request", "status", "cut", "abort", "resolve"} { if !verbs[v] { t.Errorf("the section has no example of %s", v) } } for _, code := range []string{"`0`", "`3`", "`4`", "`5`"} { if !strings.Contains(section, code) { t.Errorf("the section does not state exit code %s", code) } } } // Ruling 1 (fix round 1): a 429 from request, cut, abort or retry comes // before the handler, so the same call is sent again after the back-off, // said once; the verb never exits on it. func TestMoveMutationRetriesOn429(t *testing.T) { for _, v := range []struct { args []string want string }{ {[]string{"--host", tHost, "--to", "uk", "--no-wait"}, "POST " + tMovePth + ` {"cut":"auto","region":"uk"}`}, {[]string{"cut", "--host", tHost, "--no-wait"}, "POST " + tMovePth + "/cut {}"}, {[]string{"abort", "--host", tHost, "--no-wait"}, "DELETE " + tMovePth + " "}, {[]string{"resolve", "--host", tHost, "--retry", "--no-wait"}, "POST " + tMovePth + `/resolve {"retry":true}`}, } { limited := 2 c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", nil, nil)}} c.answer = func(string, string, string) (int, any) { if limited > 0 { limited-- return http.StatusTooManyRequests, map[string]string{"error": "too many instance requests. wait a minute and try again"} } return http.StatusAccepted, frame{"state": "preparing", "target": "uk", "id": tMoveID, "protocol": "phase1"} } if v.args[0] == "cut" || v.args[0] == "abort" || v.args[0] == "resolve" { c.frames = []frame{inFlight("ready", nil, nil)} } code, out := runMoveAgainst(t, c, v.args...) if code != 0 { t.Fatalf("%v: exit %d, want 0:\n%s", v.args, code, out) } m := c.mutations() if len(m) != 3 || m[0] != v.want || m[1] != v.want || m[2] != v.want { t.Fatalf("%v: mutations %q, want the same call three times", v.args, m) } if n := strings.Count(out, "The console is limiting requests. Checking every 10 seconds."); n != 1 { t.Fatalf("%v: the back-off line appears %d times, want once:\n%s", v.args, n, out) } wantNoLine(t, out, "too many instance requests") } } // Ruling 3: the outcome's regions are this move's own, bound from its own // answers, never from a later move's record. Here a second move flips to fr // while the first one's outcome is read. func TestMoveOutcomeRegionsAreThisMoves(t *testing.T) { second := func(last frame) frame { f := inFlight("landing", frame{"id": "mv_another", "from": "uk", "to": "fr", "landing_phase": "probing"}, frame{"from": "uk", "region": "fr"}) f["id"], f["target"], f["last"] = "mv_another", "fr", last return f } for _, c := range []struct { name string last frame code int want string }{ {"done", outcome(tMoveID, "done", 41.2), 0, mc7}, {"rolled back", outcome(tMoveID, "rolled_back", 0), 3, mc9}, {"aborted", outcome(tMoveID, "aborted", 0), 3, "The move was aborted. acme.go.tito.io stays in ie."}, } { fc := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", nil, nil), second(c.last)}} code, out := runMoveAgainst(t, fc, "--host", tHost, "--to", "uk") if code != c.code { t.Fatalf("%s: exit %d, want %d:\n%s", c.name, code, c.code, out) } wantLines(t, out, c.want) wantNoLine(t, out, "from fr", "in fr") } // Bound from the request's own start answer, before any poll: the // first poll already shows the second move, flipped to fr. fc := &fakeMoveConsole{frames: []frame{none(nil), second(outcome(tMoveID, "done", 41.2))}, startDetail: inFlight("preparing", nil, nil)} code, out := runMoveAgainst(t, fc, "--host", tHost, "--to", "uk") if code != 0 { t.Fatalf("bound at the start: exit %d:\n%s", code, out) } wantLines(t, out, mc1, mc7) wantNoLine(t, out, "from fr", "in fr") } // Ruling 4: a rollback the wait missed says M-C8 only when the outcome // carries the reason, and never invents one. func TestMoveMissedRollbackSaysItsReasonOnlyFromTheOutcome(t *testing.T) { withReason := outcome(tMoveID, "rolled_back", 0) withReason["reason"] = "the landing ran past its deadline" c := &fakeMoveConsole{frames: []frame{none(nil), inFlight("preparing", nil, nil), none(withReason)}} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 3 { t.Fatalf("exit %d, want 3:\n%s", code, out) } wantLines(t, out, mc8, mc9) if strings.Index(out, mc8) > strings.Index(out, mc9) { t.Fatalf("M-C8 after M-C9:\n%s", out) } } // L19 strictly: the detail is read only when the state or the phase changes, // whatever else (updated_at) does. A release deadline that passes while // nothing changes is still seen, from the detail already read. func TestMoveWaitReadsDetailOnlyOnStateOrPhase(t *testing.T) { frames := []frame{none(nil)} for i := 0; i < 6; i++ { f := inFlight("preparing", nil, nil) f["updated_at"] = int64(1000 + i) // an unrelated record update each poll frames = append(frames, f) } for _, p := range []string{"admitted", "admitted", "stopping", "stopping"} { frames = append(frames, inFlight("cutting", frame{"cut_phase": p, "cut_budget_s": 45}, nil)) } done := none(outcome(tMoveID, "done", 41.2)) frames = append(frames, done) c := &fakeMoveConsole{frames: frames} code, out := runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 0 { t.Fatalf("exit %d:\n%s", code, out) } // One detail read at the request's start, then one per change: preparing, // cutting/admitted, cutting/stopping, none. if c.details != 5 { t.Fatalf("%d detail reads, want 5 (the start and four changes) over %d flat polls", c.details, c.flatGets) } soon := time.Now().Add(150 * time.Millisecond).UTC().Format(time.RFC3339Nano) rb := frame{"id": "rb_x", "reason": "the landing ran past its deadline", "target_phase": "releasing", "release_deadline": soon} c = &fakeMoveConsole{frames: []frame{none(nil), inFlight("rolling_back", frame{"rollback": rb}, nil)}} code, out = runMoveAgainst(t, c, "--host", tHost, "--to", "uk") if code != 4 { t.Fatalf("deadline passing: exit %d, want 4:\n%s", code, out) } wantLines(t, out, "Waiting for uk-box in uk to confirm it has stopped acme.go.tito.io. acme.go.tito.io is offline.") if c.details > 2 { t.Fatalf("%d detail reads while nothing changed, want at most 2", c.details) } } // Ruling 6: the flag help is said through moveFlagHelp, the copy gate's // registered help emission: on -h, and after a bad flag, with the flag // package's own error. func TestMoveFlagHelp(t *testing.T) { var out bytes.Buffer if _, err := parseMoveArgs([]string{"-h"}, &out); !errors.Is(err, flag.ErrHelp) { t.Fatalf("-h: %v", err) } for _, want := range []string{"-host", "the instance's host name", "-to", "-manual-cut"} { if !strings.Contains(out.String(), want) { t.Errorf("help lacks %q:\n%s", want, out.String()) } } out.Reset() if _, err := parseMoveArgs([]string{"resolve", "--host", tHost, "--instance-id", "i-1"}, &out); err == nil { t.Fatal("--instance-id parsed") } if !strings.Contains(out.String(), "flag provided but not defined: -instance-id") || !strings.Contains(out.String(), "-retry") { t.Errorf("a bad flag's help:\n%s", out.String()) } } // Fix round 2, ruling 3: a region the wait never learnt is not filled in. A // known origin with an unknown destination says M-C9 alone, even when the // outcome carries a reason, since M-C8 names the destination. func TestMoveOutcomeNeverGuessesARegion(t *testing.T) { var out bytes.Buffer m := &mover{out: &out, host: tHost} w := newMoveWatch(m, tMoveID) w.fromRegion = "ie" last := &moveOutcome{ID: tMoveID, Outcome: "rolled_back", Reason: "the landing ran past its deadline"} a := moveAnswer{moveFlat: moveFlat{State: "none", Last: last}, Detail: &moveDetailView{Region: "fr"}} if code := w.outcome(a); code != 3 { t.Fatalf("exit %d", code) } if got := out.String(); got != mc9+"\n" { t.Fatalf("output %q, want M-C9 alone", got) } } // Fix round 2, ruling 1: what the CLI says when the console is not reached, // answers with no reason, or the session cannot be made, is this file's own // copy; only the console's own sentence is passed on, as server text. func TestMoveFixedSentences(t *testing.T) { // Nothing listens: the transport fails. ts := httptest.NewServer(http.NotFoundHandler()) dead := ts.URL ts.Close() deps := moveDeps{ session: func(string) (*apiSession, error) { return &apiSession{c: newConsoleClient(dead, "cli-token"), console: dead, address: "op@keymail.dev"}, nil }, euid: func() int { return 1000 }, poll: time.Millisecond, } var out bytes.Buffer if code := moveMain(context.Background(), []string{"status", "--host", tHost, "--account", "bab", "--app", "titogo"}, &out, deps); code != 1 { t.Fatalf("unreachable console: exit %d:\n%s", code, out.String()) } wantLines(t, out.String(), "Could not reach the console. Check the connection and try again.") // The session cannot be made: one sentence of this file. out.Reset() deps.session = func(string) (*apiSession, error) { return nil, errors.New("unreviewed resolveMode text") } if code := moveMain(context.Background(), []string{"status", "--host", tHost, "--account", "bab", "--app", "titogo"}, &out, deps); code != 1 { t.Fatalf("no session: exit %d:\n%s", code, out.String()) } wantLines(t, out.String(), "carlos move could not sign in to a console. Run carlos auth login, or check --console, CARLOS_CONSOLE and CARLOS_SERVICE_TOKEN.") wantNoLine(t, out.String(), "unreviewed resolveMode text") // A refusal with no JSON reason (the family's 404), and a 5xx with none. for _, c := range []struct { status int code int want string }{ {http.StatusNotFound, 5, "Not found. Check --account and --app, and that you are a member of the instance's account."}, {http.StatusBadGateway, 1, "The console answered 502 and gave no reason."}, } { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(c.status) _, _ = w.Write([]byte("plain text")) })) deps.session = func(string) (*apiSession, error) { return &apiSession{c: newConsoleClient(ts.URL, "cli-token"), console: ts.URL, address: "op@keymail.dev"}, nil } out.Reset() code := moveMain(context.Background(), []string{"status", "--host", tHost, "--account", "bab", "--app", "titogo"}, &out, deps) ts.Close() if code != c.code { t.Fatalf("%d with no reason: exit %d, want %d:\n%s", c.status, code, c.code, out.String()) } wantLines(t, out.String(), c.want) wantNoLine(t, out.String(), "plain text") } } // ---- Task 1b: carlos move resolve --declare-lost (claim Plan 10's door) ---- // fakeEC2 answers DescribeInstances for one instance in one state. type fakeEC2 struct { account, instance, state string calls *int } func (f fakeEC2) DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { *f.calls++ out := &ec2.DescribeInstancesOutput{Reservations: []ec2types.Reservation{{ OwnerId: aws.String(f.account), Instances: []ec2types.Instance{{InstanceId: aws.String(f.instance), State: &ec2types.InstanceState{Name: ec2types.InstanceStateName(f.state)}}}, }}} awsmiddleware.SetRequestIDMetadata(&out.ResultMetadata, "aws-request-1b") return out, nil } const tSignedChallenge = `{"envelope":"signed by the test console"}` // lossChallenge is an inspection as claim Plan 10's move door answers it. func lossChallenge(t *testing.T, transition string) hostresolve.Inspection { t.Helper() c := hostfence.Challenge{ ID: "loss-0123456789abcdef", Actor: "op@keymail.dev", Host: tHost, HolderIncarnation: "inc-1", HolderStamp: 3, HolderSession: 4, ClaimGen: 9, ClaimETag: "etag-claim", Op: hostfence.Operation{ID: tMoveID, Kind: hostclaim.KindLanding, Phase: "probing"}, Record: hostfence.RecordObservation{Key: "apps/bab/titogo/instances/acme.go.tito.io.json", ETag: "etag-record", CommitState: "present"}, Box: "uk-box", Enrollment: "0123456789abcdef0123456789abcdef", Cloud: hostfence.CloudBinding{Provider: "aws", Account: "111122223333", Region: "eu-west-2", InstanceID: "i-uk-box"}, Consequences: hostfence.Consequences{Transition: transition, PossibleWriteLoss: true}, } if transition == "destroyed-local-only" { c.Consequences = hostfence.Consequences{ Transition: "destroyed-local-only", DischargeLocal: []hostclaim.ObligationRef{{ID: "local", Op: tMoveID, Retirement: "retired-installation"}}, DestroyedStorageAssertion: []string{tMoveID + "/local/retired-installation"}, } } c, err := hostfence.PrepareChallenge(time.Now().Add(-time.Second), c) if err != nil { t.Fatal(err) } return hostresolve.Inspection{Challenge: json.RawMessage(tSignedChallenge), Details: c} } // lossConsole is a console in landing whose move door answers the // inspection, then the declaration (202, the flat view in rolling_back). func lossConsole(t *testing.T, in hostresolve.Inspection, declared *hostfence.Submission) *fakeMoveConsole { flipped := time.Date(2026, 10, 11, 9, 30, 0, 0, time.UTC) c := &fakeMoveConsole{frames: []frame{inFlight("landing", frame{"flipped_at": flipped.Format(time.RFC3339)}, nil)}} c.answer = func(method, path, body string) (int, any) { var req struct { DeclareLost string `json:"declare_lost"` Submission *hostfence.Submission `json:"submission"` } if path != tMovePth+"/resolve" || json.Unmarshal([]byte(body), &req) != nil || req.DeclareLost != "uk-box" { t.Errorf("unexpected %s %s %s", method, path, body) return http.StatusBadRequest, map[string]string{"error": "invalid resolution request"} } if req.Submission == nil { return http.StatusOK, in } *declared = *req.Submission return http.StatusAccepted, frame{"state": "rolling_back", "target": "uk", "id": tMoveID, "protocol": "phase1"} } return c } func TestMoveDeclareLost(t *testing.T) { const mc18 = "This declares uk-box lost. Anything acme.go.tito.io wrote on uk-box after 2026-10-11 09:30:00 UTC is lost. Type the host name to confirm." const localOnly = "This records that uk-box is gone with its disks. acme.go.tito.io keeps nothing there that it still needs. Type the host name to confirm." args := []string{"resolve", "--host", tHost, "--declare-lost", "uk-box", "--no-wait"} reader := func(state string, calls *int, region *string) func(*moveDeps) { return func(d *moveDeps) { d.stdin = strings.NewReader(tHost + "\n") d.ec2 = func(_ context.Context, r string) (awsinv.ExactEC2API, error) { *region = r return fakeEC2{account: "111122223333", instance: "i-uk-box", state: state, calls: calls}, nil } } } t.Run("move-target-lost", func(t *testing.T) { in := lossChallenge(t, "move-target-lost") var sub hostfence.Submission calls, region := 0, "" code, out := runMoveWith(t, lossConsole(t, in, &sub), reader("stopped", &calls, ®ion), args...) if code != 0 { t.Fatalf("exit %d:\n%s", code, out) } wantLines(t, out, mc18, "Move "+tMoveID+" of acme.go.tito.io is rolling_back.") wantNoLine(t, out, localOnly) if region != "eu-west-2" || calls != 1 { t.Fatalf("EC2 read in %q, %d calls; want the challenge's region, once", region, calls) } want := hostfence.OperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true} if string(sub.Challenge) != tSignedChallenge || sub.Confirmation != in.Details.ID || sub.RequestID != "aws-request-1b" || sub.Observation.State != "stopped" || sub.Observation.InstanceID != "i-uk-box" || sub.Attestation.MaintainComputeFence != want.MaintainComputeFence || sub.Attestation.ColdQuarantineStartOnly != want.ColdQuarantineStartOnly || len(sub.Attestation.Disks) != 0 { t.Fatalf("submission %+v", sub) } }) t.Run("destroyed-local-only", func(t *testing.T) { in := lossChallenge(t, "destroyed-local-only") var sub hostfence.Submission calls, region := 0, "" code, out := runMoveWith(t, lossConsole(t, in, &sub), reader("terminated", &calls, ®ion), args...) if code != 0 { t.Fatalf("exit %d:\n%s", code, out) } wantLines(t, out, localOnly) wantNoLine(t, out, "This declares uk-box lost.") if strings.Join(sub.Attestation.Disks, ",") != tMoveID+"/local/retired-installation" || sub.Confirmation != in.Details.ID { t.Fatalf("submission %+v", sub) } }) t.Run("a wrong host name refuses", func(t *testing.T) { in := lossChallenge(t, "move-target-lost") var sub hostfence.Submission c := lossConsole(t, in, &sub) code, out := runMoveWith(t, c, func(d *moveDeps) { d.stdin = strings.NewReader("acme.go.tito.io.evil\n") }, args...) if code != 5 { t.Fatalf("exit %d, want 5:\n%s", code, out) } wantLines(t, out, mc18, "That is not acme.go.tito.io. Nothing was declared.") if m := c.mutations(); len(m) != 1 || sub.Confirmation != "" { t.Fatalf("a refused confirmation declared: %q", m) } }) t.Run("a running box is refused before the declaration", func(t *testing.T) { in := lossChallenge(t, "move-target-lost") var sub hostfence.Submission calls, region := 0, "" c := lossConsole(t, in, &sub) code, out := runMoveWith(t, c, reader("running", &calls, ®ion), args...) if code != 5 { t.Fatalf("exit %d, want 5:\n%s", code, out) } wantLines(t, out, "AWS does not show uk-box stopped or terminated, so it cannot be declared lost.") if m := c.mutations(); len(m) != 1 { t.Fatalf("mutations %q: want the inspection only", m) } }) t.Run("no AWS credentials", func(t *testing.T) { in := lossChallenge(t, "move-target-lost") var sub hostfence.Submission c := lossConsole(t, in, &sub) code, out := runMoveWith(t, c, func(d *moveDeps) { d.stdin = strings.NewReader(tHost + "\n") d.ec2 = func(context.Context, string) (awsinv.ExactEC2API, error) { return nil, errors.New("no credential chain") } }, args...) if code != 5 { t.Fatalf("exit %d, want 5:\n%s", code, out) } wantLines(t, out, "declare-lost needs your AWS credentials to confirm the box is stopped or terminated.") wantNoLine(t, out, "no credential chain") }) // Fix round 1, finding 1: M-C18 is bound to the inspected move. A detail // that names another move, or another incarnation, than the inspection's // challenge refuses before anything is shown, read or declared. for _, c := range []struct { name string edit func(*hostresolve.Inspection) }{ {"another move", func(in *hostresolve.Inspection) { in.Details.Op.ID = "mv_bbbbbbbbbbbbbbbbbbbbbbbbbb" }}, {"another incarnation", func(in *hostresolve.Inspection) { in.Details.HolderIncarnation = "inc-other" }}, } { t.Run("the inspection names "+c.name, func(t *testing.T) { in := lossChallenge(t, "move-target-lost") c.edit(&in) var sub hostfence.Submission fc := lossConsole(t, in, &sub) fc.frames[0]["detail"].(frame)["move"].(frame)["incarnation"] = "inc-1" code, out := runMoveWith(t, fc, func(d *moveDeps) { d.stdin = strings.NewReader(tHost + "\n") }, args...) if code != 5 { t.Fatalf("exit %d, want 5:\n%s", code, out) } wantLines(t, out, "The move changed since it was read. Nothing was declared. Run the command again.") wantNoLine(t, out, "This declares uk-box lost.") if m := fc.mutations(); len(m) != 1 || sub.Confirmation != "" { t.Fatalf("mutations %q: want the inspection only", m) } }) } t.Run("the console refuses", func(t *testing.T) { c := &fakeMoveConsole{frames: []frame{inFlight("landing", nil, nil)}, answer: func(string, string, string) (int, any) { return http.StatusConflict, map[string]string{"error": "declare-lost is not enabled in this deployment yet"} }} code, out := runMoveAgainst(t, c, args...) if code != 5 { t.Fatalf("exit %d, want 5:\n%s", code, out) } wantLines(t, out, "declare-lost is not enabled in this deployment yet") }) } // The cut guard (Paul's ruling, 2026-10-11): a cut the source refused goes // back to ready with the reason recorded on the move, and `carlos move // status` says it. While the refusal is recorded (until a repin clears it) // the move is not ready to cut, so M-C12 is not said (fix round 1). A ready // move with no reason says M-C12 alone. func TestMoveStatusSaysWhyTheCutDidNotStart(t *testing.T) { for _, reason := range []string{provision.RefusedTargetUnarmed, provision.RefusedSourceUnarmed, provision.RefusedReplicaLayout} { c := &fakeMoveConsole{frames: []frame{inFlight("ready", frame{"cut": "manual", "reason": reason}, nil)}} code, out := runMoveAgainst(t, c, "status", "--host", tHost) if code != moveExitWaiting { t.Fatalf("exit %d:\n%s", code, out) } wantLines(t, out, "Move "+tMoveID+" of acme.go.tito.io is ready.", "The cut did not start: "+reason+".") wantNoLine(t, out, mc12) } c := &fakeMoveConsole{frames: []frame{inFlight("ready", frame{"cut": "manual"}, nil)}} code, out := runMoveAgainst(t, c, "status", "--host", tHost) if code != moveExitWaiting || strings.Contains(out, "did not start") { t.Fatalf("a ready move with no refusal = %d:\n%s", code, out) } wantLines(t, out, mc12) } // ---- carlos move offline ---- // tOffMovePth is the move door of the offline tests' instance, oHost of the // app shop. const tOffMovePth = "/api/cli/apps/bab/shop/instances/" + oHost + "/move" const tOfflinePth = tOffMovePth + "/offline" // offPost is one scripted answer of a mutating door; the last one repeats. type offPost struct { code int body any } // fakeOfflineConsole answers the offline doors in T6's shapes. Each GET of the // status door returns the next scripted view (the last repeats); each POST // returns the next scripted answer of its door. Anything outside the offline // doors is a 404, so a verb that read the Phase 1 door would fail. type fakeOfflineConsole struct { mu sync.Mutex gets []frame i int posts map[string][]offPost // keyed by the door's tail: "", "/seal", ... n map[string]int log []string // method path body } func (c *fakeOfflineConsole) ServeHTTP(w http.ResponseWriter, r *http.Request) { c.mu.Lock() defer c.mu.Unlock() write := func(code int, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) _ = json.NewEncoder(w).Encode(v) } body, _ := io.ReadAll(r.Body) c.log = append(c.log, r.Method+" "+strings.TrimPrefix(r.URL.Path, tOffMovePth)+" "+strings.TrimSpace(string(body))) if !strings.HasPrefix(r.URL.Path, tOfflinePth) { http.NotFound(w, r) return } if r.Method == http.MethodGet && r.URL.Path == tOfflinePth { n := c.i if n >= len(c.gets) { n = len(c.gets) - 1 } c.i++ write(http.StatusOK, c.gets[n]) return } tail := strings.TrimPrefix(r.URL.Path, tOfflinePth) list := c.posts[tail] if r.Method != http.MethodPost || len(list) == 0 { http.NotFound(w, r) return } if c.n == nil { c.n = map[string]int{} } k := c.n[tail] if k >= len(list) { k = len(list) - 1 } c.n[tail]++ write(list[k].code, list[k].body) } func (c *fakeOfflineConsole) requests() []string { c.mu.Lock() defer c.mu.Unlock() return append([]string(nil), c.log...) } // posted counts the POSTs a door got. func (c *fakeOfflineConsole) posted(tail string) int { c.mu.Lock() defer c.mu.Unlock() return c.n[tail] } func runOfflineAgainst(t *testing.T, c *fakeOfflineConsole, args ...string) (int, string) { t.Helper() ts := httptest.NewServer(c) t.Cleanup(ts.Close) var out bytes.Buffer deps := moveDeps{ session: func(string) (*apiSession, error) { return &apiSession{c: newConsoleClient(ts.URL, "cli-token"), console: ts.URL, address: "op@keymail.dev"}, nil }, euid: func() int { return 1000 }, poll: time.Millisecond, stdin: strings.NewReader(""), } args = append([]string{"offline"}, args...) args = append(args, "--host", oHost, "--account", "bab", "--app", "shop") ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() code := moveMain(ctx, args, &out, deps) return code, out.String() } // offView is the offline status view of a move in state, as T6 builds it. func offView(state string, extra frame) frame { f := frame{"host": oHost, "state": state, "id": tMoveID, "from": "ie", "to": "uk", "sealer": "ie-box", "boxes": []string{"ie-box"}} for k, v := range extra { f[k] = v } return f } func offDone(outcome string, offlineS float64) frame { return frame{"host": oHost, "state": "none", "last": frame{"id": tMoveID, "outcome": outcome, "at": "2026-10-20T10:00:00Z", "offline_s": offlineS}} } func offReport(box, role, phase string, fresh bool) frame { return frame{"box": box, "role": role, "phase": phase, "fresh": fresh} } const ( oHost = "acme.shop.example.com" oStarting = "Moving acme.shop.example.com from ie to uk." oOffline = "acme.shop.example.com is offline from now until the move opens or is aborted." oStopped = "Every box in ie has stopped acme.shop.example.com. Run carlos move offline seal --host acme.shop.example.com to copy it." oCopying = "Copying acme.shop.example.com to uk." oCopied = "Copied and checked. Run carlos move offline flip --host acme.shop.example.com to start it in uk." oFlipped = "Stopped in ie. Starting in uk." oLanded = "uk holds everything acme.shop.example.com needs. Nothing is served yet. Run carlos move offline open --host acme.shop.example.com to serve it from uk." oServed = "acme.shop.example.com is served from uk. Run carlos move offline finish --host acme.shop.example.com once ie has cleaned up." oBack = "Moving acme.shop.example.com back to ie." oPutBack = "Every box has put acme.shop.example.com back in ie. Run carlos move offline finish --host acme.shop.example.com to end the move." oWaitBox = "Waiting for ie-box in ie to confirm it has stopped acme.shop.example.com. acme.shop.example.com is offline." ) func TestParseMoveArgsOffline(t *testing.T) { for _, c := range []struct { args []string ok bool sub string }{ {[]string{"offline", "start", "--host", oHost, "--to", "uk"}, true, "start"}, {[]string{"offline", "stage", "--host", oHost, "--to", "uk"}, true, "stage"}, {[]string{"offline", "stage", "--host", oHost}, false, ""}, {[]string{"offline", "seal", "--host", oHost}, true, "seal"}, {[]string{"offline", "flip", "--host", oHost}, true, "flip"}, {[]string{"offline", "open", "--host", oHost}, true, "open"}, {[]string{"offline", "finish", "--host", oHost}, true, "finish"}, {[]string{"offline", "abort", "--host", oHost}, true, "abort"}, {[]string{"offline", "status", "--host", oHost, "--app", "x", "--account", "y"}, true, "status"}, {[]string{"offline", "start", "--host", oHost}, false, ""}, {[]string{"offline", "start", "--to", "uk"}, false, ""}, {[]string{"offline", "seal", "--host", oHost, "--to", "uk"}, false, ""}, {[]string{"offline", "seal", "--host", oHost, "--no-wait"}, false, ""}, {[]string{"offline", "seal", "--host", oHost, "extra"}, false, ""}, {[]string{"offline"}, false, ""}, {[]string{"offline", "cut", "--host", oHost}, false, ""}, {[]string{"offline", "--host", oHost}, false, ""}, } { got, err := parseMoveArgs(c.args, io.Discard) if (err == nil) != c.ok || (c.ok && (got.verb != "offline" || got.offline != c.sub)) { t.Errorf("parseMoveArgs(%q) = %+v, %v; want ok %v offline %q", c.args, got, err, c.ok, c.sub) } } var out bytes.Buffer if _, err := parseMoveArgs([]string{"offline", "cut", "--host", oHost}, &out); err == nil { t.Fatal("an unknown offline verb parsed") } wantLines(t, out.String(), "carlos move offline needs stage, start, seal, flip, open, finish, abort or status.") out.Reset() _, _ = parseMoveArgs([]string{"offline", "start", "--host", oHost}, &out) wantLines(t, out.String(), "carlos move needs --to .") // The Phase 1 verbs are untouched. if got, err := parseMoveArgs([]string{"abort", "--host", oHost}, io.Discard); err != nil || got.verb != "abort" || got.offline != "" { t.Errorf("carlos move abort = %+v, %v", got, err) } } func TestMoveOfflineStart(t *testing.T) { c := &fakeOfflineConsole{ posts: map[string][]offPost{"": {{202, offView("offline-stopping", frame{"waiting": oWaitBox})}}}, gets: []frame{ offView("offline-stopping", frame{"waiting": oWaitBox}), offView("offline-stopping", frame{"waiting": oWaitBox}), offView("offline-stopping", frame{"next": "seal"}), }, } code, out := runOfflineAgainst(t, c, "start", "--to", "uk") if code != 0 { t.Fatalf("exit %d:\n%s", code, out) } want := oStarting + "\n" + oOffline + "\n" + oWaitBox + "\n" + oStopped + "\n" if out != want { t.Fatalf("output:\n%s\nwant:\n%s", out, want) } reqs := c.requests() if reqs[0] != `POST {"region":"uk"}` && reqs[0] != `POST /offline {"region":"uk"}` { t.Fatalf("first request %q", reqs[0]) } for _, r := range reqs { if !strings.Contains(r, "/offline") { t.Errorf("a request left the offline doors: %q", r) } } } func TestMoveOfflineRefusals(t *testing.T) { for _, c := range []struct { name string code int body any exit int say string }{ {"claim console", 409, frame{"error": "offline moves are not available while host claims or Phase 1 moves are on"}, 5, "offline moves are not available while host claims or Phase 1 moves are on"}, {"not found", 404, "404 page not found", 5, "Not found. Check --account and --app, and that you are a member of the instance's account."}, {"console error", 502, frame{"error": "could not write the move. try again"}, 1, "could not write the move. try again"}, {"no reason", 500, frame{}, 1, "The console answered 500 and gave no reason."}, } { t.Run(c.name, func(t *testing.T) { f := &fakeOfflineConsole{posts: map[string][]offPost{"": {{c.code, c.body}}, "/seal": {{c.code, c.body}}}, gets: []frame{offView("offline-stopping", nil)}} for _, args := range [][]string{{"start", "--to", "uk"}, {"seal"}} { code, out := runOfflineAgainst(t, f, args...) if code != c.exit || out != c.say+"\n" { t.Errorf("%v: exit %d, output %q; want %d, %q", args, code, out, c.exit, c.say) } } for _, r := range f.requests() { if strings.HasPrefix(r, "GET") { t.Errorf("a refused verb polled: %q", r) } } }) } } func TestMoveOfflineSeal(t *testing.T) { proc := "ie-box still runs a process for acme.shop.example.com. wait for it to stop, then flip" c := &fakeOfflineConsole{ posts: map[string][]offPost{"/seal": {{202, offView("offline-sealing", nil)}}}, gets: []frame{ offView("offline-sealing", nil), offView("offline-sealing", nil), offView("offline-sealed", frame{"waiting": proc}), offView("offline-sealed", frame{"waiting": proc}), offView("offline-sealed", frame{"next": "flip"}), }, } code, out := runOfflineAgainst(t, c, "seal") if want := oCopying + "\n" + proc + "\n" + oCopied + "\n"; code != 0 || out != want { t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) } if got := c.requests()[0]; !strings.HasPrefix(got, "POST /offline/seal") { t.Errorf("first request %q", got) } } func TestMoveOfflineSealFails(t *testing.T) { item := func(status, detail string) frame { return frame{"id": "files", "owner": "seal", "status": status, "detail": detail} } for _, c := range []struct { name string extra frame say string }{ {"item", frame{"items": []frame{item("ok", ""), item("failed", "a file over 256 MiB")}}, "The move could not finish a step: a file over 256 MiB. Run carlos move offline status --host acme.shop.example.com for each item."}, {"item without detail", frame{"items": []frame{item("failed", "")}}, "The move could not finish a step: files. Run carlos move offline status --host acme.shop.example.com for each item."}, {"failed flag and report", frame{"failed": true, "reports": []frame{{"box": "ie-box", "role": "sealer", "phase": "failed", "fresh": true, "items": frame{"db": frame{"ok": true, "reason": ""}, "files": frame{"ok": false, "reason": "special file in the archive"}}}}}, "The move could not finish a step: special file in the archive. Run carlos move offline status --host acme.shop.example.com for each item."}, } { t.Run(c.name, func(t *testing.T) { f := &fakeOfflineConsole{ posts: map[string][]offPost{"/seal": {{202, offView("offline-sealing", nil)}}}, gets: []frame{offView("offline-sealing", c.extra)}, } code, out := runOfflineAgainst(t, f, "seal") if want := oCopying + "\n" + c.say + "\n"; code != 4 || out != want { t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, want) } }) } } func TestMoveOfflineFlip(t *testing.T) { c := &fakeOfflineConsole{ posts: map[string][]offPost{"/flip": {{202, offView("offline-landing", nil)}}}, gets: []frame{ offView("offline-landing", nil), offView("offline-landed", frame{"waiting": "this move is offline-landed. run carlos move offline status --host acme.shop.example.com to see what it waits for"}), offView("offline-landed", frame{"next": "open"}), }, } code, out := runOfflineAgainst(t, c, "flip") want := oFlipped + "\nthis move is offline-landed. run carlos move offline status --host acme.shop.example.com to see what it waits for\n" + oLanded + "\n" if code != 0 || out != want { t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) } } func TestMoveOfflineFlipFails(t *testing.T) { failedItem := frame{"id": "db", "owner": "landing", "status": "failed", "detail": "the listing differs"} for _, c := range []struct { name string extra frame say string }{ {"item", frame{"items": []frame{failedItem}}, "uk could not take acme.shop.example.com: the listing differs. Run carlos move offline abort --host acme.shop.example.com to move it back."}, {"failed flag, check in the report", frame{"failed": true, "reports": []frame{{"box": "uk-box", "role": "target", "phase": "failed", "fresh": true, "items": frame{"env-pin": frame{"ok": false, "reason": "the app settings differ"}}}}}, "uk could not take acme.shop.example.com: the app settings differ. Run carlos move offline abort --host acme.shop.example.com to move it back."}, {"failed flag only", frame{"failed": true}, "uk could not take acme.shop.example.com: failed. Run carlos move offline abort --host acme.shop.example.com to move it back."}, } { t.Run(c.name, func(t *testing.T) { f := &fakeOfflineConsole{ posts: map[string][]offPost{"/flip": {{202, offView("offline-landing", nil)}}}, gets: []frame{offView("offline-landing", nil), offView("offline-landing", c.extra)}, } code, out := runOfflineAgainst(t, f, "flip") if want := oFlipped + "\n" + c.say + "\n"; code != 4 || out != want { t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, want) } if f.posted("/flip") != 1 { t.Error("flip posted more than once") } }) } } func TestMoveOfflineOpen(t *testing.T) { src := func(phase string) frame { return offReport("ie-box", "source", phase, true) } tgt := func(phase string) frame { return offReport("uk-box", "target", phase, true) } c := &fakeOfflineConsole{ posts: map[string][]offPost{"/open": {{202, offView("offline-open", nil)}}}, gets: []frame{ offView("offline-open", frame{"reports": []frame{src("stopped"), tgt("landed")}}), offView("offline-open", frame{"reports": []frame{src("stopped"), tgt("landed")}}), offView("offline-open", frame{"reports": []frame{src("stopped"), tgt("opened")}}), offView("offline-open", frame{"next": "finish", "reports": []frame{src("parked"), tgt("opened")}}), }, } code, out := runOfflineAgainst(t, c, "open") // The served line waits for the target's opened report. want := "Waiting for ie-box to clean up acme.shop.example.com.\n" + oServed + "\n" if code != 0 || out != want { t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) } // Or for finish to be next, when the target's report is not in the view. c = &fakeOfflineConsole{ posts: map[string][]offPost{"/open": {{202, offView("offline-open", nil)}}}, gets: []frame{offView("offline-open", frame{"next": "finish"})}, } if code, out := runOfflineAgainst(t, c, "open"); code != 0 || out != "Waiting for ie-box to clean up acme.shop.example.com.\n"+oServed+"\n" { t.Fatalf("exit %d, output:\n%s", code, out) } } // finish posts twice for a move that was open: the first starts the clean-up // and answers offline-finishing; once the boxes are gone and the tombstone is // lifted, the status shows finish again and the second post ends the move. func TestMoveOfflineFinishPostsTwice(t *testing.T) { c := &fakeOfflineConsole{ posts: map[string][]offPost{"/finish": { {202, offView("offline-finishing", frame{"reports": []frame{offReport("ie-box", "source", "parked", true)}})}, {202, offDone("done", 1800)}, }}, gets: []frame{ offView("offline-open", frame{"next": "finish"}), offView("offline-finishing", frame{"reports": []frame{offReport("ie-box", "source", "parked", true)}}), offView("offline-finishing", frame{"reports": []frame{offReport("ie-box", "source", "gone", true), offReport("ie-box", "sealer", "sealed", true)}}), offView("offline-finishing", frame{"next": "finish", "reports": []frame{offReport("ie-box", "source", "gone", true), offReport("ie-box", "sealer", "lifted", true)}}), }, } code, out := runOfflineAgainst(t, c, "finish") want := "Waiting for ie-box to clean up acme.shop.example.com.\nMoved. acme.shop.example.com is served from uk. It was offline for 1800 seconds.\n" if code != 0 || out != want { t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) } if n := c.posted("/finish"); n != 2 { t.Fatalf("finish posted %d times, want 2:\n%v", n, c.requests()) } } // A finish before the boxes have parked does not ask the door early. func TestMoveOfflineFinishWaitsBeforeThePost(t *testing.T) { c := &fakeOfflineConsole{ posts: map[string][]offPost{"/finish": {{202, offDone("done", 12)}}}, gets: []frame{ offView("offline-open", frame{"reports": []frame{offReport("ie-box", "source", "stopped", true)}}), offView("offline-open", frame{"next": "finish"}), }, } code, out := runOfflineAgainst(t, c, "finish") if code != 0 || !strings.HasSuffix(out, "It was offline for 12 seconds.\n") || !strings.HasPrefix(out, "Waiting for ie-box to clean up acme.shop.example.com.\n") { t.Fatalf("exit %d:\n%s", code, out) } reqs := c.requests() post := -1 for i, r := range reqs { if strings.HasPrefix(r, "POST") { post = i } } if post < 2 { t.Fatalf("the door was posted before the status showed finish:\n%v", reqs) } } func TestMoveOfflineFinishAfterAbort(t *testing.T) { c := &fakeOfflineConsole{ posts: map[string][]offPost{"/finish": {{202, offDone("aborted", 0)}}}, gets: []frame{offView("offline-reverting", frame{"next": "finish"})}, } code, out := runOfflineAgainst(t, c, "finish") if want := "acme.shop.example.com is back in ie and serving. Nothing was lost.\n"; code != 3 || out != want { t.Fatalf("exit %d, output:\n%s\nwant 3:\n%s", code, out, want) } if c.posted("/finish") != 1 { t.Error("finish after an abort posted more than once") } } func TestMoveOfflineFinishRefusedAndOutcomeNotRecorded(t *testing.T) { // Nothing in flight: the door's own sentence, exit 5. c := &fakeOfflineConsole{ posts: map[string][]offPost{"/finish": {{409, frame{"error": "acme.shop.example.com has no offline move in flight"}}}}, gets: []frame{{"host": oHost, "state": "none"}}, } if code, out := runOfflineAgainst(t, c, "finish"); code != 5 || out != "acme.shop.example.com has no offline move in flight\n" { t.Fatalf("exit %d:\n%s", code, out) } // An outcome that is another move's is not this move's. c = &fakeOfflineConsole{ posts: map[string][]offPost{"/finish": {{202, frame{"host": oHost, "state": "none", "last": frame{"id": "mv_other", "outcome": "done"}}}}}, gets: []frame{offView("offline-reverting", frame{"next": "finish"})}, } if code, out := runOfflineAgainst(t, c, "finish"); code != 4 || out != "This move's outcome is not recorded.\n" { t.Fatalf("exit %d:\n%s", code, out) } } func TestMoveOfflineAbort(t *testing.T) { c := &fakeOfflineConsole{ posts: map[string][]offPost{"/abort": {{202, offView("offline-reverting", nil)}}}, gets: []frame{ offView("offline-reverting", frame{"reports": []frame{offReport("ie-box", "source", "stopped", true)}}), offView("offline-reverting", frame{"next": "finish", "reports": []frame{offReport("ie-box", "source", "reverted", true)}}), }, } code, out := runOfflineAgainst(t, c, "abort") want := oBack + "\nWaiting for ie-box to clean up acme.shop.example.com.\n" + oPutBack + "\n" if code != 3 || out != want { t.Fatalf("exit %d, output:\n%s\nwant 3:\n%s", code, out, want) } if c.posted("/finish") != 0 { t.Error("abort ran finish by itself") } // Once open, the door refuses and the CLI says so as sent. c = &fakeOfflineConsole{posts: map[string][]offPost{"/abort": {{409, frame{"error": "acme.shop.example.com is already served from uk. move it back with a new move"}}}}} if code, out := runOfflineAgainst(t, c, "abort"); code != 5 || out != "acme.shop.example.com is already served from uk. move it back with a new move\n" { t.Fatalf("exit %d:\n%s", code, out) } } // A verb whose step is overtaken (another terminal aborted) says where the // move stands and leaves the decision to the operator. func TestMoveOfflineOvertaken(t *testing.T) { c := &fakeOfflineConsole{ posts: map[string][]offPost{"/seal": {{202, offView("offline-sealing", nil)}}}, gets: []frame{offView("offline-reverting", nil)}, } code, out := runOfflineAgainst(t, c, "seal") if want := oCopying + "\n" + oStarting + "\n" + oBack + "\nWaiting for ie-box to clean up acme.shop.example.com.\n"; code != 4 || out != want { t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, want) } } func TestMoveOfflineStatus(t *testing.T) { c := &fakeOfflineConsole{gets: []frame{offView("offline-sealed", frame{ "next": "flip", "items": []frame{ {"id": "db", "owner": "landing", "status": "waiting"}, {"id": "files", "owner": "seal", "status": "ok", "detail": "12 files"}, }, "reports": []frame{offReport("ie-box", "source", "stopped", true), offReport("ie-box", "sealer", "sealed", false)}, })}} code, out := runOfflineAgainst(t, c, "status") want := oStarting + "\n" + oCopied + "\n" + "Item db is waiting.\nItem files is ok: 12 files.\n" + "ie-box is stopped as the source.\nie-box is sealed as the sealer. Its report is not current.\n" if code != 0 || out != want { t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) } if reqs := c.requests(); len(reqs) != 1 || !strings.HasPrefix(reqs[0], "GET /offline") { t.Fatalf("status sent %v", reqs) } // A failed item is the operator's: exit 4. c = &fakeOfflineConsole{gets: []frame{offView("offline-landing", frame{"items": []frame{{"id": "db", "owner": "landing", "status": "failed", "detail": "digest differs"}}})}} code, out = runOfflineAgainst(t, c, "status") wantF := oStarting + "\nuk could not take acme.shop.example.com: digest differs. Run carlos move offline abort --host acme.shop.example.com to move it back.\nItem db is failed: digest differs.\n" if code != 4 || out != wantF { t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, wantF) } // Waiting states say the console's sentence. c = &fakeOfflineConsole{gets: []frame{offView("offline-stopping", frame{"waiting": oWaitBox})}} if code, out := runOfflineAgainst(t, c, "status"); code != 0 || out != oStarting+"\n"+oWaitBox+"\n" { t.Fatalf("exit %d:\n%s", code, out) } // Nothing in flight. c = &fakeOfflineConsole{gets: []frame{{"host": oHost, "state": "none"}}} if code, out := runOfflineAgainst(t, c, "status"); code != 0 || out != "No move is in flight for acme.shop.example.com.\n" { t.Fatalf("exit %d:\n%s", code, out) } c = &fakeOfflineConsole{gets: []frame{offDone("done", 5)}} if code, out := runOfflineAgainst(t, c, "status"); code != 0 || out != "Move "+tMoveID+" of acme.shop.example.com is done.\n" { t.Fatalf("exit %d:\n%s", code, out) } } func TestMoveOfflineWaitingAfterOpenAndAbort(t *testing.T) { // Status of an open move names the boxes still cleaning up. c := &fakeOfflineConsole{gets: []frame{offView("offline-finishing", frame{ "reports": []frame{offReport("ie-box", "source", "gone", true), offReport("ie-box", "sealer", "sealed", true)}})}} code, out := runOfflineAgainst(t, c, "status") want := oStarting + "\n" + oServed + "\nWaiting for ie-box to clean up acme.shop.example.com.\n" + "ie-box is gone as the source.\nie-box is sealed as the sealer.\n" if code != 0 || out != want { t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) } } func TestMoveOfflineNeedsNoPhase1AndRefusesRoot(t *testing.T) { // The Phase 1 door is never read: the fake 404s everything outside // /move/offline, and the verb still works (the other tests). Root is // refused as for every carlos move. c := &fakeOfflineConsole{gets: []frame{{"host": oHost, "state": "none"}}} ts := httptest.NewServer(c) defer ts.Close() var out bytes.Buffer deps := moveDeps{session: func(string) (*apiSession, error) { t.Error("signed in as root"); return nil, nil }, euid: func() int { return 0 }, poll: time.Millisecond} if code := moveMain(context.Background(), []string{"offline", "status", "--host", oHost, "--app", "shop"}, &out, deps); code != 5 { t.Fatalf("exit %d:\n%s", code, out.String()) } wantLines(t, out.String(), "Run carlos move as yourself, not as root. It uses your own credentials.") if len(c.requests()) != 0 { t.Error("a root run reached the console") } } func TestMoveOfflineRidesOutARateLimit(t *testing.T) { c := &fakeOfflineConsole{ posts: map[string][]offPost{"/seal": {{429, frame{"error": "too many instance requests. wait a minute and try again"}}, {202, offView("offline-sealing", nil)}}}, gets: []frame{offView("offline-sealed", frame{"next": "flip"})}, } code, out := runOfflineAgainst(t, c, "seal") if want := "The console is limiting requests. Checking every 10 seconds.\n" + oCopying + "\n" + oCopied + "\n"; code != 0 || out != want { t.Fatalf("exit %d, output:\n%s\nwant:\n%s", code, out, want) } if c.posted("/seal") != 2 { t.Errorf("seal posted %d times, want 2", c.posted("/seal")) } } // The page's paragraph is the copy record's rows MV-O32 and MV-O33, word for // word, and its examples parse as the seven offline verbs. func TestMoveDocsOfflineParagraph(t *testing.T) { raw, err := os.ReadFile("docs/site/cli.md") if err != nil { t.Fatal(err) } page := string(raw) rec, err := os.ReadFile("docs/superpowers/plans/2026-10-06-moves-phase1-copy.md") if err != nil { t.Fatal(err) } for _, id := range []string{"MV-O32", "MV-O33"} { text := "" for _, line := range strings.Split(string(rec), "\n") { if cells := strings.Split(line, "|"); len(cells) >= 5 && strings.TrimSpace(cells[1]) == id { text = strings.TrimSpace(cells[3]) } } if text == "" { t.Fatalf("the copy record has no row %s", id) } if !strings.Contains(page, "\n"+text+"\n") { t.Errorf("the page does not carry %s word for word: %q", id, text) } } seen := map[string]bool{} for _, line := range strings.Split(page, "\n") { fields := strings.Fields(line) if len(fields) < 4 || fields[0] != "carlos" || fields[1] != "move" || fields[2] != "offline" { continue } args := fields[2:] for i, a := range args { if strings.HasPrefix(a, "<") { args[i] = "x" } } got, err := parseMoveArgs(args, io.Discard) if err != nil { t.Errorf("the page's example %q does not parse: %v", line, err) continue } seen[got.offline] = true } for _, v := range offlineVerbs { if !seen[v] { t.Errorf("the page has no example of carlos move offline %s", v) } } } // status of a failed move: exit 4 on the flag alone, a failed seal is not // headlined as progress, and the role the view marks decides MV-O6 or MV-O9. func TestMoveOfflineStatusFailed(t *testing.T) { sealerFailed := frame{"box": "ie-box", "role": "sealer", "phase": "failed", "fresh": true, "items": frame{"files": frame{"ok": false, "reason": "special file in the archive"}}} targetFailed := frame{"box": "uk-box", "role": "target", "phase": "failed", "fresh": true} for _, c := range []struct { name string state string extra frame say string }{ {"failed seal, flag only", "offline-sealing", frame{"failed": true}, "The move could not finish a step: failed. Run carlos move offline status --host acme.shop.example.com for each item."}, {"failed seal, sealer report", "offline-sealing", frame{"failed": true, "reports": []frame{sealerFailed}}, "The move could not finish a step: special file in the archive. Run carlos move offline status --host acme.shop.example.com for each item."}, {"failed landing, target report", "offline-landed", frame{"failed": true, "reports": []frame{targetFailed}}, "uk could not take acme.shop.example.com: failed. Run carlos move offline abort --host acme.shop.example.com to move it back."}, {"failed target before the flip", "offline-stopping", frame{"failed": true, "reports": []frame{targetFailed}}, "uk could not take acme.shop.example.com: failed. Run carlos move offline abort --host acme.shop.example.com to move it back."}, } { t.Run(c.name, func(t *testing.T) { f := &fakeOfflineConsole{gets: []frame{offView(c.state, c.extra)}} code, out := runOfflineAgainst(t, f, "status") if code != 4 { t.Errorf("exit %d, want 4:\n%s", code, out) } if !strings.HasPrefix(out, oStarting+"\n"+c.say+"\n") { t.Errorf("output:\n%s\nwant the failure line after the headline: %s", out, c.say) } wantNoLine(t, out, "Copying") }) } } func TestMoveOfflineStatusShowsPathsLeft(t *testing.T) { c := &fakeOfflineConsole{gets: []frame{offView("offline-finishing", frame{ "next": "finish", "left": []frame{{"box": "ie-box", "path": "/opt/carlos/live/acme.shop.example.com", "reason": "a root-owned directory"}}, })}} code, out := runOfflineAgainst(t, c, "status") if code != 0 { t.Fatalf("exit %d:\n%s", code, out) } wantLines(t, out, "ie-box left /opt/carlos/live/acme.shop.example.com in place: a root-owned directory.") } // A failed report seen after the open is not silent: status says it and exits // 4, and open stops polling with the same line. func TestMoveOfflineFailedReportAfterTheOpen(t *testing.T) { failedRep := func(box, role, check string) frame { f := offReport(box, role, "failed", true) f["items"] = frame{check: frame{"ok": false, "reason": "the " + check + " check failed"}} return f } for _, c := range []struct { name string state string rep frame say string }{ // Abort refuses once open, so a target failure there says MV-O9. {"target, open", "offline-open", failedRep("uk-box", "target", "config"), "The move could not finish a step: the config check failed. Run carlos move offline status --host acme.shop.example.com for each item."}, {"sealer, finishing", "offline-finishing", failedRep("ie-box", "sealer", "lease"), "The move could not finish a step: the lease check failed. Run carlos move offline status --host acme.shop.example.com for each item."}, {"source, open", "offline-open", failedRep("ie-box", "source", "warm-cache"), "ie-box could not clean up acme.shop.example.com: the warm-cache check failed. Run carlos move offline status --host acme.shop.example.com for each item."}, {"source, reverting", "offline-reverting", failedRep("ie-box", "source", "warm-cache"), "ie-box could not clean up acme.shop.example.com: the warm-cache check failed. Run carlos move offline status --host acme.shop.example.com for each item."}, } { t.Run(c.name, func(t *testing.T) { f := &fakeOfflineConsole{gets: []frame{offView(c.state, frame{"reports": []frame{c.rep}})}} code, out := runOfflineAgainst(t, f, "status") if code != 4 || !strings.HasPrefix(out, oStarting+"\n"+c.say+"\n") { t.Errorf("status exit %d, output:\n%s\nwant 4 and %q after the headline", code, out, c.say) } if c.state == "offline-open" { f := &fakeOfflineConsole{ posts: map[string][]offPost{"/open": {{202, offView("offline-open", nil)}}}, gets: []frame{offView(c.state, frame{"reports": []frame{c.rep}})}, } code, out := runOfflineAgainst(t, f, "open") if code != 4 || !strings.HasSuffix(out, c.say+"\n") { t.Errorf("open exit %d, output:\n%s\nwant 4 ending in %q", code, out, c.say) } } }) } // A target or sealer that failed before an abort is being put back: a // reverting move shows only a failed source box. stale := offReport("uk-box", "target", "failed", true) f := &fakeOfflineConsole{gets: []frame{offView("offline-reverting", frame{"next": "finish", "reports": []frame{stale}})}} if code, out := runOfflineAgainst(t, f, "status"); code != 0 || strings.Contains(out, "could not") { t.Errorf("a reverting move with an old failed target: exit %d:\n%s", code, out) } } // ---- carlos move offline stage ---- const ( oStaging = "Staging acme.shop.example.com in uk. acme.shop.example.com keeps serving while the data is copied." oReady = "acme.shop.example.com keeps serving. Run carlos move offline start --host acme.shop.example.com --to uk when you are ready to take it offline." oUnstage = "Removing the staged copy of acme.shop.example.com from uk." oUnstaged = "The staged copy of acme.shop.example.com is removed. Run carlos move offline finish --host acme.shop.example.com to end the move." ) // stageView is a staging move's view with the stager's last round, which // answered the request at answered (0: no round yet). func stageView(stagedAt, answered int64, phase string, extra frame) frame { f := frame{"staged_at": stagedAt, "never_offline": true, "sealer": "", "boxes": nil, "can_abort": true} if answered != 0 { f["stage"] = frame{"box": "ie-box", "phase": phase, "at": int64(1792573200), "replica_objects": 305, "replica_bytes": 1_580_000_000, "staged_objects": 305, "staged_bytes": 1_580_000_000, "round_objects": 305, "round_bytes": 1_580_000_000, "request": answered, "error": "mirror: copy instances/x: boom"} } for k, v := range extra { f[k] = v } return offView("offline-staging", f) } // stage posts and then waits for the round that answers its own request: // not the earlier round the answer already showed. func TestMoveOfflineStage(t *testing.T) { c := &fakeOfflineConsole{ posts: map[string][]offPost{"/stage": {{202, stageView(200, 100, "staged", nil)}}}, gets: []frame{ stageView(200, 100, "staged", nil), stageView(200, 200, "staged", frame{"next": "start"}), }, } code, out := runOfflineAgainst(t, c, "stage", "--to", "uk") want := oStaging + "\nStaged 305 of 305 objects for acme.shop.example.com in uk (1580.0 of 1580.0 MB) at 2026-10-21 09:00 UTC.\n" + oReady + "\n" if code != 0 || out != want { t.Fatalf("exit %d, output:\n%s\nwant 0:\n%s", code, out, want) } if got := c.requests(); len(got) == 0 || got[0] != `POST /offline/stage {"region":"uk"}` { t.Fatalf("requests %q", got) } if c.posted("") != 0 { t.Error("stage started the move by itself") } // A failed round: the box's reason, the way on, exit 4. c = &fakeOfflineConsole{ posts: map[string][]offPost{"/stage": {{202, stageView(300, 0, "", nil)}}}, gets: []frame{stageView(300, 300, "failed", nil)}, } code, out = runOfflineAgainst(t, c, "stage", "--to", "uk") want = oStaging + "\nThe last staging round failed: mirror: copy instances/x: boom. Run carlos move offline stage --host acme.shop.example.com --to uk to try again.\n" if code != 4 || out != want { t.Fatalf("exit %d, output:\n%s\nwant 4:\n%s", code, out, want) } // A refusal is the console's sentence, as sent. refusal := "acme.shop.example.com is staged for uk. abort the move to move it to another region" c = &fakeOfflineConsole{posts: map[string][]offPost{"/stage": {{409, frame{"error": refusal}}}}} if code, out := runOfflineAgainst(t, c, "stage", "--to", "us"); code != 5 || out != refusal+"\n" { t.Fatalf("exit %d:\n%s", code, out) } } // status while staging: the stage's line instead of the manifest's items. func TestMoveOfflineStatusWhileStaging(t *testing.T) { c := &fakeOfflineConsole{gets: []frame{stageView(200, 0, "", nil)}} code, out := runOfflineAgainst(t, c, "status") if want := oStaging + "\nWaiting for the first staging round of acme.shop.example.com.\n"; code != 0 || out != want { t.Fatalf("exit %d, output:\n%s\nwant 0:\n%s", code, out, want) } c = &fakeOfflineConsole{gets: []frame{stageView(200, 200, "staged", frame{"next": "start"})}} code, out = runOfflineAgainst(t, c, "status") want := oStaging + "\nStaged 305 of 305 objects for acme.shop.example.com in uk (1580.0 of 1580.0 MB) at 2026-10-21 09:00 UTC.\n" + oReady + "\n" if code != 0 || out != want { t.Fatalf("exit %d, output:\n%s\nwant 0:\n%s", code, out, want) } c = &fakeOfflineConsole{gets: []frame{stageView(200, 200, "failed", nil)}} if code, _ := runOfflineAgainst(t, c, "status"); code != 4 { t.Fatalf("status of a failed round exit %d, want 4", code) } } // abort of a move still staging: nothing to move back, only the staged copy. func TestMoveOfflineAbortStaged(t *testing.T) { c := &fakeOfflineConsole{ posts: map[string][]offPost{"/abort": {{202, offView("offline-reverting", frame{"never_offline": true, "sealer": ""})}}}, gets: []frame{ offView("offline-reverting", frame{"never_offline": true, "sealer": ""}), offView("offline-reverting", frame{"never_offline": true, "sealer": "", "next": "finish"}), }, } code, out := runOfflineAgainst(t, c, "abort") if want := oUnstage + "\n" + oUnstaged + "\n"; code != 3 || out != want { t.Fatalf("exit %d, output:\n%s\nwant 3:\n%s", code, out, want) } if c.posted("/finish") != 0 { t.Error("abort ran finish by itself") } }