package provision import ( "context" "encoding/json" "errors" "strings" "testing" "time" "github.com/carlosframework/platform/internal/release" ) func moveTestRecord(t *testing.T, s release.Store) Record { t.Helper() rec := Record{Schema: 1, Host: "a.go.example.com", Slug: "s", App: "app", Account: "bab", Region: "ie", Channel: "stable", CreatedAt: time.Now().UTC()} if err := CreateRecord(context.Background(), s, rec); err != nil { t.Fatal(err) } return rec } func TestMoveStartWritesTheJournalHead(t *testing.T) { s := release.NewDirStore(t.TempDir()) rec := moveTestRecord(t, s) now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) p := MoveStart{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32), From: "ie", To: "uk", Cut: "auto", By: "ops@example.com", Now: now} got, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MoveStartTransition(p)) if err != nil { t.Fatal(err) } m := got.Move if m.ID != p.ID || m.Incarnation != p.Incarnation || m.From != "ie" || m.To != "uk" || m.Cut != "auto" || m.RequestedBy != p.By || m.State != MovePreparing || !m.RequestedAt.Equal(now) || m.UpdatedAt.IsZero() || m.Legacy() { t.Fatalf("journal head: %+v", m) } if got.Region != "ie" { t.Fatal("the request moved the region") } // Refusals: in flight, same region, a replica, a bad id, a bad cut mode. for name, bad := range map[string]MoveStart{ "in flight": p, } { if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MoveStartTransition(bad)); !errors.Is(err, ErrMoveConflict) { t.Fatalf("%s: err %v, want ErrMoveConflict", name, err) } } s2 := release.NewDirStore(t.TempDir()) rec2 := moveTestRecord(t, s2) for name, bad := range map[string]MoveStart{ "same region": {ID: p.ID, Incarnation: p.Incarnation, From: "ie", To: "ie", Cut: "auto", Now: now}, "bad id": {ID: "mv_short", Incarnation: p.Incarnation, From: "ie", To: "uk", Cut: "auto", Now: now}, "bad cut": {ID: p.ID, Incarnation: p.Incarnation, From: "ie", To: "uk", Cut: "sometime", Now: now}, "wrong from": {ID: p.ID, Incarnation: p.Incarnation, From: "us", To: "uk", Cut: "auto", Now: now}, "no inc": {ID: p.ID, To: "uk", From: "ie", Cut: "auto", Now: now}, } { if _, err := SetRecordMove(context.Background(), s2, rec2.Account, rec2.App, rec2.Host, MoveStartTransition(bad)); !errors.Is(err, ErrMoveConflict) { t.Fatalf("%s: err %v, want ErrMoveConflict", name, err) } } replica := Record{Schema: 1, Host: "r.go.example.com", Slug: "s", App: "app", Account: "bab", Region: "ie", Channel: "stable", CreatedAt: time.Now().UTC(), Role: "replica"} if err := CreateRecord(context.Background(), s2, replica); err != nil { t.Fatal(err) } if _, err := SetRecordMove(context.Background(), s2, "bab", "app", replica.Host, MoveStartTransition(p)); !errors.Is(err, ErrMoveConflict) { t.Fatalf("replica: %v", err) } } // Review Focus 1: a move a v0.22 console wrote has no id. The journal // calls it legacy; the legacy transitions still drive it (the legacy path // stays while the switch is off, Task 0), and no guarded transition // accepts it. func TestMoveStateLegacyRecordIsRefusedByTheGuard(t *testing.T) { s := release.NewDirStore(t.TempDir()) rec := moveTestRecord(t, s) if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MoveRequest("uk", time.Now())); err != nil { t.Fatal(err) } got, _ := GetRecord(context.Background(), s, rec.Account, rec.App, rec.Host) if !got.Move.Legacy() { t.Fatal("an id-less move is legacy") } g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)} for name, tr := range map[string]func(*Record) error{ "promote": MoveGuardedPromoteReady(g, time.Now(), MoveInputs{}), "cut": MoveGuardedCut(g), "abort": MoveGuardedAbort(g), "abort complete": MoveGuardedAbortComplete(g, time.Now()), // A guard with no id at all must not match an id-less move either. "promote, empty guard": MoveGuardedPromoteReady(MoveGuard{}, time.Now(), MoveInputs{}), } { if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, tr); !errors.Is(err, ErrMoveConflict) { t.Fatalf("guarded %s on a legacy move: %v", name, err) } } // The legacy transitions still work, unchanged. if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MovePromoteReady(time.Now())); err != nil { t.Fatal(err) } if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MoveAbort()); err != nil { t.Fatal(err) } if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MoveAbortComplete()); err != nil { t.Fatal(err) } } // The switch-off contract: a record written by the legacy transitions // carries exactly the journal keys main wrote, none of the new ones. func TestLegacyMoveRecordKeepsMainsKeys(t *testing.T) { rec := &Record{Region: "ie"} if err := MoveRequest("uk", time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC))(rec); err != nil { t.Fatal(err) } body, err := json.Marshal(rec.Move) if err != nil { t.Fatal(err) } want := `{"to":"uk","state":"preparing","requested_at":"2026-10-06T09:00:00Z","updated_at":"0001-01-01T00:00:00Z"}` if string(body) != want { t.Fatalf("legacy journal drifted:\n got %s\nwant %s", body, want) } out, err := json.Marshal(Record{Region: "ie"}) if err != nil { t.Fatal(err) } if strings.Contains(string(out), "last_move") { t.Fatalf("a record at rest grew last_move: %s", out) } } func TestGuardedPromoteReadyPinsTheInputs(t *testing.T) { g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)} now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) in := MoveInputs{BundleETags: map[string]string{"app": "e1"}, OverlayETag: "o1", Release: "v1", FilesPlain: "f1", EnvReport: "r1"} rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MovePreparing}} if err := MoveGuardedPromoteReady(g, now, in)(rec); err != nil { t.Fatal(err) } m := rec.Move if m.State != MoveReady || m.ReadyAt == nil || !m.ReadyAt.Equal(now) { t.Fatalf("not ready: %+v", m) } if m.Inputs == nil || m.Inputs.OverlayETag != "o1" || m.Inputs.Release != "v1" || m.Inputs.FilesPlain != "f1" || m.Inputs.EnvReport != "r1" || m.Inputs.BundleETags["app"] != "e1" { t.Fatalf("inputs not pinned: %+v", m.Inputs) } in.BundleETags["app"] = "changed" if m.Inputs.BundleETags["app"] != "e1" { t.Fatal("the pinned inputs alias the caller's map") } // A second promote is refused and does not re-pin. if err := MoveGuardedPromoteReady(g, now, MoveInputs{Release: "v2"})(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Inputs.Release != "v1" { t.Fatalf("second promote: %v %+v", err, rec.Move.Inputs) } // A refused promote leaves a preparing move without inputs. other := &Record{Move: &MoveState{ID: "mv_bbbbbbbbbbbbbbbbbbbbbbbbbb", Incarnation: g.Incarnation, State: MovePreparing}} if err := MoveGuardedPromoteReady(g, now, in)(other); !errors.Is(err, ErrMoveConflict) || other.Move.Inputs != nil || other.Move.State != MovePreparing { t.Fatalf("guard mismatch: %v %+v", err, other.Move) } } func TestGuardedCutAndAbortComplete(t *testing.T) { g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)} rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MovePreparing}} if err := MoveGuardedCut(g)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("cut from preparing: %v", err) } rec.Move.State = MoveReady // G1: a ready move with no pinned inputs cannot be cut. if err := MoveGuardedCut(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveReady || rec.Move.CutPhase != "" { t.Fatalf("cut on nil inputs: %v %+v", err, rec.Move) } rec.Move.Inputs = &MoveInputs{Release: "v1"} if err := MoveGuardedCut(g)(rec); err != nil || rec.Move.State != MoveCutting || rec.Move.CutPhase != CutRequested { t.Fatalf("cut from ready: %v %+v", err, rec.Move) } if err := MoveGuardedCut(g)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("second cut: %v", err) } // Incarnation mismatch alone is refused. stale := MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)} if err := MoveGuardedAbort(stale)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveCutting { t.Fatalf("stale incarnation: %v", err) } // Abort from cutting/requested, then abort-complete records the outcome. if err := MoveGuardedAbort(g)(rec); err != nil || rec.Move.State != MoveAborting { t.Fatalf("abort: %v %+v", err, rec.Move) } now := time.Date(2026, 10, 6, 9, 5, 0, 0, time.UTC) if err := MoveGuardedAbortComplete(g, now)(rec); err != nil { t.Fatal(err) } if rec.Move != nil || rec.LastMove == nil || rec.LastMove.ID != g.ID || rec.LastMove.Outcome != "aborted" || !rec.LastMove.At.Equal(now) { t.Fatalf("outcome: move %+v last %+v", rec.Move, rec.LastMove) } if err := MoveGuardedAbortComplete(g, now)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("abort-complete with no move: %v", err) } } func TestGuardedAbortLegalStates(t *testing.T) { g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)} cases := []struct { state, cutPhase string ok bool }{ {MovePreparing, "", true}, {MoveReady, "", true}, {MoveCutting, CutRequested, true}, {MoveCutting, CutAdmitted, false}, {MoveLanding, "", false}, {MoveRollingBack, "", false}, {MoveReverting, "", false}, {MoveAborting, "", false}, {"future", "", false}, } for _, c := range cases { m := &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: c.state, CutPhase: c.cutPhase} rec := &Record{Move: m} err := MoveGuardedAbort(g)(rec) if (err == nil) != c.ok { t.Fatalf("abort from %s/%s: err %v", c.state, c.cutPhase, err) } } // The legacy MoveAbort keeps its allowlist (P0's rule): cutting refused. rec := &Record{Move: &MoveState{State: MoveCutting}} if err := MoveAbort()(rec); !errors.Is(err, ErrMoveConflict) { t.Fatal("legacy abort from cutting must stay refused") } } func TestGuardedTransitionsRefuseAReplacementMove(t *testing.T) { now := time.Now() gA := MoveGuard{ID: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Incarnation: strings.Repeat("ab", 32)} // A's worker observed preparing; the operator aborted A and requested // B, which is preparing too. rec := &Record{Region: "ie", Move: &MoveState{ID: "mv_bbbbbbbbbbbbbbbbbbbbbbbbbb", Incarnation: gA.Incarnation, From: "ie", To: "uk", State: MovePreparing}} if err := MoveGuardedPromoteReady(gA, now, MoveInputs{})(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MovePreparing { t.Fatalf("A's round promoted B: %v %+v", err, rec.Move) } rec.Move.State = MoveAborting if err := MoveGuardedAbortComplete(gA, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil { t.Fatalf("A's abort cleared B: %v", err) } if rec.LastMove != nil { t.Fatalf("A's refused abort left an outcome: %+v", rec.LastMove) } } func TestInstanceStoreEnv(t *testing.T) { if got := InstanceStoreEnv(&Record{Slug: "s"}); got != "s" { t.Fatalf("no environment: %q", got) } if got := InstanceStoreEnv(&Record{Slug: "s", Environment: "canary"}); got != "canary" { t.Fatalf("named environment: %q", got) } } func TestManifestReadyPathDefault(t *testing.T) { if (Manifest{}).ReadyPathOrDefault() != "/" { t.Fatal("default ready path") } if (Manifest{Moves: &MovesManifest{ReadyPath: "/healthz"}}).ReadyPathOrDefault() != "/healthz" { t.Fatal("declared ready path") } if (Manifest{Moves: &MovesManifest{}}).ReadyPathOrDefault() != "/" { t.Fatal("empty declaration keeps the default") } } func cutTestGuard() MoveGuard { return MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)} } func TestCutPhasesAdvanceInOrderUnderTheGuard(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReady, Inputs: &MoveInputs{Release: "v1"}}} if err := MoveGuardedCut(g)(rec); err != nil || rec.Move.State != MoveCutting || rec.Move.CutPhase != CutRequested { t.Fatalf("auto cut: %v %+v", err, rec.Move) } deadline := now.Add(115 * time.Second) if err := MoveAdmit(g, now, 120, deadline, 3)(rec); err != nil || rec.Move.CutPhase != CutAdmitted || !rec.Move.CutDeadline.Equal(deadline) || !rec.Move.CutAt.Equal(now) || rec.Move.CutBudgetS != 120 || rec.Move.Inputs.Release != "v1" { t.Fatalf("admit: %v %+v", err, rec.Move) } // Wrong incarnation: refused. if err := MoveCutPhase(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, CutAdmitted, CutStopping, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("wrong guard: %v", err) } // Skipping a phase: refused. if err := MoveCutPhase(g, CutStopping, CutStopped, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("skip: %v", err) } // Source digests are recorded at stopped only. if err := MoveRecordSource(g, MoveSource{FinalTXID: 1})(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Source != nil { t.Fatalf("source before stopped: %v", err) } for _, step := range [][2]string{{CutAdmitted, CutStopping}, {CutStopping, CutStopped}} { if err := MoveCutPhase(g, step[0], step[1], nil)(rec); err != nil { t.Fatal(err) } } if err := MoveRecordSource(g, MoveSource{FinalTXID: 42, TimersSHA256: "sha256:t"})(rec); err != nil || rec.Move.Source.FinalTXID != 42 { t.Fatalf("source: %v", err) } // A replay of the same transition succeeds, and the record keeps its // own copy of the files digest: the caller mutating its value after // the call does not reach the record (P2-T1-1). files := &MoveFilesDigest{Plain: "sha256:p", Sealed: "sha256:s", Bytes: 7, Entries: 2} src := MoveSource{FinalTXID: 42, TimersSHA256: "sha256:t", Files: files} for i := 0; i < 2; i++ { if err := MoveRecordSource(g, src)(rec); err != nil || rec.Move.Source.FinalTXID != 42 { t.Fatalf("source replay %d: %v", i, err) } } if rec.Move.Source.Files == nil || rec.Move.Source.Files == files { t.Fatalf("the record shares the caller's files digest: %p vs %p", rec.Move.Source.Files, files) } want := MoveFilesDigest{Plain: "sha256:p", Sealed: "sha256:s", Bytes: 7, Entries: 2} files.Plain, files.Sealed, files.Bytes, files.Entries = "x", "y", 99, 99 src.Files = nil if got := *rec.Move.Source.Files; got != want { t.Fatalf("the record's files digest followed the caller's: %+v, want %+v", got, want) } for _, step := range [][2]string{{CutStopped, CutStaged}, {CutStaged, CutArchived}} { if err := MoveCutPhase(g, step[0], step[1], nil)(rec); err != nil { t.Fatal(err) } } // Abort is refused past requested. if err := MoveGuardedAbort(g)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatal("abort past admitted") } // The cut cannot step to flipped: only MoveFlip writes it. if err := MoveCutPhase(g, CutArchived, CutFlipped, nil)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.CutPhase != CutArchived { t.Fatalf("a phase step to flipped: %v", err) } ld := now.Add(10 * time.Minute) if err := MoveFlip(g, now, ld)(rec); err != nil || rec.Region != "uk" || rec.Move.State != MoveLanding || rec.Move.CutPhase != CutFlipped || rec.Move.LandingPhase != LandChecking || !rec.Move.LandingDeadline.Equal(ld) || !rec.Move.FlippedAt.Equal(now) { t.Fatalf("flip: %v %+v", err, rec) } if err := MoveFlip(g, now, ld)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatal("a second flip") } if err := MoveLandingPhase(g, LandChecking, LandParking, nil)(rec); err != nil || rec.Move.LandingPhase != LandParking { t.Fatalf("landing phase: %v", err) } if err := MoveLandingPhase(g, LandParking, LandPublished, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatal("a landing phase skipped") } stamped := false if err := MoveLandingPhase(g, LandParking, LandBuilding, func(m *MoveState) { stamped = m.LandingPhase == LandBuilding })(rec); err != nil || !stamped { t.Fatalf("landing stamp: %v", err) } if err := MoveLandingPhase(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, LandBuilding, LandPublished, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatal("landing phase under a stale guard") } } // Every legal cut and landing step, and only those, under both orders. func TestCutAndLandingStepsAreTheOrderedPairsOnly(t *testing.T) { g := cutTestGuard() cuts := []string{CutRequested, CutAdmitted, CutStopping, CutStopped, CutStaged, CutArchived} all := append(append([]string{}, cuts...), CutFlipped, "") for _, from := range all { for _, to := range all { rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveCutting, CutPhase: from, Inputs: &MoveInputs{}}} err := MoveCutPhase(g, from, to, nil)(rec) want := false for i := 0; i+1 < len(cuts); i++ { if cuts[i] == from && cuts[i+1] == to { want = true } } // archived to flipped is MoveFlip's, not a phase step's. if from == CutArchived && to == CutFlipped { want = false } if (err == nil) != want { t.Fatalf("cut %q to %q: err %v, want ok=%v", from, to, err, want) } } } lands := []string{LandChecking, LandParking, LandBuilding, LandPublished, LandLocal, LandConfigured, LandRestored, LandProbing, LandProbed} for i, from := range lands { for j, to := range lands { rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveLanding, LandingPhase: from}} err := MoveLandingPhase(g, from, to, nil)(rec) if (err == nil) != (j == i+1) { t.Fatalf("landing %q to %q: err %v", from, to, err) } } } // A landing step from a state that is not landing is refused. rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveCutting, LandingPhase: LandChecking}} if err := MoveLandingPhase(g, LandChecking, LandParking, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatal("landing step in cutting") } } func TestBackToReadyAndRevert(t *testing.T) { g := cutTestGuard() now := time.Now() rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveCutting, CutPhase: CutAdmitted, CutAt: &now, CutBudgetS: 120, CutDeadline: &now, Inputs: &MoveInputs{Release: "v1"}}} if err := MoveBackToReady(g)(rec); err != nil || rec.Move.State != MoveReady || rec.Move.CutPhase != "" || rec.Move.CutAt != nil || rec.Move.CutBudgetS != 0 || rec.Move.CutDeadline != nil || rec.Move.Inputs != nil { t.Fatalf("back to ready: %v %+v", err, rec.Move) } if err := MoveBackToReady(g)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatal("back to ready from ready") } for _, ph := range []string{CutStopping, CutStopped, CutStaged, CutArchived} { rec.Move.State, rec.Move.CutPhase = MoveCutting, ph if err := MoveBackToReady(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveCutting { t.Fatalf("back to ready from %s: %v", ph, err) } if err := MoveRevert(g, RevertStopEvidenceBudget)(rec); err != nil || rec.Move.State != MoveReverting || rec.Move.Reason != "stop evidence budget" { t.Fatalf("revert from %s: %v %+v", ph, err, rec.Move) } } // Not from requested or admitted: nothing ran yet, so back-to-ready is the exit there. for _, ph := range []string{CutRequested, CutAdmitted, CutFlipped, ""} { rec2 := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveCutting, CutPhase: ph}} if err := MoveRevert(g, "x")(rec2); !errors.Is(err, ErrMoveConflict) || rec2.Move.State != MoveCutting { t.Fatalf("revert from %q: %v", ph, err) } } // Only a recorded sentence can be the reason: anything else is refused // and nothing is written. rec4 := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveCutting, CutPhase: CutStopped}} if err := MoveRevert(g, "some error text")(rec4); err == nil || errors.Is(err, ErrMoveConflict) || rec4.Move.State != MoveCutting || rec4.Move.Reason != "" { t.Fatalf("an unrecorded reason: %v %+v", err, rec4.Move) } // Every recorded reason is accepted and written as itself (Task 5's // reverts: the deadline, the deferred final rounds, the archive). for _, reason := range []string{RevertStopEvidenceBudget, RevertCutDeadline, RevertDeferredRounds, RevertArchiveRefused} { rec5 := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveCutting, CutPhase: CutStaged}} if err := MoveRevert(g, reason)(rec5); err != nil || rec5.Move.State != MoveReverting || rec5.Move.Reason != reason { t.Fatalf("revert for %q: %v %+v", reason, err, rec5.Move) } } // Revert is a cut's recovery only. rec3 := &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveLanding, CutPhase: CutFlipped}} if err := MoveRevert(g, "x")(rec3); !errors.Is(err, ErrMoveConflict) { t.Fatal("revert from landing") } } // G1: the whole loop. Inputs pinned at readiness, drift found at admission, // back to ready (Inputs nil), repin, cut, admit. func TestInputsLoopPinDriftBackRepinCutAdmit(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) pin := MoveInputs{BundleETags: map[string]string{"app": "e1"}, Release: "v1"} rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MovePreparing}} if err := MoveGuardedPromoteReady(g, now, pin)(rec); err != nil || rec.Move.Inputs == nil { t.Fatalf("pin: %v", err) } // Repin is legal only with Inputs nil. if err := MoveGuardedRepin(g, MoveInputs{Release: "v2"})(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Inputs.Release != "v1" { t.Fatalf("repin over a pin: %v", err) } if err := MoveGuardedCut(g)(rec); err != nil { t.Fatal(err) } // Drift found: back to ready clears the pin. if err := MoveBackToReady(g)(rec); err != nil || rec.Move.Inputs != nil { t.Fatalf("drift, back to ready: %v %+v", err, rec.Move) } // A ready move with no pin cannot be cut. if err := MoveGuardedCut(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveReady { t.Fatalf("cut on nil inputs: %v", err) } // Repin: guarded, legal in ready only. if err := MoveGuardedRepin(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, MoveInputs{Release: "v2"})(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Inputs != nil { t.Fatalf("repin under a stale guard: %v", err) } fresh := MoveInputs{BundleETags: map[string]string{"app": "e2"}, Release: "v2"} if err := MoveGuardedRepin(g, fresh)(rec); err != nil || rec.Move.Inputs == nil || rec.Move.Inputs.Release != "v2" || rec.Move.State != MoveReady { t.Fatalf("repin: %v %+v", err, rec.Move) } fresh.BundleETags["app"] = "changed" if rec.Move.Inputs.BundleETags["app"] != "e2" { t.Fatal("the repin aliases the caller's map") } if err := MoveGuardedRepin(g, MoveInputs{Release: "v3"})(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Inputs.Release != "v2" { t.Fatalf("second repin: %v", err) } if err := MoveGuardedCut(g)(rec); err != nil { t.Fatal(err) } if err := MoveGuardedRepin(g, MoveInputs{Release: "v3"})(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("repin in cutting: %v", err) } // Admit pins nothing: it stamps the clock fields and keeps the pin. if err := MoveAdmit(g, now, 120, now.Add(time.Minute), 3)(rec); err != nil || rec.Move.CutPhase != CutAdmitted || rec.Move.Inputs.Release != "v2" { t.Fatalf("admit: %v %+v", err, rec.Move) } // Admit on a cut with no pin is refused and writes nothing. bare := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveCutting, CutPhase: CutRequested}} if err := MoveAdmit(g, now, 120, now.Add(time.Minute), 3)(bare); !errors.Is(err, ErrMoveConflict) || bare.Move.CutPhase != CutRequested || bare.Move.CutAt != nil || bare.Move.CutBudgetS != 0 { t.Fatalf("admit with no pin: %v %+v", err, bare.Move) } } // G18: the flip refuses any state but cutting/archived, so a record decision // written first (the C7 conversion's pre-flip guard, a revert, an abort) // wins over a flip that was already in flight. func TestFlipRefusesAnyStateButCuttingArchived(t *testing.T) { g := cutTestGuard() now := time.Now() for _, c := range []struct{ state, phase string }{ {MoveReverting, CutArchived}, {MoveAborting, CutArchived}, {MoveRollingBack, CutArchived}, {MoveStuck, CutArchived}, {MoveReady, ""}, {MoveCutting, CutStaged}, {MoveCutting, CutRequested}, {MoveCutting, CutFlipped}, {MoveLanding, CutFlipped}, } { rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: c.state, CutPhase: c.phase}} if err := MoveFlip(g, now, now.Add(time.Minute))(rec); !errors.Is(err, ErrMoveConflict) || rec.Region != "ie" || rec.Move.State != c.state || rec.Move.FlippedAt != nil { t.Fatalf("flip from %s/%s: %v region %s", c.state, c.phase, err, rec.Region) } } // The sequence that matters: a revert is recorded, then the late flip. rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveCutting, CutPhase: CutArchived}} if err := MoveRevert(g, RevertStopEvidenceBudget)(rec); err != nil { t.Fatal(err) } if err := MoveFlip(g, now, now.Add(time.Minute))(rec); !errors.Is(err, ErrMoveConflict) || rec.Region != "ie" || rec.Move.State != MoveReverting { t.Fatalf("late flip after a revert: %v %+v", err, rec) } // And a stale guard cannot flip an archived cut. rec = &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveCutting, CutPhase: CutArchived}} if err := MoveFlip(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, now, now.Add(time.Minute))(rec); !errors.Is(err, ErrMoveConflict) || rec.Region != "ie" { t.Fatalf("flip under a stale guard: %v", err) } } // The transitions run through SetRecordMove end to end, and the flip's // region change and state change land in one record write. func TestFlipIsOneRecordWrite(t *testing.T) { s := release.NewDirStore(t.TempDir()) rec := moveTestRecord(t, s) g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) ctx := context.Background() seq := []func(*Record) error{ MoveStartTransition(MoveStart{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", Cut: "auto", By: "ops@example.com", Now: now}), MoveGuardedPromoteReady(g, now, MoveInputs{Release: "v1"}), MoveGuardedCut(g), MoveAdmit(g, now, 120, now.Add(115*time.Second), 3), MoveCutPhase(g, CutAdmitted, CutStopping, nil), MoveCutPhase(g, CutStopping, CutStopped, nil), MoveCutPhase(g, CutStopped, CutStaged, nil), MoveCutPhase(g, CutStaged, CutArchived, nil), } for i, tr := range seq { if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, tr); err != nil { t.Fatalf("step %d: %v", i, err) } } got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, MoveFlip(g, now, now.Add(10*time.Minute))) if err != nil || got.Region != "uk" || got.Move.State != MoveLanding { t.Fatalf("flip: %v %+v", err, got) } read, _ := GetRecord(ctx, s, rec.Account, rec.App, rec.Host) if read.Region != "uk" || read.Move.State != MoveLanding || read.Move.LandingPhase != LandChecking || read.Move.Inputs == nil || read.Move.Inputs.Release != "v1" { t.Fatalf("persisted: %+v", read.Move) } } // A back-to-ready may record why (Task 5 fix round 1, ruling 1): the // refusal of a holder whose replica layout Phase 1 cannot cut. The reason // is a recorded sentence; anything else is refused and nothing is written. // The next admission clears it, so a reason never outlives the refusal. func TestBackToReadyRecordsARefusal(t *testing.T) { g := cutTestGuard() now := time.Now() rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveCutting, CutPhase: CutAdmitted, CutAt: &now, CutBudgetS: 120, CutDeadline: &now, Inputs: &MoveInputs{Release: "v1"}}} if err := MoveBackToReadyRefused(g, "x")(rec); err == nil || rec.Move.State != MoveCutting || rec.Move.Reason != "" { t.Fatalf("an unrecorded reason: %v %+v", err, rec.Move) } if err := MoveBackToReadyRefused(g, RefusedReplicaLayout)(rec); err != nil || rec.Move.State != MoveReady || rec.Move.CutPhase != "" || rec.Move.Inputs != nil || rec.Move.Reason != RefusedReplicaLayout { t.Fatalf("refused: %v %+v", err, rec.Move) } if err := MoveBackToReadyRefused(g, RefusedReplicaLayout)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatal("refused from ready") } rec.Move.Inputs = &MoveInputs{Release: "v1"} rec.Move.State, rec.Move.CutPhase = MoveCutting, CutRequested if err := MoveAdmit(g, now, 120, now, 3)(rec); err != nil || rec.Move.Reason != "" { t.Fatalf("the admission did not clear the reason: %v %+v", err, rec.Move) } } // The cut guard (Paul's ruling, 2026-10-11): an unarmed target or an // unarmed source sends the cut back to ready from requested or admitted, // and each records its own sentence, not the replica layout's. func TestBackToReadyRecordsTheCutGuardRefusals(t *testing.T) { g := cutTestGuard() now := time.Now() for _, reason := range []string{RefusedTargetUnarmed, RefusedSourceUnarmed} { for _, phase := range []string{CutRequested, CutAdmitted} { rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveCutting, CutPhase: phase, CutAt: &now, CutBudgetS: 120, CutDeadline: &now, Inputs: &MoveInputs{Release: "v1"}}} if err := MoveBackToReadyRefused(g, reason)(rec); err != nil || rec.Move.State != MoveReady || rec.Move.CutPhase != "" || rec.Move.Inputs != nil || rec.Move.Reason != reason { t.Fatalf("%s at %s: %v %+v", reason, phase, err, rec.Move) } } } if RefusedTargetUnarmed == RefusedSourceUnarmed || RefusedTargetUnarmed == RefusedReplicaLayout || RefusedSourceUnarmed == RefusedReplicaLayout { t.Fatal("the refusals do not each have their own sentence") } } // Fix round 1: a repin clears a recorded refusal. A move is pinned again // only once the refusal's cause is gone (decideReady and T13), so from then // on it is ready to cut and `carlos move status` no longer says the cut did // not start. func TestRepinClearsARecordedRefusal(t *testing.T) { g := cutTestGuard() now := time.Now() rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveCutting, CutPhase: CutRequested, CutAt: &now, Inputs: &MoveInputs{Release: "v1"}}} if err := MoveBackToReadyRefused(g, RefusedSourceUnarmed)(rec); err != nil || rec.Move.Reason != RefusedSourceUnarmed { t.Fatalf("fixture: %v %+v", err, rec.Move) } if err := MoveGuardedRepin(g, MoveInputs{Release: "v2"})(rec); err != nil || rec.Move.Inputs == nil || rec.Move.Reason != "" { t.Fatalf("the repin kept the refusal: %v %+v", err, rec.Move) } } // The landing marks a rollback it cannot yet start (Plan 2 Task 10): the // reason is written on the landing record, the state and phase stay as they // are, and Plan 4's rollback transition is the one call site that replaces // it. Only a recorded sentence is accepted; a second write of the same // sentence is a no-op, a different one is refused; and nothing but a landing // record takes it. func TestMarkRollbackRecordsASentenceOnALandingRecord(t *testing.T) { g := cutTestGuard() rec := &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveLanding, LandingPhase: LandConfigured}} if err := MoveMarkRollback(g, "some error text")(rec); err == nil || errors.Is(err, ErrMoveConflict) || rec.Move.Reason != "" { t.Fatalf("an unrecorded reason: %v %+v", err, rec.Move) } for _, reason := range []string{RollbackConfigChanged, RollbackLandingDeadline, RollbackClaimWired, RollbackLandingLayout, RollbackInstallMismatch, RollbackFilesOccupied} { rec := &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveLanding, LandingPhase: LandChecking}} if err := MoveMarkRollback(g, reason)(rec); err != nil || rec.Move.State != MoveLanding || rec.Move.LandingPhase != LandChecking || rec.Move.Reason != reason { t.Fatalf("%q: %v %+v", reason, err, rec.Move) } if err := MoveMarkRollback(g, reason)(rec); err != nil || rec.Move.Reason != reason { t.Fatalf("%q again: %v %+v", reason, err, rec.Move) } if err := MoveMarkRollback(g, RollbackLandingDeadline)(rec); reason != RollbackLandingDeadline && (!errors.Is(err, ErrMoveConflict) || rec.Move.Reason != reason) { t.Fatalf("%q then another: %v %+v", reason, err, rec.Move) } } for _, state := range []string{MoveCutting, MoveReady, MoveReverting, MoveCommitted} { rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: state}} if err := MoveMarkRollback(g, RollbackLandingDeadline)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Reason != "" { t.Fatalf("from %s: %v %+v", state, err, rec.Move) } } if got := LandingOrder(); len(got) != 9 || got[0] != LandChecking || got[8] != LandProbed { t.Fatalf("landing order: %v", got) } got := LandingOrder() got[0] = "x" if LandingOrder()[0] != LandChecking { t.Fatal("LandingOrder hands out the package's slice") } } func commitTestRecord(g MoveGuard, dl time.Time) *Record { return &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveLanding, LandingPhase: LandProbed, CutDeadline: &dl}} } func TestCommitRefusedPastTheDeadline(t *testing.T) { g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)} now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) dl := now.Add(115 * time.Second) cut := now.Add(-10 * time.Second) rec := &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveLanding, LandingPhase: LandProbed, CutDeadline: &dl, CutAt: &cut}} if err := MoveCommit(g, dl.Add(time.Second))(rec); !errors.Is(err, ErrCutDeadline) || !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveLanding { t.Fatalf("late commit: %v %s", err, rec.Move.State) } if err := MoveCommit(g, dl)(rec); err != nil || rec.Move.State != MoveCommitted || rec.Move.CommittedAt == nil { t.Fatalf("commit at the deadline: %v", err) } if err := MoveCommit(g, dl)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatal("second commit") } if err := MoveClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatal("close before open and park") } if err := MoveOpened(g, now)(rec); err != nil || rec.Move.OpenedAt == nil { t.Fatal(err) } if err := MoveClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil { t.Fatalf("close before the source park: %v", err) } if err := MoveSourceParked(g, now)(rec); err != nil { t.Fatal(err) } if err := MoveClose(g, now)(rec); err != nil || rec.Move != nil || rec.Region != "uk" || rec.LastMove == nil || rec.LastMove.ID != g.ID || rec.LastMove.Outcome != "done" { t.Fatalf("close: %v %+v", err, rec) } if rec.LastMove.OfflineS != 10 || !rec.LastMove.At.Equal(now) { t.Fatalf("last move: %+v", rec.LastMove) } } // Ruling H8: a recorded rollback wins over a commit, and the earlier of the // cut and landing deadlines bounds it. func TestCommitRefusesARecordedRollbackAndTheEarlierDeadline(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) cutDL, landDL := now.Add(115*time.Second), now.Add(60*time.Second) rec := commitTestRecord(g, cutDL) rec.Move.LandingDeadline = &landDL if err := MoveCommit(g, landDL.Add(time.Second))(rec); !errors.Is(err, ErrCutDeadline) || rec.Move.State != MoveLanding { t.Fatalf("past the landing deadline, inside the cut's: %v", err) } if err := MoveCommit(g, landDL)(rec); err != nil || rec.Move.State != MoveCommitted { t.Fatalf("at the earlier deadline: %v", err) } // The landing deadline alone does not stand in for the cut's. rec = commitTestRecord(g, landDL) rec.Move.LandingDeadline = &cutDL if err := MoveCommit(g, landDL.Add(time.Second))(rec); !errors.Is(err, ErrCutDeadline) { t.Fatalf("past the cut deadline, inside the landing's: %v", err) } // A recorded rollback reason refuses the commit; nothing is written. rec = commitTestRecord(g, cutDL) if err := MoveMarkRollback(g, RollbackLandingDeadline)(rec); err != nil { t.Fatal(err) } if err := MoveCommit(g, now)(rec); !errors.Is(err, ErrMoveConflict) || errors.Is(err, ErrCutDeadline) || rec.Move.State != MoveLanding || rec.Move.CommittedAt != nil { t.Fatalf("commit over a rollback marker: %v %+v", err, rec.Move) } // No cut deadline recorded: the commit cannot be shown to be in time. rec = commitTestRecord(g, cutDL) rec.Move.CutDeadline = nil if err := MoveCommit(g, now)(rec); !errors.Is(err, ErrCutDeadline) { t.Fatalf("no cut deadline: %v", err) } // Not probed yet, a replacement move, and a record with no move. for _, phase := range []string{LandChecking, LandRestored, LandProbing} { rec = commitTestRecord(g, cutDL) rec.Move.LandingPhase = phase if err := MoveCommit(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveLanding { t.Fatalf("commit from %s: %v", phase, err) } } other := MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)} if err := MoveCommit(other, now)(commitTestRecord(g, cutDL)); !errors.Is(err, ErrMoveConflict) { t.Fatalf("another incarnation: %v", err) } if err := MoveCommit(g, now)(&Record{}); !errors.Is(err, ErrMoveConflict) { t.Fatalf("no move: %v", err) } } func TestOpenedParkedAndCloseAreCommittedOnly(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) later := now.Add(time.Minute) for _, state := range []string{MoveLanding, MoveCutting, MoveReverting, MoveRollingBack} { for name, step := range map[string]func(*Record) error{"opened": MoveOpened(g, now), "parked": MoveSourceParked(g, now), "close": MoveClose(g, now)} { rec := commitTestRecord(g, now) rec.Move.State = state if err := step(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil || rec.Move.OpenedAt != nil || rec.Move.SourceParkedAt != nil { t.Fatalf("%s from %s: %v", name, state, err) } } } // A replay keeps the first stamp. rec := commitTestRecord(g, now) rec.Move.State = MoveCommitted if err := MoveOpened(g, now)(rec); err != nil { t.Fatal(err) } if err := MoveOpened(g, later)(rec); err != nil || !rec.Move.OpenedAt.Equal(now) { t.Fatalf("opened replay: %v %v", err, rec.Move.OpenedAt) } if err := MoveSourceParked(g, now)(rec); err != nil { t.Fatal(err) } if err := MoveSourceParked(g, later)(rec); err != nil || !rec.Move.SourceParkedAt.Equal(now) { t.Fatalf("parked replay: %v %v", err, rec.Move.SourceParkedAt) } // Close with no cut_at recorded (a record from before it was stamped): // the outcome is written and the offline time is left at zero. if err := MoveClose(g, later)(rec); err != nil || rec.Move != nil || rec.LastMove == nil || rec.LastMove.OfflineS != 0 { t.Fatalf("close without cut_at: %v %+v", err, rec.LastMove) } // Close refuses a record whose region is not the target's. rec = commitTestRecord(g, now) rec.Move.State, rec.Move.OpenedAt, rec.Move.SourceParkedAt = MoveCommitted, &now, &now rec.Region = "ie" if err := MoveClose(g, later)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil || rec.LastMove != nil { t.Fatalf("close with the region still the source's: %v", err) } } func TestProbeAttemptRecordsTheCountOnAProbingLanding(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) rec := commitTestRecord(g, now) rec.Move.LandingPhase = LandProbing for _, n := range []int{1, 1, 2, 3} { if err := MoveProbeAttempt(g, n)(rec); err != nil || rec.Move.Landing == nil || rec.Move.Landing.Attempts != n { t.Fatalf("attempt %d: %v %+v", n, err, rec.Move.Landing) } } rec.Move.Landing.InstalledDBSHA256 = "keep" if err := MoveProbeAttempt(g, 4)(rec); err != nil || rec.Move.Landing.InstalledDBSHA256 != "keep" { t.Fatalf("the install digest was lost: %v %+v", err, rec.Move.Landing) } for _, n := range []int{3, 0, -1} { if err := MoveProbeAttempt(g, n)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Landing.Attempts != 4 { t.Fatalf("attempt %d after 4: %v %+v", n, err, rec.Move.Landing) } } for _, phase := range []string{LandChecking, LandRestored, LandProbed} { rec := commitTestRecord(g, now) rec.Move.LandingPhase = phase if err := MoveProbeAttempt(g, 1)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Landing != nil { t.Fatalf("from %s: %v", phase, err) } } rec = commitTestRecord(g, now) rec.Move.LandingPhase = LandProbing if err := MoveMarkRollback(g, RollbackProbeAttempts)(rec); err != nil || rec.Move.Reason != RollbackProbeAttempts { t.Fatalf("out of attempts: %v %q", err, rec.Move.Reason) } if err := MoveProbeAttempt(g, 1)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Landing != nil { t.Fatalf("an attempt over a rollback marker: %v", err) } rec = commitTestRecord(g, now) if err := MoveMarkRollback(g, RollbackCommitDeadline)(rec); err != nil || rec.Move.Reason != RollbackCommitDeadline { t.Fatalf("commit past the deadline: %v %q", err, rec.Move.Reason) } rec = commitTestRecord(g, now) rec.Move.State = MoveCommitted if err := MoveProbeAttempt(g, 1)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("committed: %v", err) } } // ---- Plan 4 Task 1: rollback, revert, stuck and retry (§7.8, §7.10) ---- const rbTestID = "rb_abcdefghijklmnopqrstuvwxyz" // rollbackTestLanding is a landing record on the target at phase, inside // both deadlines, with the cut stamped ten seconds before now. func rollbackTestLanding(g MoveGuard, now time.Time, phase string) *Record { cut, dl, ldl := now.Add(-10*time.Second), now.Add(115*time.Second), now.Add(10*time.Minute) return &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveLanding, CutPhase: CutFlipped, LandingPhase: phase, CutAt: &cut, CutDeadline: &dl, LandingDeadline: &ldl, Landing: &MoveLandingJournal{InstalledDBSHA256: "sha256:db", Attempts: 2}}} } // rollingBackRecord is a record MoveRollBack has just written. func rollingBackRecord(t *testing.T, g MoveGuard, now time.Time) *Record { t.Helper() rec := rollbackTestLanding(g, now, LandProbing) if err := MoveRollBack(g, rbTestID, RollbackProbeAttempts, now)(rec); err != nil { t.Fatal(err) } return rec } func TestRollBackRacesTheCommit(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) // The commit first: the rollback is refused and nothing moves. rec := rollbackTestLanding(g, now, LandProbed) if err := MoveCommit(g, now)(rec); err != nil { t.Fatal(err) } err := MoveRollBack(g, rbTestID, RollbackLandingDeadline, now)(rec) if !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveCommitted || rec.Region != "uk" || rec.Move.Rollback != nil { t.Fatalf("rollback after the commit: %v %+v", err, rec.Move) } if !strings.HasSuffix(err.Error(), ": "+RefusedCommitted) { t.Fatalf("the commit's refusal: %v", err) } // The rollback first: the commit is refused and nothing moves. rec = rollbackTestLanding(g, now, LandProbed) if err := MoveRollBack(g, rbTestID, RollbackLandingDeadline, now)(rec); err != nil { t.Fatal(err) } if err := MoveCommit(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveRollingBack || rec.Move.CommittedAt != nil || rec.Region != "ie" { t.Fatalf("commit after the rollback: %v %+v", err, rec.Move) } } func TestRollBackStartsTheJournal(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) for _, phase := range LandingOrder() { rec := rollbackTestLanding(g, now, phase) if err := MoveRollBack(g, rbTestID, RollbackLandingDeadline, now)(rec); err != nil { t.Fatalf("from %s: %v", phase, err) } m, rb := rec.Move, rec.Move.Rollback if rec.Region != "ie" || m.State != MoveRollingBack || m.LandingPhase != phase || m.Landing == nil || m.Landing.InstalledDBSHA256 != "sha256:db" || m.Reason != "" { t.Fatalf("from %s: %+v", phase, m) } if rb == nil || rb.ID != rbTestID || rb.RequestedAt == nil || !rb.RequestedAt.Equal(now) || rb.Reason != RollbackLandingDeadline || rb.TargetPhase != RbTargetReleasing || rb.SourcePhase != RbSourceWaiting || rb.ReleaseDeadline == nil || !rb.ReleaseDeadline.Equal(now.Add(15*time.Minute)) || rb.Dirty || rb.StuckAt != "" || rb.LossRef != "" { t.Fatalf("from %s: rollback %+v", phase, rb) } } // A marked landing: the mark is copied to the rollback's reason, whatever // the caller names (the first reason stands), and stays on the move. for _, reason := range []string{"", RollbackDeclaredLost, RollbackLandingDeadline} { rec := rollbackTestLanding(g, now, LandConfigured) if err := MoveMarkRollback(g, RollbackConfigChanged)(rec); err != nil { t.Fatal(err) } if err := MoveRollBack(g, rbTestID, reason, now)(rec); err != nil || rec.Move.Rollback.Reason != RollbackConfigChanged || rec.Move.Reason != RollbackConfigChanged || rec.Move.StuckReason != "" { t.Fatalf("marked, caller says %q: %v %+v", reason, err, rec.Move.Rollback) } } // A marked landing still refuses free text from the caller. marked := rollbackTestLanding(g, now, LandConfigured) if err := MoveMarkRollback(g, RollbackConfigChanged)(marked); err != nil { t.Fatal(err) } if err := MoveRollBack(g, rbTestID, "declared lost: box-7", now)(marked); err == nil || marked.Move.State != MoveLanding || marked.Move.Rollback != nil || marked.Move.Reason != RollbackConfigChanged { t.Fatalf("marked, free text: %v %+v", err, marked.Move) } // Every recorded rollback reason is taken; the declared loss is one. for _, reason := range []string{RollbackConfigChanged, RollbackLandingDeadline, RollbackClaimWired, RollbackLandingLayout, RollbackInstallMismatch, RollbackFilesOccupied, RollbackProbeAttempts, RollbackCommitDeadline, RollbackDeclaredLost} { rec := rollbackTestLanding(g, now, LandChecking) if err := MoveRollBack(g, rbTestID, reason, now)(rec); err != nil || rec.Move.Rollback.Reason != reason { t.Fatalf("%q: %v", reason, err) } } // Refusals that write nothing: no reason on an unmarked landing, an // unrecorded reason, a malformed rollback id, a zero time. for name, step := range map[string]func(*Record) error{ "no reason": MoveRollBack(g, rbTestID, "", now), "free text": MoveRollBack(g, rbTestID, "declared lost: box-7", now), "move id": MoveRollBack(g, g.ID, RollbackLandingDeadline, now), "short id": MoveRollBack(g, "rb_short", RollbackLandingDeadline, now), "zero time": MoveRollBack(g, rbTestID, RollbackLandingDeadline, time.Time{}), "another guard": MoveRollBack(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, rbTestID, RollbackLandingDeadline, now), } { rec := rollbackTestLanding(g, now, LandChecking) if err := step(rec); err == nil || rec.Move.State != MoveLanding || rec.Region != "uk" || rec.Move.Rollback != nil { t.Fatalf("%s: %v %+v", name, err, rec.Move) } } // Only a landing rolls back: a replay finds rolling_back and is a // conflict, which the caller answers by re-reading. for _, state := range []string{MoveReady, MoveCutting, MoveReverting, MoveRollingBack, MoveStuck, MoveCommitted} { rec := rollbackTestLanding(g, now, LandChecking) rec.Move.State = state if err := MoveRollBack(g, rbTestID, RollbackLandingDeadline, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Region != "uk" || rec.Move.Rollback != nil { t.Fatalf("from %s: %v", state, err) } } } func TestRollbackTargetPhasesInOrder(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) step := func(rec *Record, from, to string, stamp func(*MoveRollback)) error { return MoveRollbackTargetPhase(g, from, to, stamp)(rec) } recovered := func(rb *MoveRollback) { at := now; rb.RecoveredAt = &at } // Dirty: every phase, in order; cleaning waits for the source's // recovery (Review Focus 4). rec := rollingBackRecord(t, g, now) if err := step(rec, RbTargetReleasing, RbTargetReturning, nil); !errors.Is(err, ErrMoveConflict) { t.Fatalf("skip released: %v", err) } if err := step(rec, RbTargetReleasing, RbTargetReleased, func(rb *MoveRollback) { rb.Dirty, rb.Evidence = true, "stopped-target.json" }); err != nil || !rec.Move.Rollback.Dirty { t.Fatal(err) } if err := step(rec, RbTargetReleased, RbTargetCleaning, nil); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.TargetPhase != RbTargetReleased { t.Fatalf("a dirty release skipped the return: %v", err) } for _, s := range [][2]string{{RbTargetReleased, RbTargetReturning}, {RbTargetReturning, RbTargetReturned}} { if err := step(rec, s[0], s[1], nil); err != nil { t.Fatalf("%s to %s: %v", s[0], s[1], err) } } if err := step(rec, RbTargetReturned, RbTargetCleaning, nil); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.TargetPhase != RbTargetReturned { t.Fatalf("cleaning before the source recovered: %v", err) } recovered(rec.Move.Rollback) if err := step(rec, RbTargetReturned, RbTargetCleaning, nil); err != nil { t.Fatal(err) } if err := step(rec, RbTargetCleaning, RbTargetCleaned, func(rb *MoveRollback) { at := now; rb.TargetParkedAt = &at }); err != nil || rec.Move.Rollback.TargetParkedAt == nil { t.Fatal(err) } if err := step(rec, RbTargetCleaned, RbTargetReleasing, nil); !errors.Is(err, ErrMoveConflict) { t.Fatalf("past cleaned: %v", err) } // Clean: released goes straight to cleaning and never returns. rec = rollingBackRecord(t, g, now) if err := step(rec, RbTargetReleasing, RbTargetReleased, nil); err != nil { t.Fatal(err) } if err := step(rec, RbTargetReleased, RbTargetReturning, nil); !errors.Is(err, ErrMoveConflict) { t.Fatalf("a clean release returned: %v", err) } recovered(rec.Move.Rollback) if err := step(rec, RbTargetReleased, RbTargetCleaning, nil); err != nil || rec.Move.Rollback.TargetPhase != RbTargetCleaning { t.Fatalf("the clean skip: %v", err) } // The wrong from, the wrong guard, a stuck record, a landing: refused. rec = rollingBackRecord(t, g, now) if err := step(rec, RbTargetReleased, RbTargetReturning, nil); !errors.Is(err, ErrMoveConflict) { t.Fatalf("wrong from: %v", err) } if err := MoveRollbackTargetPhase(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, RbTargetReleasing, RbTargetReleased, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("another guard: %v", err) } rec.Move.State = MoveStuck if err := step(rec, RbTargetReleasing, RbTargetReleased, nil); !errors.Is(err, ErrMoveConflict) { t.Fatalf("stuck: %v", err) } if err := step(rollbackTestLanding(g, now, LandProbing), RbTargetReleasing, RbTargetReleased, nil); !errors.Is(err, ErrMoveConflict) { t.Fatalf("landing: %v", err) } } func TestRollbackTargetLostAndTheLossCommit(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) const ref = "control/losses/a.go.example.com/loss_1.json" at := func(phase string, dirty bool) *Record { rec := rollingBackRecord(t, g, now) rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = phase, dirty return rec } for _, c := range []struct { phase string dirty bool }{{RbTargetReleasing, false}, {RbTargetReturning, true}, {RbTargetReleased, true}} { rec := at(c.phase, c.dirty) if err := MoveRollbackTargetLost(g, ref, now)(rec); err != nil { t.Fatalf("from %s: %v", c.phase, err) } rb := rec.Move.Rollback if rb.TargetPhase != RbTargetLost || rb.LossRef != ref || rb.LostAt == nil || !rb.LostAt.Equal(now) || rb.Dirty != c.dirty || rec.Move.State != MoveRollingBack { t.Fatalf("from %s: %+v", c.phase, rb) } // A replay of the same declaration changes nothing; another is refused. if err := MoveRollbackTargetLost(g, ref, now.Add(time.Minute))(rec); err != nil || !rb.LostAt.Equal(now) { t.Fatalf("replay from %s: %v", c.phase, err) } if err := MoveRollbackTargetLost(g, ref+"x", now)(rec); !errors.Is(err, ErrMoveConflict) || rb.LossRef != ref { t.Fatalf("another declaration from %s: %v", c.phase, err) } // The forward order never leaves lost, and the source never takes it. for _, to := range []string{RbTargetReleased, RbTargetReturning, RbTargetReturned, RbTargetCleaning} { if err := MoveRollbackTargetPhase(g, RbTargetLost, to, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("lost to %s: %v", to, err) } } if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("the source consumed lost: %v", err) } // The loss commit: lost to released, clean, evidence the loss record. if err := MoveRollbackLossCommitted(g)(rec); err != nil || rb.TargetPhase != RbTargetReleased || rb.Dirty || rb.Evidence != ref || rb.LossRef != ref { t.Fatalf("loss commit from %s: %v %+v", c.phase, err, rb) } if err := MoveRollbackLossCommitted(g)(rec); err != nil || rb.TargetPhase != RbTargetReleased { t.Fatalf("loss commit replay: %v", err) } if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); err != nil { t.Fatalf("the source after the loss commit: %v", err) } } // Refused: released clean (nothing left to declare), returned, cleaning, // cleaned; and the loss commit from anything but lost. for _, c := range []struct { phase string dirty bool }{{RbTargetReleased, false}, {RbTargetReturned, true}, {RbTargetCleaning, true}, {RbTargetCleaned, false}} { rec := at(c.phase, c.dirty) err := MoveRollbackTargetLost(g, ref, now)(rec) if !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.TargetPhase != c.phase || rec.Move.Rollback.LossRef != "" { t.Fatalf("lost from %s: %v", c.phase, err) } if !strings.HasSuffix(err.Error(), ": "+RefusedNothingToDeclare) { t.Fatalf("lost from %s: %v", c.phase, err) } } for _, phase := range []string{RbTargetReleasing, RbTargetReleased, RbTargetReturning, RbTargetReturned} { rec := at(phase, true) if err := MoveRollbackLossCommitted(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.TargetPhase != phase || !rec.Move.Rollback.Dirty { t.Fatalf("loss commit from %s: %v", phase, err) } } rec := at(RbTargetReleasing, false) rec.Move.State = MoveStuck if err := MoveRollbackTargetLost(g, ref, now)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("stuck: %v", err) } if err := MoveRollbackTargetLost(g, "", now)(at(RbTargetReleasing, false)); err == nil { t.Fatal("an empty loss reference") } } func TestRollbackSourcePhasesCleanAndDirty(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) walk := func(rec *Record, order []string) { t.Helper() for i := 0; i+1 < len(order); i++ { if err := MoveRollbackSourcePhase(g, order[i], order[i+1], nil)(rec); err != nil || rec.Move.Rollback.SourcePhase != order[i+1] { t.Fatalf("%s to %s: %v", order[i], order[i+1], err) } } } // waiting holds until the target's evidence: released clean, returned dirty. rec := rollingBackRecord(t, g, now) if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("waiting while the target releases: %v", err) } rec.Move.Rollback.TargetPhase = RbTargetReleased walk(rec, []string{RbSourceWaiting, RbSourceClaimed, RbSourceRestoring}) if err := MoveRollbackSourcePhase(g, RbSourceRestoring, RbSourceParking, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("a clean return installed: %v", err) } if err := MoveRollbackSourcePhase(g, RbSourceRestoring, RbSourceProbing, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("a clean return skipped verification: %v", err) } walk(rec, []string{RbSourceRestoring, RbSourceVerified, RbSourceProbing, RbSourceProbed}) if err := MoveRollbackSourcePhase(g, RbSourceProbed, RbSourceRecovered, func(rb *MoveRollback) { at := now; rb.RecoveredAt = &at })(rec); err != nil || rec.Move.Rollback.RecoveredAt == nil { t.Fatal(err) } walk(rec, []string{RbSourceRecovered, RbSourceOpened}) if err := MoveRollbackSourcePhase(g, RbSourceOpened, RbSourceWaiting, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("past opened: %v", err) } // Dirty: a released dirty target is not yet evidence; returned is. rec = rollingBackRecord(t, g, now) rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = RbTargetReleased, true if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("waiting while the target has not returned: %v", err) } rec.Move.Rollback.TargetPhase = RbTargetReturned walk(rec, []string{RbSourceWaiting, RbSourceClaimed, RbSourceRestoring}) if err := MoveRollbackSourcePhase(g, RbSourceRestoring, RbSourceVerified, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("a dirty return verified instead of installing: %v", err) } walk(rec, []string{RbSourceRestoring, RbSourceParking, RbSourceBuilding, RbSourcePublished, RbSourceLocal, RbSourceConfigured, RbSourceRestored, RbSourceProbing, RbSourceProbed, RbSourceRecovered, RbSourceOpened}) // Skips, the wrong guard and a stuck record are refused. rec = rollingBackRecord(t, g, now) rec.Move.Rollback.TargetPhase = RbTargetReleased if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceRestoring, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("skip claimed: %v", err) } if err := MoveRollbackSourcePhase(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("another guard: %v", err) } rec.Move.State = MoveStuck if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("stuck: %v", err) } } func TestMoveRetryEpochRefusals(t *testing.T) { for _, rollback := range []bool{false, true} { for _, epoch := range []uint64{0, 7, ^uint64(0)} { g := cutTestGuard() rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveStuck, StuckReason: StuckRecoveryProbeAttempts, RecoveryRetryEpoch: epoch}} if rollback { rec.Move.Rollback = &MoveRollback{ID: rbTestID, StuckAt: RbSourceProbing} } before, _ := json.Marshal(rec) err := MoveRetryRecovery(g)(rec) if epoch == ^uint64(0) { after, _ := json.Marshal(rec) if !errors.Is(err, ErrMoveConflict) || string(before) != string(after) { t.Fatalf("epoch overflow mutated record: %v %+v", err, rec.Move) } continue } if err != nil || rec.Move.RecoveryRetryEpoch != epoch+1 { t.Fatalf("retry epoch: %v %+v", err, rec.Move) } before, _ = json.Marshal(rec) err = MoveRetryRecovery(g)(rec) after, _ := json.Marshal(rec) if !errors.Is(err, ErrMoveConflict) || string(before) != string(after) { t.Fatalf("retry replay mutated record: %v %+v", err, rec.Move) } } } } // The legacy retry (flag off) must write the same bytes it always did: it // never touches the recovery epoch, set or not. func TestLegacyMoveRetryLeavesTheEpochAlone(t *testing.T) { for _, rollback := range []bool{false, true} { for _, epoch := range []uint64{0, 7, ^uint64(0)} { g := cutTestGuard() rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveStuck, StuckReason: StuckRecoveryProbeAttempts, RecoveryRetryEpoch: epoch}} if rollback { rec.Move.Rollback = &MoveRollback{ID: rbTestID, StuckAt: RbSourceProbing} } if err := MoveRetry(g)(rec); err != nil || rec.Move.RecoveryRetryEpoch != epoch { t.Fatalf("legacy retry: %v epoch %d, want %d", err, rec.Move.RecoveryRetryEpoch, epoch) } raw, _ := json.Marshal(rec.Move) if epoch == 0 && strings.Contains(string(raw), "recovery_retry_epoch") { t.Fatalf("legacy retry wrote the epoch: %s", raw) } } } } func TestStuckAndRetryReturnToThePhase(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) at := func(phase string) *Record { rec := rollingBackRecord(t, g, now) rec.Move.Rollback.TargetPhase, rec.Move.Rollback.SourcePhase = RbTargetReleased, phase return rec } for _, c := range []struct{ phase, reason string }{ {RbSourceRestoring, StuckBaselineMismatch}, {RbSourceVerified, StuckBaselineMismatch}, {RbSourceProbing, StuckRecoveryProbeAttempts}, } { rec := at(c.phase) if err := MoveMarkStuck(g, c.reason)(rec); err != nil { t.Fatalf("%s: %v", c.phase, err) } m, rb := rec.Move, rec.Move.Rollback if m.State != MoveStuck || m.StuckReason != c.reason || m.Reason != "" || rb.StuckAt != c.phase || rb.SourcePhase != c.phase || rb.Reason != RollbackProbeAttempts { t.Fatalf("%s: %+v %+v", c.phase, m, rb) } // The same mark again is a replay; another is refused. if err := MoveMarkStuck(g, c.reason)(rec); err != nil { t.Fatalf("%s replay: %v", c.phase, err) } other := StuckBaselineMismatch if c.reason == other { other = StuckRecoveryProbeAttempts } if err := MoveMarkStuck(g, other)(rec); !errors.Is(err, ErrMoveConflict) || m.StuckReason != c.reason { t.Fatalf("%s another reason: %v", c.phase, err) } // Nothing advances a stuck source. if err := MoveRollbackSourcePhase(g, c.phase, RbSourceProbed, nil)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("%s advanced while stuck: %v", c.phase, err) } // Retry: back to rolling_back at the phase it stuck in, so a // mismatch verifies again; the stuck reason goes, the rollback's stays. if err := MoveRetry(g)(rec); err != nil { t.Fatalf("%s retry: %v", c.phase, err) } if m.State != MoveRollingBack || m.StuckReason != "" || m.Reason != "" || rb.SourcePhase != c.phase || rb.StuckAt != "" || rb.Reason != RollbackProbeAttempts { t.Fatalf("%s after retry: %+v %+v", c.phase, m, rb) } if err := MoveRetry(g)(rec); !errors.Is(err, ErrMoveConflict) || !strings.HasSuffix(err.Error(), ": "+RefusedNotStuck) { t.Fatalf("%s retry twice: %v", c.phase, err) } } // Each stuck reason is the phase's own, and only these phases stick. for _, c := range []struct{ phase, reason string }{ {RbSourceProbing, StuckBaselineMismatch}, {RbSourceVerified, StuckRecoveryProbeAttempts}, {RbSourceVerified, StuckRevertProbeBudget}, {RbSourceWaiting, StuckBaselineMismatch}, {RbSourceClaimed, StuckBaselineMismatch}, {RbSourceParking, StuckBaselineMismatch}, {RbSourceProbed, StuckRecoveryProbeAttempts}, {RbSourceOpened, StuckRecoveryProbeAttempts}, } { rec := at(c.phase) if err := MoveMarkStuck(g, c.reason)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveRollingBack || rec.Move.Rollback.StuckAt != "" { t.Fatalf("%s with %q: %v", c.phase, c.reason, err) } } if err := MoveMarkStuck(g, "the probe failed: connection refused")(at(RbSourceProbing)); err == nil || errors.Is(err, ErrMoveConflict) { t.Fatalf("free text: %v", err) } // A marked rollback keeps its cause, on the move and on the journal, // through a stuck and a retry: only the stuck reason comes and goes. marked := rollbackTestLanding(g, now, LandProbing) if err := MoveMarkRollback(g, RollbackProbeAttempts)(marked); err != nil { t.Fatal(err) } if err := MoveRollBack(g, rbTestID, "", now)(marked); err != nil { t.Fatal(err) } marked.Move.Rollback.TargetPhase, marked.Move.Rollback.SourcePhase = RbTargetReleased, RbSourceProbing if err := MoveMarkStuck(g, StuckRecoveryProbeAttempts)(marked); err != nil || marked.Move.StuckReason != StuckRecoveryProbeAttempts || marked.Move.Reason != RollbackProbeAttempts || marked.Move.Rollback.Reason != RollbackProbeAttempts { t.Fatalf("marked rollback stuck: %v %+v", err, marked.Move) } if err := MoveRetry(g)(marked); err != nil || marked.Move.StuckReason != "" || marked.Move.Reason != RollbackProbeAttempts || marked.Move.Rollback.Reason != RollbackProbeAttempts { t.Fatalf("marked rollback retry: %v %+v", err, marked.Move) } // A revert sticks with no rollback journal and retries to reverting; its // own cause survives both. rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReverting, CutPhase: CutStaged, Reason: RevertCutDeadline}} // A revert's probe has the recovery probe's nine attempts as well as its // fifteen minutes (§7.7; Task 6 fix round 1), so both reasons are a // revert's. for _, reason := range []string{StuckRevertProbeBudget, StuckBaselineMismatch, StuckRecoveryProbeAttempts} { if err := MoveMarkStuck(g, reason)(rec); err != nil || rec.Move.State != MoveStuck || rec.Move.StuckReason != reason || rec.Move.Reason != RevertCutDeadline || rec.Move.Rollback != nil { t.Fatalf("revert %q: %v %+v", reason, err, rec.Move) } if err := MoveRetry(g)(rec); err != nil || rec.Move.State != MoveReverting || rec.Move.StuckReason != "" || rec.Move.Reason != RevertCutDeadline || rec.Move.Rollback != nil || rec.Move.CutPhase != CutStaged { t.Fatalf("revert retry: %v %+v", err, rec.Move) } } // Neither a landing nor a committed move sticks or retries. for _, state := range []string{MoveLanding, MoveCommitted, MoveCutting} { rec := rollbackTestLanding(g, now, LandProbing) rec.Move.State = state if err := MoveMarkStuck(g, StuckRecoveryProbeAttempts)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != state { t.Fatalf("stuck from %s: %v", state, err) } if err := MoveRetry(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != state { t.Fatalf("retry from %s: %v", state, err) } } // A stuck rollback that records no phase is not retried blind. rec = at(RbSourceProbing) rec.Move.State, rec.Move.StuckReason = MoveStuck, StuckRecoveryProbeAttempts if err := MoveRetry(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveStuck { t.Fatalf("retry with no stuck phase: %v", err) } } // The dirty return's install (Plan 4 Task 3, ruling J13) refuses into stuck, // never into a landing's rollback: the files paths still occupied after the // source's own pre-move files were parked stop it at local, and a build that // does not verify stops it at configured (where the database is installed). // Each reason is its phase's own, for a dirty return only, and a retry goes // back to that phase. func TestStuckReturnInstallReasons(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) at := func(phase string, dirty bool) *Record { rec := rollingBackRecord(t, g, now) rec.Move.Rollback.TargetPhase, rec.Move.Rollback.SourcePhase, rec.Move.Rollback.Dirty = RbTargetReturned, phase, dirty return rec } for _, c := range []struct{ phase, reason string }{ {RbSourceLocal, StuckReturnFilesOccupied}, {RbSourceConfigured, StuckReturnInstallMismatch}, {RbSourceLocal, StuckReturnUnknownStamp}, } { rec := at(c.phase, true) if err := MoveMarkStuck(g, c.reason)(rec); err != nil { t.Fatalf("%s: %v", c.phase, err) } m, rb := rec.Move, rec.Move.Rollback if m.State != MoveStuck || m.StuckReason != c.reason || rb.StuckAt != c.phase || rb.Reason != RollbackProbeAttempts { t.Fatalf("%s: %+v %+v", c.phase, m, rb) } if err := MoveMarkStuck(g, c.reason)(rec); err != nil { t.Fatalf("%s replay: %v", c.phase, err) } if err := MoveRetry(g)(rec); err != nil || m.State != MoveRollingBack || rb.SourcePhase != c.phase || m.StuckReason != "" { t.Fatalf("%s retry: %v %+v %+v", c.phase, err, m, rb) } } for _, c := range []struct { phase, reason string dirty bool }{ {RbSourceLocal, StuckReturnFilesOccupied, false}, {RbSourceConfigured, StuckReturnInstallMismatch, false}, {RbSourceConfigured, StuckReturnFilesOccupied, true}, {RbSourceLocal, StuckReturnInstallMismatch, true}, {RbSourceParking, StuckReturnFilesOccupied, true}, {RbSourceRestored, StuckReturnInstallMismatch, true}, {RbSourceVerified, StuckReturnFilesOccupied, false}, {RbSourceLocal, StuckReturnUnknownStamp, false}, {RbSourceConfigured, StuckReturnUnknownStamp, true}, {RbSourceParking, StuckReturnUnknownStamp, true}, } { rec := at(c.phase, c.dirty) if err := MoveMarkStuck(g, c.reason)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveRollingBack { t.Fatalf("%s (dirty %v) with %q: %v", c.phase, c.dirty, c.reason, err) } } // A revert installs nothing, so neither reason is a revert's. rev := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReverting, CutPhase: CutStaged, Reason: RevertCutDeadline}} for _, reason := range []string{StuckReturnFilesOccupied, StuckReturnInstallMismatch, StuckReturnUnknownStamp} { if err := MoveMarkStuck(g, reason)(rev); !errors.Is(err, ErrMoveConflict) || rev.Move.State != MoveReverting { t.Fatalf("a revert with %q: %v", reason, err) } } } // A dirty return the target cannot build (Task 4 fix round 1, ruling 2) // stops for an operator: the database is missing, the target changed after // it stopped, or its files cannot be archived. Each reason is the target's // returning phase's own, on a dirty rollback; the target phase and Dirty // stay, the source's phase (waiting) is StuckAt, and a retry goes back to // it, so the target runs its return again. func TestStuckReturnTargetReasons(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) at := func(target string, dirty bool) *Record { rec := rollingBackRecord(t, g, now) rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = target, dirty return rec } reasons := []string{StuckReturnTargetDatabase, StuckReturnTargetChanged, StuckReturnTargetFiles} for _, reason := range reasons { rec := at(RbTargetReturning, true) if err := MoveMarkStuck(g, reason)(rec); err != nil { t.Fatalf("%q: %v", reason, err) } m, rb := rec.Move, rec.Move.Rollback if m.State != MoveStuck || m.StuckReason != reason || rb.StuckAt != RbSourceWaiting || rb.TargetPhase != RbTargetReturning || !rb.Dirty { t.Fatalf("%q: %+v %+v", reason, m, rb) } if err := MoveMarkStuck(g, reason)(rec); err != nil { t.Fatalf("%q replay: %v", reason, err) } if err := MoveRetry(g)(rec); err != nil || m.State != MoveRollingBack || rb.SourcePhase != RbSourceWaiting || rb.TargetPhase != RbTargetReturning || m.StuckReason != "" { t.Fatalf("%q retry: %v %+v %+v", reason, err, m, rb) } } for _, reason := range reasons { for _, c := range []struct { target string dirty bool }{{RbTargetReturning, false}, {RbTargetReleasing, true}, {RbTargetReleased, true}, {RbTargetReturned, true}, {RbTargetCleaning, true}} { rec := at(c.target, c.dirty) if err := MoveMarkStuck(g, reason)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveRollingBack { t.Fatalf("%q at %s (dirty %v): %v", reason, c.target, c.dirty, err) } } // The source's own install phases are not the target's. rec := at(RbTargetReturned, true) rec.Move.Rollback.SourcePhase = RbSourceLocal if err := MoveMarkStuck(g, reason)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("%q at the source's local: %v", reason, err) } rev := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReverting, CutPhase: CutStaged, Reason: RevertCutDeadline}} if err := MoveMarkStuck(g, reason)(rev); !errors.Is(err, ErrMoveConflict) { t.Fatalf("a revert with %q: %v", reason, err) } } } // TestStuckReleaseTargetFiles (Plan 5 Task 9a): the target's release finds // files it can never archive. The reason belongs to the target's releasing, // with the source still waiting, dirty decided or not; the target phase // stays releasing, the source's waiting is StuckAt, and a retry runs the // release again. Anywhere else it is a conflict. func TestStuckReleaseTargetFiles(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) at := func(target string, dirty bool) *Record { rec := rollingBackRecord(t, g, now) rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = target, dirty return rec } for _, dirty := range []bool{false, true} { rec := at(RbTargetReleasing, dirty) if err := MoveMarkStuck(g, StuckReleaseTargetFiles)(rec); err != nil { t.Fatalf("dirty %v: %v", dirty, err) } m, rb := rec.Move, rec.Move.Rollback if m.State != MoveStuck || m.StuckReason != StuckReleaseTargetFiles || rb.StuckAt != RbSourceWaiting || rb.TargetPhase != RbTargetReleasing { t.Fatalf("dirty %v: %+v %+v", dirty, m, rb) } if err := MoveMarkStuck(g, StuckReleaseTargetFiles)(rec); err != nil { t.Fatalf("replay: %v", err) } if err := MoveRetry(g)(rec); err != nil || m.State != MoveRollingBack || rb.SourcePhase != RbSourceWaiting || rb.TargetPhase != RbTargetReleasing || m.StuckReason != "" { t.Fatalf("retry: %v %+v %+v", err, m, rb) } } for _, target := range []string{RbTargetReleased, RbTargetReturning, RbTargetReturned, RbTargetCleaning} { if err := MoveMarkStuck(g, StuckReleaseTargetFiles)(at(target, true)); !errors.Is(err, ErrMoveConflict) { t.Fatalf("at %s: %v", target, err) } } rec := at(RbTargetReleasing, false) rec.Move.Rollback.SourcePhase = RbSourceClaimed if err := MoveMarkStuck(g, StuckReleaseTargetFiles)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("with the source claimed: %v", err) } if StuckReleaseTargetFiles == StuckReturnTargetFiles { t.Fatal("the release's reason must differ from the return's") } } func TestRolledBackCloseNeedsAllThreeStamps(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) later := now.Add(3 * time.Minute) opened := now.Add(50 * time.Second) full := func() *Record { rec := rollingBackRecord(t, g, now) rb := rec.Move.Rollback rb.TargetPhase, rb.SourcePhase = RbTargetCleaned, RbSourceOpened rb.RecoveredAt, rb.OpenedAt, rb.TargetParkedAt = &now, &opened, &now return rec } for name, clear := range map[string]func(*MoveRollback){ "recovered": func(rb *MoveRollback) { rb.RecoveredAt = nil }, "opened": func(rb *MoveRollback) { rb.OpenedAt = nil }, "target parked": func(rb *MoveRollback) { rb.TargetParkedAt = nil }, } { rec := full() clear(rec.Move.Rollback) if err := MoveRolledBackClose(g, later)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil || rec.LastMove != nil { t.Fatalf("close without %s: %v", name, err) } } rec := full() rec.Move.State = MoveStuck if err := MoveRolledBackClose(g, later)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil { t.Fatalf("close a stuck record: %v", err) } rec = full() rec.Region = "uk" if err := MoveRolledBackClose(g, later)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil { t.Fatalf("close with the region not the source's: %v", err) } rec = full() if err := MoveRolledBackClose(g, later)(rec); err != nil || rec.Move != nil || rec.Region != "ie" { t.Fatalf("close: %v %+v", err, rec) } // OfflineS is the rollback's open minus the cut: 50 s + 10 s. if lm := rec.LastMove; lm == nil || lm.ID != g.ID || lm.Outcome != "rolled_back" || !lm.At.Equal(later) || lm.OfflineS != 60 { t.Fatalf("last move: %+v", rec.LastMove) } } func TestRevertCloseCountsToTheReopen(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) cut := now.Add(-30 * time.Second) reverting := func() *Record { return &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReverting, CutPhase: CutStopped, CutAt: &cut, Reason: RevertCutDeadline}} } rec := reverting() if err := MoveRevertClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil { t.Fatalf("close before the reopen: %v", err) } if err := MoveRevertReopened(g, now)(rec); err != nil || rec.Move.ReopenedAt == nil || !rec.Move.ReopenedAt.Equal(now) { t.Fatalf("reopened: %v", err) } if err := MoveRevertReopened(g, now.Add(time.Minute))(rec); err != nil || !rec.Move.ReopenedAt.Equal(now) { t.Fatalf("reopened replay keeps the first stamp: %v", err) } if err := MoveRevertClose(g, now.Add(time.Minute))(rec); err != nil || rec.Move != nil || rec.Region != "ie" { t.Fatalf("revert close: %v %+v", err, rec) } if lm := rec.LastMove; lm == nil || lm.ID != g.ID || lm.Outcome != "rolled_back" || lm.OfflineS != 30 || !lm.At.Equal(now.Add(time.Minute)) { t.Fatalf("last move: %+v", rec.LastMove) } // Only a revert reopens or closes this way. for _, state := range []string{MoveCutting, MoveLanding, MoveRollingBack, MoveStuck, MoveCommitted} { rec := reverting() rec.Move.State, rec.Move.ReopenedAt = state, &now if err := MoveRevertClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil { t.Fatalf("revert close from %s: %v", state, err) } rec.Move.ReopenedAt = nil if err := MoveRevertReopened(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.ReopenedAt != nil { t.Fatalf("reopened from %s: %v", state, err) } } rec = reverting() rec.Move.ReopenedAt, rec.Region = &now, "uk" if err := MoveRevertClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil { t.Fatalf("revert close away from the source: %v", err) } } // The new journal fields survive the record's own write path, under their // own keys, and stay off a record that does not use them. func TestRollbackJournalThroughTheRecordWrite(t *testing.T) { s := release.NewDirStore(t.TempDir()) rec := moveTestRecord(t, s) g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) put := rollbackTestLanding(g, now, LandProbing) ctx := context.Background() if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { r.Region, r.Move = put.Region, put.Move; return nil }); err != nil { t.Fatal(err) } const ref = "control/losses/a.go.example.com/loss_1.json" for _, step := range []func(*Record) error{ MoveRollBack(g, rbTestID, RollbackDeclaredLost, now), MoveRollbackTargetLost(g, ref, now), } { if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, step); err != nil { t.Fatal(err) } } got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host) if err != nil { t.Fatal(err) } if got.Region != "ie" || got.Move.State != MoveRollingBack || got.Move.Rollback.ID != rbTestID || got.Move.Rollback.Reason != RollbackDeclaredLost || got.Move.Rollback.TargetPhase != RbTargetLost || got.Move.Rollback.LossRef != ref || got.Move.Rollback.LostAt == nil { t.Fatalf("read back: %+v %+v", got.Move, got.Move.Rollback) } raw, err := json.Marshal(MoveState{ReopenedAt: &now, StuckReason: StuckRecoveryProbeAttempts, Rollback: &MoveRollback{StuckAt: RbSourceProbing, LossRef: ref, LostAt: &now}}) if err != nil { t.Fatal(err) } for _, key := range []string{`"reopened_at"`, `"stuck_reason":"the recovery probe ran out of attempts"`, `"stuck_at":"probing"`, `"loss_ref"`, `"lost_at"`} { if !strings.Contains(string(raw), key) { t.Fatalf("%s missing from %s", key, raw) } } if raw, _ := json.Marshal(MoveState{Rollback: &MoveRollback{}}); strings.Contains(string(raw), "stuck_at") || strings.Contains(string(raw), "stuck_reason") || strings.Contains(string(raw), "reopened_at") || strings.Contains(string(raw), "loss_ref") { t.Fatalf("the new keys are not omitempty: %s", raw) } } func TestRollbackIDShape(t *testing.T) { for id, want := range map[string]bool{rbTestID: true, "mv_abcdefghijklmnopqrstuvwxyz": false, "rb_short": false, "rb_ABCDEFGHIJKLMNOPQRSTUVWXYZ": false, "": false} { if ValidRollbackID(id) != want { t.Fatalf("ValidRollbackID(%q) = %v", id, !want) } } } // A recovery probe records the inputs it renders before it renders them // (Plan 4 Task 5; §7.7, J2-R2-5; Task 3b concern 3): MoveRecordRecoveryInputs // writes Rollback.RecoveryInputs on this rollback while its source is at // probing, and nothing else. It replaces what configured recorded (a bundle // published since is the probe's to render, never compared with // move.inputs), and is refused at any other phase, on another rollback, on // a stuck record and without a release. func TestRecoveryInputsRecordedAtProbing(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) at := func(phase string, dirty bool) *Record { rec := rollingBackRecord(t, g, now) rec.Move.Rollback.TargetPhase, rec.Move.Rollback.SourcePhase, rec.Move.Rollback.Dirty = RbTargetReturned, phase, dirty rec.Move.Inputs = &MoveInputs{BundleETags: map[string]string{"default": "e1"}, Release: "v7"} return rec } first := MoveInputs{BundleETags: map[string]string{"default": "e2"}, OverlayETag: "o2", Release: "v8"} later := MoveInputs{BundleETags: map[string]string{"default": "e3"}, Release: "v8"} for _, dirty := range []bool{false, true} { rec := at(RbSourceProbing, dirty) if dirty { rec.Move.Rollback.RecoveryInputs = &MoveInputs{BundleETags: map[string]string{"default": "e2"}, Release: "v8"} } if err := MoveRecordRecoveryInputs(g, rbTestID, first)(rec); err != nil { t.Fatalf("dirty %v: %v", dirty, err) } rb := rec.Move.Rollback if rb.RecoveryInputs == nil || rb.RecoveryInputs.BundleETags["default"] != "e2" || rb.RecoveryInputs.OverlayETag != "o2" || rb.RecoveryInputs.Release != "v8" { t.Fatalf("dirty %v: recorded %+v", dirty, rb.RecoveryInputs) } if rec.Move.Inputs.BundleETags["default"] != "e1" || rec.Move.Inputs.Release != "v7" || rb.SourcePhase != RbSourceProbing || rec.Move.State != MoveRollingBack { t.Fatalf("dirty %v: the record changed beyond the inputs: %+v %+v", dirty, rec.Move, rb) } // The caller's map is copied, not aliased. first.BundleETags["default"] = "mutated" if rb.RecoveryInputs.BundleETags["default"] != "e2" { t.Fatalf("dirty %v: the record aliases the caller's map", dirty) } first.BundleETags["default"] = "e2" // A restarted probe records what it renders now. if err := MoveRecordRecoveryInputs(g, rbTestID, later)(rec); err != nil || rb.RecoveryInputs.BundleETags["default"] != "e3" { t.Fatalf("dirty %v: a later probe: %v %+v", dirty, err, rb.RecoveryInputs) } } for _, phase := range []string{RbSourceWaiting, RbSourceClaimed, RbSourceRestoring, RbSourceVerified, RbSourceConfigured, RbSourceRestored, RbSourceProbed, RbSourceRecovered, RbSourceOpened} { rec := at(phase, phase == RbSourceConfigured || phase == RbSourceRestored) if err := MoveRecordRecoveryInputs(g, rbTestID, first)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.RecoveryInputs != nil { t.Fatalf("at %s: %v %+v", phase, err, rec.Move.Rollback.RecoveryInputs) } } rec := at(RbSourceProbing, false) if err := MoveRecordRecoveryInputs(g, "rb_bbbbbbbbbbbbbbbbbbbbbbbbbb", first)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.RecoveryInputs != nil { t.Fatalf("another rollback: %v", err) } if err := MoveRecordRecoveryInputs(g, rbTestID, MoveInputs{BundleETags: map[string]string{"default": "e2"}})(rec); err == nil || rec.Move.Rollback.RecoveryInputs != nil { t.Fatalf("no release: %v", err) } if err := MoveRecordRecoveryInputs(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, rbTestID, first)(rec); !errors.Is(err, ErrMoveConflict) { t.Fatalf("another guard: %v", err) } rec.Move.State = MoveStuck if err := MoveRecordRecoveryInputs(g, rbTestID, first)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.RecoveryInputs != nil { t.Fatalf("stuck: %v", err) } } // A revert's probe records the inputs it renders before it renders them // (Plan 4 Task 6; J18, §7.7): MoveRecordRevertInputs writes the move's own // RecoveryInputs on a reverting record that has not reopened, and nothing // else. A later probe replaces them; they are never compared with // move.inputs. It is refused on any other state, after the reopen, without a // release and under another guard. func TestRevertInputsRecordedBeforeTheRender(t *testing.T) { g := cutTestGuard() now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) reverting := func() *Record { return &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReverting, CutPhase: CutStopping, Reason: RevertStopEvidenceBudget, Inputs: &MoveInputs{BundleETags: map[string]string{"default": "e1"}, Release: "v7"}}} } first := MoveInputs{BundleETags: map[string]string{"default": "e2"}, OverlayETag: "o2", Release: "v8"} rec := reverting() if err := MoveRecordRevertInputs(g, first)(rec); err != nil { t.Fatal(err) } m := rec.Move if m.RecoveryInputs == nil || m.RecoveryInputs.BundleETags["default"] != "e2" || m.RecoveryInputs.OverlayETag != "o2" || m.RecoveryInputs.Release != "v8" { t.Fatalf("recorded %+v", m.RecoveryInputs) } if m.Inputs.BundleETags["default"] != "e1" || m.Inputs.Release != "v7" || m.State != MoveReverting || m.Reason != RevertStopEvidenceBudget || m.CutPhase != CutStopping { t.Fatalf("the record changed beyond the inputs: %+v", m) } first.BundleETags["default"] = "mutated" if m.RecoveryInputs.BundleETags["default"] != "e2" { t.Fatal("the record aliases the caller's map") } if err := MoveRecordRevertInputs(g, MoveInputs{BundleETags: map[string]string{"default": "e3"}, Release: "v8"})(rec); err != nil || m.RecoveryInputs.BundleETags["default"] != "e3" { t.Fatalf("a later probe: %v %+v", err, m.RecoveryInputs) } for _, state := range []string{MoveCutting, MoveLanding, MoveRollingBack, MoveStuck, MoveCommitted, MoveAborting} { rec := reverting() rec.Move.State = state if err := MoveRecordRevertInputs(g, first)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.RecoveryInputs != nil { t.Fatalf("from %s: %v", state, err) } } rec = reverting() rec.Move.ReopenedAt = &now if err := MoveRecordRevertInputs(g, first)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.RecoveryInputs != nil { t.Fatalf("after the reopen: %v", err) } rec = reverting() if err := MoveRecordRevertInputs(g, MoveInputs{BundleETags: map[string]string{"default": "e2"}})(rec); err == nil || rec.Move.RecoveryInputs != nil { t.Fatalf("no release: %v", err) } if err := MoveRecordRevertInputs(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, first)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.RecoveryInputs != nil { t.Fatalf("another guard: %v", err) } if raw, _ := json.Marshal(MoveState{}); strings.Contains(string(raw), "recovery_inputs") { t.Fatalf("recovery_inputs is not omitempty: %s", raw) } if raw, _ := json.Marshal(MoveState{RecoveryInputs: &MoveInputs{Release: "v8"}}); !strings.Contains(string(raw), `"recovery_inputs":{`) { t.Fatalf("recovery_inputs missing: %s", raw) } } // The legacy abort completion is the legacy record's only (Plan 5 Task 1): // it refuses a per-op move, which completes through MoveGuardedAbortComplete // once the target region has parked its lane. A legacy aborting record still // completes, unchanged. func TestLegacyAbortCompleteRefusesAPerOpMove(t *testing.T) { g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)} rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveAborting}} if err := MoveAbortComplete()(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil { t.Fatalf("legacy completion of a per-op abort: %v %+v", err, rec.Move) } if err := MoveGuardedAbortComplete(g, time.Now())(rec); err != nil || rec.Move != nil { t.Fatalf("guarded completion: %v %+v", err, rec.Move) } legacy := &Record{Move: &MoveState{To: "uk", State: MoveAborting}} if err := MoveAbortComplete()(legacy); err != nil || legacy.Move != nil { t.Fatalf("legacy completion of a legacy abort: %v %+v", err, legacy.Move) } }