package host import ( "context" "encoding/json" "errors" "fmt" "os/exec" "reflect" "strings" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // plan10J2AdapterInputs are what the source would render now. func plan10J2AdapterInputs() provision.MoveInputs { return provision.MoveInputs{BundleETags: map[string]string{"default": "b1"}, OverlayETag: "o1", Release: "v7", EnvReport: "env"} } // plan10J2AdapterRecord is the rolling_back record of the recovery fixture's // claim: rollback plan10J2Rollback with the source at probing. func plan10J2AdapterRecord(incarnation string, dirty bool) provision.Record { return provision.Record{Account: "bab", App: "jam", Host: recoveryHost, Move: &provision.MoveState{ID: "m1", Incarnation: incarnation, From: "ie", To: "uk", State: provision.MoveRollingBack, Rollback: &provision.MoveRollback{ID: plan10J2Rollback, Dirty: dirty, SourcePhase: provision.RbSourceProbing}}} } // plan10J2AdapterRequest is the request movesrecover.go's probing builds, // recording into rec through the real provision transition. func plan10J2AdapterRequest(rec *provision.Record, srcGen string, events *[]string) recoveryProbeRequest { g := provision.MoveGuard{ID: rec.Move.ID, Incarnation: rec.Move.Incarnation} req := recoveryProbeRequest{RollbackID: rec.Move.Rollback.ID, Clean: !rec.Move.Rollback.Dirty, Attempts: recoveryProbeAttempts, SourceGeneration: srcGen, Record: func(_ context.Context, in provision.MoveInputs) error { *events = append(*events, "record") return provision.MoveRecordRecoveryInputs(g, rec.Move.Rollback.ID, in)(rec) }} if rec.Move.Rollback.Dirty { req.Coverage, req.Generation = recoveryCoverageSealed, rec.Move.Rollback.ID } else { req.Coverage = recoveryCoverageClean } return req } // plan10J2AdapterClaim puts the fixture's claim at restoring/probing with // box-1 executing and returns the incarnation. func plan10J2AdapterClaim(t *testing.T, f *recoveryFixture, phase, srcGen string) string { t.Helper() o, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error { c.Op = plan10J2Op(hostclaim.KindRestoring, phase) c.Op.Journal = map[string]string{hostclaim.JournalSrcGen: srcGen} return nil }) if err != nil { t.Fatal(err) } return o.Claim.Holder.Incarnation } // The adapter records the recovery inputs at probing before anything is // rendered, then asks the activator for a probe-shaped wake carrying exactly // those inputs. That wake runs the real Group D path: it is admitted at // probing as a recovery probe, and Prepare binds the operation attempts and // probe readiness to the admitted session. The held render (Materialize) // follows the record. func TestPlan10J2AdapterRecordsInputsBeforeTheGroupDProbe(t *testing.T) { for _, tc := range []struct{ dirty, revert bool }{{}, {dirty: true}, {revert: true}} { dirty := tc.dirty t.Run(fmt.Sprintf("dirty=%v/revert=%v", dirty, tc.revert), func(t *testing.T) { f := newRecoveryFixture(t) inc := plan10J2AdapterClaim(t, f, hostclaim.PhaseProbing, "src-gen") c := f.controller("boot-1", "controller-1", 4101, "start-1") ops := &plan9D2Attempts{} c.wake.Inputs, c.wake.Continue, c.wake.Operations = plan9D2Inputs{recoveryInputs{f}}, hostclaim.SessionContinues, ops c.wake.ProbeReady = func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil } rec := plan10J2AdapterRecord(inc, dirty) var events []string req := plan10J2AdapterRequest(&rec, "src-gen", &events) kind := hostclaim.KindRestoring if tc.revert { kind = hostclaim.KindReverting o, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error { c.Op = plan10J2Op(hostclaim.KindReverting, hostclaim.PhaseProbing) c.Op.Journal = map[string]string{hostclaim.JournalRevPhase: hostclaim.PhaseProbing} return nil }) if err != nil { t.Fatal(err) } rec.Move.State, rec.Move.Rollback = provision.MoveReverting, nil req = recoveryProbeRequest{Clean: true, SourceGeneration: o.Claim.Holder.Generation, Attempts: recoveryProbeAttempts, Record: func(_ context.Context, in provision.MoveInputs) error { events = append(events, "record") return provision.MoveRecordRevertInputs(provision.MoveGuard{ID: rec.Move.ID, Incarnation: inc}, in)(&rec) }} } recorded := func() *provision.MoveInputs { if tc.revert { return rec.Move.RecoveryInputs } return rec.Move.Rollback.RecoveryInputs } f.materialize = func(context.Context, hostclaim.Observed) error { events = append(events, "render") if recorded() == nil { t.Error("rendered before the inputs were recorded") } return nil } var policies []activator.StartPolicy wake := func(ctx context.Context, rt registry.Route, p activator.StartPolicy) error { events = append(events, "wake") policies = append(policies, p) if recorded() == nil || p.Inputs == nil || !reflect.DeepEqual(*p.Inputs, *recorded()) { t.Errorf("the wake does not carry the recorded inputs: %+v / %+v", p.Inputs, recorded()) } if err := c.wake.admitsOperation(f.read(), p, "box-1"); err != nil { return err } // Prerequisites' S3 lease stands in, as plan10J2Recovery does. live, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) done := make(chan struct{}) go func() { <-live.Done(); close(done) }() o := f.read() c.wake.mu.Lock() c.wake.pending[rt.Host] = &claimWakeAttempt{observed: o, held: recoveryHeld{}, ctx: live, cancel: cancel, done: done} c.wake.mu.Unlock() admitted, err := c.runtime.Admit(ctx, rt, o, p) if err != nil { return err } events = append(events, "admit") s, err := c.runtime.Prepare(ctx, rt, admitted, p) if err != nil { return err } if _, err := s.OpenProbe(ctx, time.Now()); err != nil { return err } if err := s.Reserve(ctx, 0); err != nil { return err } return s.Materialize(ctx) } a := recoveryProbeAdapter{Box: "box-1", Claims: true, Wake: wake, Route: func(host string) (*registry.Route, error) { return ®istry.Route{Host: host}, nil }, Current: func(context.Context, release.Store, provision.Record) (provision.MoveInputs, error) { return plan10J2AdapterInputs(), nil }} out, err := a.probe(t.Context(), f.claims, rec, req) if err != nil || out != recoveryProbeAnswered { t.Fatalf("probe: %v %v", out, err) } if got := strings.Join(events, ","); got != "record,wake,admit,render" { t.Fatalf("order: %s", got) } if len(policies) != 1 || !policies[0].Probe || policies[0].MaxStarts != recoveryProbeStartsPerWake || policies[0].Window != 30*time.Second || policies[0].RecoveryFence != moveFenceGeneration(rec.Move) { t.Fatalf("policy: %+v", policies) } if got := strings.Join(ops.calls, ","); got != "open,reserve-0" { t.Fatalf("attempt journal calls: %s", got) } for _, s := range ops.starts { if !s.Policy.Probe || s.Admitted.Claim.Op.Kind != kind || s.Admitted.Claim.Op.Phase != hostclaim.PhaseProbing || !reflect.DeepEqual(*s.Policy.Inputs, plan10J2AdapterInputs()) { t.Fatalf("attempt bound to %+v", s) } } }) } } // A failed record write, inputs naming no release, or a claim that is not // this rollback's probe at probing by this box stops the attempt before any // render or wake. func TestPlan10J2AdapterStopsBeforeTheWake(t *testing.T) { t.Parallel() type tc struct { claimPhase, exec, srcGen string in provision.MoveInputs recordErr error mutateReq func(*recoveryProbeRequest) mutateRec func(*provision.Record) want error } recordFailed := errors.New("the record write failed") for name, c := range map[string]tc{ "record write fails": {recordErr: recordFailed, want: recordFailed}, "no release": {in: provision.MoveInputs{OverlayETag: "o1"}, want: errRecoveryInputsNoRelease}, "claim at claimed": {claimPhase: hostclaim.PhaseClaimed, want: hostbox.ErrChanged}, "claim at probed": {claimPhase: hostclaim.PhaseProbed, want: hostbox.ErrChanged}, "other executor": {exec: "box-2", want: hostbox.ErrChanged}, "other rollback": {mutateReq: func(r *recoveryProbeRequest) { r.RollbackID = "rb_zzzzzzzzzzzzzzzzzzzzzzzzzz" }, want: errRecoveryProbeRequest}, "record not probing": {mutateRec: func(r *provision.Record) { r.Move.Rollback.SourcePhase = provision.RbSourceProbed }, want: errRecoveryProbeRequest}, "budget not nine": {mutateReq: func(r *recoveryProbeRequest) { r.Attempts = 3 }, want: errRecoveryProbeRequest}, "clean as sealed": {mutateReq: func(r *recoveryProbeRequest) { r.Coverage = recoveryCoverageSealed }, want: errRecoveryProbeRequest}, "clean with a gen": {mutateReq: func(r *recoveryProbeRequest) { r.Generation = plan10J2Rollback }, want: errRecoveryProbeRequest}, "src_gen differs": {srcGen: "other-gen", want: hostbox.ErrChanged}, "no record hook": {mutateReq: func(r *recoveryProbeRequest) { r.Record = nil }, want: errRecoveryProbeRequest}, } { t.Run(name, func(t *testing.T) { f := newRecoveryFixture(t) phase := c.claimPhase if phase == "" { phase = hostclaim.PhaseProbing } srcGen := c.srcGen if srcGen == "" { srcGen = "src-gen" } inc := plan10J2AdapterClaim(t, f, phase, srcGen) if c.exec != "" { if _, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(cl *hostclaim.Claim) error { cl.Op.Exec = c.exec; return nil }); err != nil { t.Fatal(err) } } rec := plan10J2AdapterRecord(inc, false) var events []string req := plan10J2AdapterRequest(&rec, "src-gen", &events) if c.recordErr != nil { req.Record = func(context.Context, provision.MoveInputs) error { events = append(events, "record") return c.recordErr } } if c.mutateReq != nil { c.mutateReq(&req) } if c.mutateRec != nil { c.mutateRec(&rec) } in := c.in if in.Release == "" && in.OverlayETag == "" { in = plan10J2AdapterInputs() } a := recoveryProbeAdapter{Box: "box-1", Claims: true, Wake: func(context.Context, registry.Route, activator.StartPolicy) error { events = append(events, "wake") return nil }, Route: func(host string) (*registry.Route, error) { return ®istry.Route{Host: host}, nil }, Current: func(context.Context, release.Store, provision.Record) (provision.MoveInputs, error) { return in, nil }} out, err := a.probe(t.Context(), f.claims, rec, req) if !errors.Is(err, c.want) || out != recoveryProbeWaiting { t.Fatalf("got %v %v, want %v", out, err, c.want) } for _, e := range events { if e == "wake" { t.Fatalf("woke: %v", events) } } if c.recordErr == nil && len(events) != 0 { t.Fatalf("recorded: %v", events) } }) } } // Without the host claim or the activator's recovery probe entry the // adapter is a typed dependency refusal: nothing is read, recorded or woken. func TestPlan10J2AdapterRefusesWithoutItsDependencies(t *testing.T) { route := func(host string) (*registry.Route, error) { return ®istry.Route{Host: host}, nil } current := func(context.Context, release.Store, provision.Record) (provision.MoveInputs, error) { return plan10J2AdapterInputs(), nil } wake := func(context.Context, registry.Route, activator.StartPolicy) error { return nil } for name, a := range map[string]recoveryProbeAdapter{ "runtime unavailable": {Box: "box-1", Claims: true, Wake: wake, Route: route, Current: current, Ready: func() bool { return false }}, "claim off": {Box: "box-1", Claims: false, Wake: wake, Route: route, Current: current}, "no wake entry": {Box: "box-1", Claims: true, Route: route, Current: current}, "no route": {Box: "box-1", Claims: true, Wake: wake, Current: current}, "no inputs": {Box: "box-1", Claims: true, Wake: wake, Route: route}, } { t.Run(name, func(t *testing.T) { rec := plan10J2AdapterRecord("inc", false) var events []string out, err := a.probe(t.Context(), release.NewDirStore(t.TempDir()), rec, plan10J2AdapterRequest(&rec, "src-gen", &events)) var refusal *ClaimDependencyError if !errors.As(err, &refusal) || out != recoveryProbeWaiting || len(events) != 0 || rec.Move.Rollback.RecoveryInputs != nil { t.Fatalf("%v %v %v", out, err, events) } }) } } // The wake's outcome maps onto the recovery's: answered goes to probed, a // spent nine-attempt budget stops for an operator, a lapsed window or a // changed claim waits for the next pass, anything else is an error. func TestPlan10J2AdapterOutcomes(t *testing.T) { t.Parallel() other := errors.New("boom") for name, tc := range map[string]struct { wakeErr error out recoveryProbeOutcome err error }{ "answered": {nil, recoveryProbeAnswered, nil}, "exhausted": {fmt.Errorf("reserve: %w", ErrRecoveryProbeExhausted), recoveryProbeExhausted, nil}, "window": {moves.ErrAttemptOver, recoveryProbeWaiting, nil}, "changed": {hostbox.ErrChanged, recoveryProbeWaiting, nil}, "other": {other, recoveryProbeWaiting, other}, } { t.Run(name, func(t *testing.T) { f := newRecoveryFixture(t) rec := plan10J2AdapterRecord(plan10J2AdapterClaim(t, f, hostclaim.PhaseProbing, "src-gen"), false) var events []string a := recoveryProbeAdapter{Box: "box-1", Claims: true, Wake: func(context.Context, registry.Route, activator.StartPolicy) error { return tc.wakeErr }, Route: func(host string) (*registry.Route, error) { return ®istry.Route{Host: host}, nil }, Current: func(context.Context, release.Store, provision.Record) (provision.MoveInputs, error) { return plan10J2AdapterInputs(), nil }} out, err := a.probe(t.Context(), f.claims, rec, plan10J2AdapterRequest(&rec, "src-gen", &events)) if out != tc.out || (tc.err == nil) != (err == nil) || (tc.err != nil && !errors.Is(err, tc.err)) { t.Fatalf("got %v %v", out, err) } }) } } // An unavailable wake entry keeps the recovery at probing without recording // inputs. The production binding itself is checked below with both flags. func TestPlan10J2MissingWakeKeepsRecoveryAtProbing(t *testing.T) { t.Parallel() cfg := moveCfg{Box: "ie-box", claimRecords: true, Route: func(string) (*registry.Route, error) { return nil, nil }} w := &movesWorker{mirrors: map[string]lease.MirrorStore{}} armRecoveryProbe(&cfg, w, nil, nil) if cfg.recoveryProbe == nil { t.Fatal("the production recovery probe hook is nil") } h := newRecoverHarness(t, false) h.released(t) h.sw = &movesWorker{mirrors: map[string]lease.MirrorStore{}} for i := 0; i < 2; i++ { c := h.sourceCfg(t) c.claimRecords = true armRecoveryProbe(&c, h.sw, nil, nil) h.sw.cfg = c if err := movesPass(context.Background(), h.s, h.sw); err != nil { t.Fatal(err) } } if rb := h.sourceAt(t, provision.RbSourceProbing); rb.RecoveryInputs != nil { t.Fatalf("recorded without a wake entry: %+v", rb.RecoveryInputs) } } // With a wake entry the same production adapter drives the real recovery // pass: each return records the inputs it renders at probing, before the // wake, and the recovery goes on to opened. // The production adapter accepts the claim's legacy encoding of a source // generation (whole-branch fixes, item 4): "" is a source whose sessions // predate admission records, which recoverRun.probing passes through // from the claim's journal unchanged and the owners normalise as @root. // A declared return and a clean return of such a source reach the wake; // a request naming another generation than the claim's still does not. func TestPlan10J2AdapterAcceptsALegacySourceGeneration(t *testing.T) { ctx := context.Background() for name, declared := range map[string]bool{"a clean return": false, "a declared return": true} { t.Run(name, func(t *testing.T) { s := release.NewDirStore(t.TempDir()) created := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) inc, err := hostclaim.Incarnation("bab", "app", p1Host, created) if err != nil { t.Fatal(err) } rec := provision.Record{Host: p1Host, Account: "bab", App: "app", Region: "ie", Move: &provision.MoveState{ID: p1ID, Incarnation: inc, From: "ie", To: "uk", State: provision.MoveRollingBack, Rollback: &provision.MoveRollback{ID: "rb_legacy", SourcePhase: provision.RbSourceProbing, TargetPhase: provision.RbTargetReleased}}} journal := map[string]string{hostclaim.JournalSrcGen: "", hostclaim.JournalSourceStamp: "15", hostclaim.JournalCutSession: "15"} if declared { lossKey, err := hostclaim.LossRecordKey("l1") if err != nil { t.Fatal(err) } rec.Move.Rollback.LossRef = lossKey journal["loss.l1"] = `{"kind":"move-target-lost","target":"uk-box"}` } c := &hostclaim.Claim{V: 2, Host: p1Host, Gen: 9, Holder: &hostclaim.Holder{Family: "instance", Key: "apps/bab/app/instances/" + p1Host + ".json", Account: "bab", App: "app", Incarnation: inc, CreatedAt: created, Location: "instances/" + p1Host, Stamp: 15, Session: 20, WriterBox: "uk-box", Admissions: []hostclaim.AdmissionRecord{{Session: 20, Box: "uk-box", Generation: p1ID, Region: "uk", PredecessorRegion: "ie", Mode: "cold", Coverage: "sealed"}}}, Op: hostclaim.Op{Kind: hostclaim.KindRestoring, ID: p1ID, Phase: hostclaim.PhaseProbing, Exec: "ie-box", Participants: []string{"ie-box", "uk-box"}, Intent: map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box", "rollback": "rb_legacy"}, Journal: journal}} if declared { c.LossRefs, c.Lost = []string{"l1"}, []string{"uk-box"} } key, _ := hostclaim.Key(p1Host) body, _ := json.Marshal(c) if _, err := s.Put(ctx, key, body, ""); err != nil { t.Fatal(err) } woke := 0 a := recoveryProbeAdapter{Box: "ie-box", Claims: true, Route: func(string) (*registry.Route, error) { return ®istry.Route{Host: p1Host, DBPath: "/data/x.db"}, nil }, Current: func(context.Context, release.Store, provision.Record) (provision.MoveInputs, error) { return provision.MoveInputs{Release: "v1"}, nil }, Wake: func(context.Context, registry.Route, activator.StartPolicy) error { woke++; return nil }} req := recoveryProbeRequest{RollbackID: "rb_legacy", Clean: true, Declared: declared, Attempts: recoveryProbeAttempts, SourceGeneration: "", Record: func(context.Context, provision.MoveInputs) error { return nil }} if !declared { req.Coverage = recoveryCoverageClean } if out, err := a.probe(ctx, s, rec, req); err != nil || out != recoveryProbeAnswered || woke != 1 { t.Fatalf("a legacy source's return: %v %v woke=%d", out, err, woke) } req.SourceGeneration = "other" if _, err := a.probe(ctx, s, rec, req); !errors.Is(err, hostbox.ErrChanged) || woke != 1 { t.Fatalf("another generation than the claim's: %v woke=%d", err, woke) } }) } } func TestPlan10J2AdapterDrivesTheRecoveryPass(t *testing.T) { t.Parallel() for _, dirty := range []bool{false, true} { t.Run(fmt.Sprintf("dirty=%v", dirty), func(t *testing.T) { h := newRecoverHarness(t, dirty) h.released(t) h.sw = &movesWorker{mirrors: map[string]lease.MirrorStore{}} var wakes []activator.StartPolicy wake := func(ctx context.Context, rt registry.Route, p activator.StartPolicy) error { rb := h.read(t, p1Host).Move.Rollback if rb.SourcePhase != provision.RbSourceProbing || rb.RecoveryInputs == nil || p.Inputs == nil || !reflect.DeepEqual(*rb.RecoveryInputs, *p.Inputs) { t.Errorf("woke before the record: %+v / %+v", rb, p.Inputs) } c := h.claim(t) if rt.Host != p1Host || c.Op.Kind != hostclaim.KindRestoring || c.Op.Phase != hostclaim.PhaseProbing || c.Op.Exec != "ie-box" { t.Errorf("woke %s with the claim at %+v", rt.Host, c.Op) } wakes = append(wakes, p) return nil } for i := 0; i < 3; i++ { c := h.sourceCfg(t) c.claimRecords = true armRecoveryProbe(&c, h.sw, wake, nil) a := c.recoveryProbe c.recoveryProbe = func(ctx context.Context, s release.Store, rec provision.Record, req recoveryProbeRequest) (recoveryProbeOutcome, error) { h.probes = append(h.probes, req) return a(ctx, s, rec, req) } h.sw.cfg = c h.sw.cfg.LaunchEnv = func(string) (map[string]string, map[string]bool, error) { return map[string]string{"A": "1"}, nil, nil } h.sw.cfg.InputVersions = func(context.Context, release.Store, provision.Record) (map[string]string, string, error) { in := h.renderedInputs() return in.BundleETags, in.OverlayETag, nil } if err := movesPass(context.Background(), h.s, h.sw); err != nil { t.Fatal(err) } } h.opened(t) if len(wakes) != 1 || !wakes[0].Probe || len(h.probes) != 1 { t.Fatalf("wakes %+v probes %d", wakes, len(h.probes)) } req := h.probes[0] if req.Clean == dirty || req.Attempts != recoveryProbeAttempts || (dirty && (req.Coverage != recoveryCoverageSealed || req.Generation == "")) || (!dirty && (req.Coverage != recoveryCoverageClean || req.SourceGeneration != h.claim(t).Op.Journal[hostclaim.JournalSrcGen])) { t.Fatalf("request %+v", req) } }) } } func TestPlan10J2ArmPhase1ConnectsRecoveryOnlyWithClaimFlag(t *testing.T) { for _, enabled := range []bool{false, true} { t.Run(fmt.Sprint(enabled), func(t *testing.T) { cfg := moveCfg{claimRecords: enabled} w := &movesWorker{mirrors: map[string]lease.MirrorStore{}} armPhase1(&cfg, w, activator.Config{}, t.TempDir(), nil, &activator.Service{}, nil, hostwrite.Config{Enabled: enabled}, func(string) string { return "" }) if (cfg.recoveryProbe != nil) != enabled { t.Fatal("production recovery hook does not follow claim flag") } }) } } func TestPlan10J2RevertRunUsesProductionAdapter(t *testing.T) { t.Parallel() h := newRevertHarness(t) h.fromStopped(t) h.arm(t) wakes := 0 a := recoveryProbeAdapter{Box: "ie-box", Claims: true, Route: func(string) (*registry.Route, error) { return &h.route, nil }, Current: func(context.Context, release.Store, provision.Record) (provision.MoveInputs, error) { return revertRendered(), nil }, Wake: func(ctx context.Context, rt registry.Route, p activator.StartPolicy) error { wakes++ m := h.move(t) if m.RecoveryInputs == nil || !reflect.DeepEqual(*m.RecoveryInputs, *p.Inputs) { t.Fatal("wake before revert inputs") } c := h.claim(t) if c.Op.Kind != hostclaim.KindReverting || c.Op.Phase != hostclaim.PhaseProbing || c.Op.Journal[hostclaim.JournalRevPhase] != hostclaim.PhaseProbing { t.Fatal("wrong claim") } return nil }} h.cfg.recoveryProbe = a.probe h.pass(t) if wakes != 1 { t.Fatalf("wakes=%d", wakes) } h.reopened(t) } func TestPlan10J2RevertMissingProviderWaitsWithoutInputsOrWake(t *testing.T) { t.Parallel() h := newRevertHarness(t) h.fromStopped(t) h.arm(t) calls := 0 a := recoveryProbeAdapter{Box: "ie-box", Claims: true, Ready: func() bool { return false }, Route: func(string) (*registry.Route, error) { return &h.route, nil }, Current: func(context.Context, release.Store, provision.Record) (provision.MoveInputs, error) { calls++ return revertRendered(), nil }, Wake: func(context.Context, registry.Route, activator.StartPolicy) error { calls++; return nil }} h.cfg.recoveryProbe = a.probe h.pass(t) before := h.move(t).UpdatedAt h.clock.Advance(16 * time.Minute) h.pass(t) m := h.move(t) if calls != 0 || m.RecoveryInputs != nil || m.State != provision.MoveReverting || !m.UpdatedAt.Equal(before) { t.Fatalf("unbound provider changed recovery: calls=%d move=%+v", calls, m) } } func TestPlan10J2NinthFailedStartSticksAndKeepsJournal(t *testing.T) { t.Parallel() h := newRevertHarness(t) h.fromStopped(t) h.arm(t) starts, ids := 0, 0 var last OperationStart provider := func() *movesProbeAttempts { return &movesProbeAttempts{Claims: h.s, DataDir: h.cfg.DataDir, Records: func(ctx context.Context, account, app, host string) (*provision.Record, error) { return provision.GetRecord(ctx, h.s, account, app, host) }, Admitted: func(context.Context, *provision.Record) (int, error) { return 9, nil }, NewID: func() string { ids++; return fmt.Sprintf("attempt-%d", ids) }, Now: time.Now} } a := recoveryProbeAdapter{Box: "ie-box", Claims: true, Ready: func() bool { return provider().RecoveryProbeAttemptsReady() }, Route: func(string) (*registry.Route, error) { return &h.route, nil }, Current: func(context.Context, release.Store, provision.Record) (provision.MoveInputs, error) { return revertRendered(), nil }, Wake: func(ctx context.Context, rt registry.Route, p activator.StartPolicy) error { o, err := hostclaim.Read(ctx, h.s, rt.Host) if err != nil { return err } holder := o.Claim.Holder last = OperationStart{Admitted: o, Policy: p, Owner: hostguard.FenceTarget{Host: rt.Host, Incarnation: holder.Incarnation, Stamp: holder.Stamp, Session: holder.Session, Box: "ie-box"}} ops := provider() for i := 0; i < 3; i++ { if err := ops.Reserve(ctx, last, starts); err != nil { return err } starts++ if err := ops.Failed(ctx, last, starts-1); err != nil { return err } } return moves.ErrAttemptOver }} h.cfg.recoveryProbe = a.probe for i := 0; i < 3; i++ { h.pass(t) } m := h.move(t) if starts != 9 || m.State != provision.MoveStuck || m.StuckReason != provision.StuckRecoveryProbeAttempts { t.Fatalf("starts=%d move=%+v", starts, m) } j, err := moves.ReadJournal(h.cfg.DataDir, p1Host, p1ID) if err != nil { t.Fatal(err) } if count, err := recoveryAttemptCount(j, recoveryAttemptKey(last, 0)); err != nil || count != 9 { t.Fatalf("journal count=%d err=%v", count, err) } } func TestPlan10J2RevertAdapterRejectsChangedBinding(t *testing.T) { t.Parallel() for _, tc := range []string{"reopened", "generation", "rev_phase", "kind", "executor"} { t.Run(tc, func(t *testing.T) { f := newRecoveryFixture(t) o, err := hostclaim.Update(t.Context(), f.claims, f.read(), func(c *hostclaim.Claim) error { c.Op = plan10J2Op(hostclaim.KindReverting, hostclaim.PhaseProbing) c.Op.Journal = map[string]string{hostclaim.JournalRevPhase: hostclaim.PhaseProbing} switch tc { case "rev_phase": c.Op.Journal[hostclaim.JournalRevPhase] = hostclaim.PhaseProbed case "kind": c.Op.Kind = hostclaim.KindRestoring case "executor": c.Op.Exec = "box-2" } return nil }) if err != nil { t.Fatal(err) } rec := provision.Record{Host: recoveryHost, Move: &provision.MoveState{ID: o.Claim.Op.ID, Incarnation: o.Claim.Holder.Incarnation, State: provision.MoveReverting}} req := recoveryProbeRequest{Clean: true, SourceGeneration: o.Claim.Holder.Generation, Attempts: recoveryProbeAttempts, Record: func(context.Context, provision.MoveInputs) error { t.Fatal("recorded invalid revert"); return nil }} if tc == "generation" { req.SourceGeneration = "other" } if tc == "reopened" { now := time.Now() rec.Move.ReopenedAt = &now } a := recoveryProbeAdapter{Claims: true, Box: "box-1", Route: func(string) (*registry.Route, error) { t.Fatal("looked up invalid revert"); return nil, nil }, Current: func(context.Context, release.Store, provision.Record) (provision.MoveInputs, error) { t.Fatal("selected invalid revert") return provision.MoveInputs{}, nil }, Wake: func(context.Context, registry.Route, activator.StartPolicy) error { t.Fatal("woke invalid revert") return nil }} if _, err := a.probe(t.Context(), f.claims, rec, req); err == nil { t.Fatal("invalid revert accepted") } }) } }