package host import ( "context" "errors" "fmt" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) // The moves retirement evidence provider (Plan 5 Task 5, rulings K10, M7, // M8). The fake bucket's clock starts at 2026-10-05T00:00:00Z and ticks a // second per write, so every object a fixture writes is older than evAt. var ( evCreated = time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) evAt = time.Date(2026, 10, 5, 1, 0, 0, 0, time.UTC) // LastMove.At, the close evNow = time.Date(2026, 10, 5, 2, 0, 0, 0, time.UTC) // the sweep's clock ) const ( evHost = "ev.example" evAcct = "acct" evApp = "app" evMoveID = "mv_retired" ) type evFixture struct { inc, location string claims release.Store bucket *leasetest.VersionedBucket rec *provision.Record reads int } // newEvFixture is a serving claim released by evMoveID, whose record closed // that move at evAt and carries no live move. func newEvFixture(t *testing.T) *evFixture { t.Helper() inc, err := hostclaim.Incarnation(evAcct, evApp, evHost, evCreated) if err != nil { t.Fatal(err) } f := &evFixture{inc: inc, location: "instances/" + evHost + "/" + inc, claims: release.NewDirStore(t.TempDir()), bucket: leasetest.NewVersionedBucket()} f.bucket.AddBucket("data") f.rec = &provision.Record{Schema: 1, Host: evHost, App: evApp, Account: evAcct, CreatedAt: evCreated, LastMove: &provision.MoveOutcome{ID: evMoveID, Outcome: "done", At: evAt}} f.setClaim(t, func(c *hostclaim.Claim) { c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey(evAcct, evApp, evHost), Account: evAcct, App: evApp, Incarnation: inc, CreatedAt: evCreated, Location: f.location, Stamp: 2, Session: 2, WriterBox: "box", Generation: evMoveID, Admissions: evAdmissions(evMoveID)} c.Op = servingReleasedBy(evMoveID) c.Touched = hostclaim.Touched{Complete: true, Regions: []string{"ie"}} }) return f } // evAdmissions is the holder's admission history: an origin generation // "old", then the move's prepared landing, which admits the generation // named after the move (movesprobeowners.go), then any later generations, // each the next one's predecessor. The last is the holder's current. func evAdmissions(gens ...string) []hostclaim.AdmissionRecord { out := []hostclaim.AdmissionRecord{{Session: 1, Box: "box", Generation: "old", Mode: "cold", Region: "ie"}} for i, g := range gens { out = append(out, hostclaim.AdmissionRecord{Session: uint64(i + 2), Box: "box", Generation: g, Predecessor: out[i].Generation, Mode: "cold", Region: "ie", PredecessorRegion: "ie"}) } return out } // genKey is an object of generation gen at the fixture's live location. func (f *evFixture) genKey(gen string) string { return f.location + "/g-" + gen + "/0000/db" } // servingReleasedBy is the serving op hostclaim.ReleaseMove leaves. func servingReleasedBy(id string) hostclaim.Op { return hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box", Journal: map[string]string{"released_move": id}} } func (f *evFixture) setClaim(t *testing.T, mut func(*hostclaim.Claim)) { t.Helper() obs, err := hostclaim.Read(t.Context(), f.claims, evHost) if err != nil { t.Fatal(err) } if _, err := hostclaim.Update(t.Context(), f.claims, obs, func(c *hostclaim.Claim) error { mut(c); return nil }); err != nil { t.Fatal(err) } } func (f *evFixture) observe(t *testing.T) hostclaim.Observed { t.Helper() obs, err := hostclaim.Read(t.Context(), f.claims, evHost) if err != nil { t.Fatal(err) } return obs } func (f *evFixture) census(t *testing.T) []lease.ObjectVersion { t.Helper() var out []lease.ObjectVersion for _, p := range []string{"instances/" + evHost + "/", "moves/" + evHost + "/"} { vs, err := f.bucket.ListVersions(t.Context(), "data", p) if err != nil { t.Fatal(err) } out = append(out, vs...) } return out } func (f *evFixture) provider() movesRetirementEvidence { return movesRetirementEvidence{ Records: func(_ context.Context, acct, app, host string) (*provision.Record, error) { f.reads++ if acct != evAcct || app != evApp || host != evHost { return nil, fmt.Errorf("read %s/%s/%s", acct, app, host) } if f.rec == nil { return nil, errors.New("no record") } cp := *f.rec return &cp, nil }, Now: func() time.Time { return evNow }, } } // lane writes one object of a move's staging lane. func (f *evFixture) lane(op, name string) { f.bucket.Put("data", "moves/"+evHost+"/"+op+"/"+name, []byte(op)) } func wantRefusal(t *testing.T, err error, anomaly bool) { t.Helper() var r *MovesSweepRefusal if !errors.As(err, &r) { t.Fatalf("err = %v, want a *MovesSweepRefusal", err) } if r.Host != evHost || r.Reason == "" || r.Anomaly != anomaly { t.Fatalf("refusal = %+v, want host %s, a reason and anomaly %v", r, evHost, anomaly) } } // bothRefuse asserts the two methods refuse alike: VerifyRetirement's // label is the proof's, so neither may answer where the other refuses. func bothRefuse(t *testing.T, f *evFixture, anomaly bool) { t.Helper() obs, census := f.observe(t), f.census(t) if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err == nil { t.Fatal("VerifyMovesSweep answered") } else { wantRefusal(t, err, anomaly) } inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census) if err == nil || inc != "" || !at.IsZero() { t.Fatalf("VerifyRetirement = %q, %v, %v; want a refusal and no evidence", inc, at, err) } wantRefusal(t, err, anomaly) } // TestMovesSweepProofRetiredAtIsTheClose (M8): the proof binds the exact // claim observation, names LastMove's id, the holder's incarnation and the // close's time, and the engine's Check accepts it only for a listing that // started after that close. func TestMovesSweepProofRetiredAtIsTheClose(t *testing.T) { f := newEvFixture(t) f.lane(evMoveID, "move.json") f.lane(evMoveID, "replica/0000/db") f.bucket.Put("data", "moves/"+evHost+"/lock.json", []byte("lease")) obs := f.observe(t) proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, f.census(t)) if err != nil { t.Fatal(err) } want := park.MoveSweepProof{Host: evHost, ClaimETag: obs.ETag, MoveID: evMoveID, Incarnation: f.inc, RetiredAt: evAt} if proof != want { t.Fatalf("proof = %+v\nwant %+v", proof, want) } if err := proof.Check(obs, f.inc, evNow); err != nil { t.Fatalf("a listing after the close: %v", err) } for _, listing := range []time.Time{evAt, evAt.Add(-time.Second)} { if err := proof.Check(obs, f.inc, listing); !errors.Is(err, park.ErrUnsafe) { t.Fatalf("a listing at %v, not after the close %v: %v", listing, evAt, err) } } } // TestVerifyRetirementLabelsMovesResidueWithTheProofsIncarnation (M7): // Decide labels every moves/ candidate with VerifyRetirement's // answer, and Check requires the proof's incarnation to equal it, so the // two answers come from one source. The evidence time is the close's, never // zero and never the clock's. func TestVerifyRetirementLabelsMovesResidueWithTheProofsIncarnation(t *testing.T) { f := newEvFixture(t) f.lane(evMoveID, "move.json") f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) obs, census := f.observe(t), f.census(t) inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census) if err != nil { t.Fatal(err) } if inc != f.inc || !at.Equal(evAt) { t.Fatalf("VerifyRetirement = %q, %v; want %q, %v", inc, at, f.inc, evAt) } proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, census) if err != nil { t.Fatal(err) } if proof.Incarnation != inc { t.Fatalf("label %q, proof %q", inc, proof.Incarnation) } if err := proof.Check(obs, inc, evNow); err != nil { t.Fatal(err) } } // retirement runs the claim's retirement answer, the one the regional worker // binds to every instances/ candidate, over the real census with this // fixture's provider as the claim's moves evidence. func (f *evFixture) retirement(t *testing.T, evidence hostbox.RegionalEvidence) (string, time.Time, error) { t.Helper() p := productionClaimRegional{Now: func() time.Time { return evNow }, Parks: &claimMoveParkSources{Evidence: evidence}} return p.retirement(t.Context(), f.observe(t), f.census(t)) } // TestRetirementLeavesQualifiedResidueToTheCensus (Q4): residue under // instances/ outside the holder's live location is the claim's census // to place, not the moves provider's to refuse. The census labels a // qualified path by its own identity (ClaimRetirementCensus) and the claim's // view refuses unmapped bare history per location, so the provider answers // with the move's close and the moves/ lane sweep goes ahead. func TestRetirementLeavesQualifiedResidueToTheCensus(t *testing.T) { older := strings.Repeat("ab", 32) for name, key := range map[string]string{ "a legacy level object": "instances/" + evHost + "/0000/db", "a legacy generation object": "instances/" + evHost + "/g-old/0000/db", "an older incarnation": "instances/" + evHost + "/" + older + "/g-old/0000/db", } { t.Run(name, func(t *testing.T) { f := newEvFixture(t) f.lane(evMoveID, "move.json") f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) f.bucket.Put("data", key, []byte("residue")) inc, at, err := f.retirement(t, f.provider()) if err != nil || inc != f.inc || !at.Equal(evAt) { t.Fatalf("retirement = %q, %v, %v; want %q, %v and no refusal", inc, at, err, f.inc, evAt) } if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil { t.Fatalf("the moves proof does not depend on instances residue: %v", err) } }) } t.Run("the census labels a qualified path by its own identity", func(t *testing.T) { f := newEvFixture(t) f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) f.bucket.Put("data", "instances/"+evHost+"/"+older+"/g-old/0000/db", []byte("older")) labels, err := ClaimRetirementCensus(f.observe(t), f.census(t)) if err != nil || labels["instances/"+evHost+"/"+older] != older { t.Fatalf("labels = %v, %v", labels, err) } if _, _, err := f.retirement(t, f.provider()); err != nil { t.Fatal(err) } }) t.Run("an incarnation directory beside a bare holder", func(t *testing.T) { f := newEvFixture(t) f.bare(t) f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) f.bucket.Put("data", "instances/"+evHost+"/"+f.inc+"/g-old/0000/db", []byte("scoped")) inc, at, err := f.retirement(t, f.provider()) if err != nil || inc != f.inc || !at.Equal(evAt) { t.Fatalf("retirement = %q, %v, %v", inc, at, err) } }) } // TestRetirementStillRefusesAGenerationTheMoveDidNotRetire (Q4): the close // is the evidence time for every instances/ candidate the worker binds // it to, so at the live location it must be true. retiredGenerations stays: // a generation admitted from the move's own admission on, or written after // the close, is refused with *NotMovesEvidence. A generation the move did // retire passes beside it. func TestRetirementStillRefusesAGenerationTheMoveDidNotRetire(t *testing.T) { refused := func(t *testing.T, f *evFixture, gen string) { t.Helper() inc, at, err := f.retirement(t, f.provider()) var n *NotMovesEvidence if !errors.As(err, &n) || n.Generation != gen || inc != "" || !at.IsZero() { t.Fatalf("retirement = %q, %v, %v; want *NotMovesEvidence for %q and no evidence", inc, at, err, gen) } if !errors.Is(err, ErrNotMovesEvidence) { t.Fatalf("err does not unwrap to ErrNotMovesEvidence: %v", err) } } t.Run("a generation the move retired passes", func(t *testing.T) { f := newEvFixture(t) f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) f.bucket.Put("data", f.genKey("old"), []byte("superseded by the move")) inc, at, err := f.retirement(t, f.provider()) if err != nil || inc != f.inc || !at.Equal(evAt) { t.Fatalf("retirement = %q, %v, %v", inc, at, err) } }) t.Run("a generation admitted after the move", func(t *testing.T) { f := newEvFixture(t) f.setClaim(t, func(c *hostclaim.Claim) { c.Holder.Admissions = evAdmissions(evMoveID, "next") c.Holder.Generation = "next" }) f.bucket.Put("data", f.genKey("next"), []byte("live")) f.bucket.Put("data", f.genKey(evMoveID), []byte("superseded after the move")) refused(t, f, evMoveID) }) t.Run("an object written after the close", func(t *testing.T) { f := newEvFixture(t) f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) f.bucket.Put("data", f.genKey("old"), []byte("late")) f.rec.LastMove.At = time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC) refused(t, f, "old") }) t.Run("a claim that holds no admission of the move", func(t *testing.T) { f := newEvFixture(t) f.setClaim(t, func(c *hostclaim.Claim) { c.Holder.Admissions = evAdmissions()[:1]; c.Holder.Generation = "old" }) f.bucket.Put("data", f.genKey("old"), []byte("live")) f.bucket.Put("data", f.genKey("other"), []byte("no admission to date it by")) refused(t, f, "other") }) } // TestJoiningClaimRefusesQuietly (Q4): a joining claim (a parking sweep with // no holder) and a free one name no record, so the moves provider has no // close to prove from. It refuses with a *MovesSweepRefusal that is not an // anomaly, never a dependency, and reads no record: the agent logs it once an // hour and pages nobody. The claim's own free-host proof answers a free claim // before it gets here (claimFreeMoveProof). func TestJoiningClaimRefusesQuietly(t *testing.T) { f := newEvFixture(t) f.setClaim(t, func(c *hostclaim.Claim) { c.Holder, c.Touched = nil, hostclaim.Touched{} c.Op = hostclaim.Op{Kind: hostclaim.KindParking, ID: "op", Intent: map[string]string{"park_sweep": "v1"}} c.Obligations = []hostclaim.Obligation{{ID: "chore", Op: "op", Kind: "park-replica", Region: "ie", Owner: "region:ie", Retirement: "ret", Park: &hostclaim.ParkScope{Location: f.location, Incarnation: f.inc, EvidenceAt: evAt, RequestLimit: hostbox.RequestLimit}}} }) f.lane(evMoveID, "move.json") var pages pagesSeen l := &refusalLog{now: evNow} r := newMovesRefusals("ie-box", pages.hook(), l.logf, l.clock) for name, evidence := range map[string]hostbox.RegionalEvidence{"the provider": f.provider(), "the quiet wrapper": r.evidence(f.provider())} { t.Run(name, func(t *testing.T) { inc, at, err := f.retirement(t, evidence) var dep *ClaimDependencyError if errors.As(err, &dep) { t.Fatalf("a dependency, not a refusal: %v", err) } wantRefusal(t, err, false) if inc != "" || !at.IsZero() { t.Fatalf("evidence %q, %v with a refusal", inc, at) } }) } if f.reads != 0 { t.Fatalf("read %d records with no holder to name one", f.reads) } _, _, err := f.retirement(t, f.provider()) r.report(err) r.report(err) if got := l.take(); len(got) != 1 { t.Fatalf("logged %q; want one line within the hour", got) } if len(pages) != 0 { t.Fatalf("a joining claim paged: %+v", pages) } } // bare puts the holder at the activator's own lease location, // instances/: the only holder a move cuts (ruling T5-lease, // movescut.go writerLease), so every moved host's holder. func (f *evFixture) bare(t *testing.T) { t.Helper() f.location = "instances/" + evHost f.setClaim(t, func(c *hostclaim.Claim) { c.Holder.Location = f.location }) } // TestMovedHostAtABareLocationIsSwept (controller finding, Plan 12 review): // a moved host's holder is at the bare location, and its objects there are // its live location's, not residue outside it. The provider answers, and // the worker queues the moves/ lane chore. func TestMovedHostAtABareLocationIsSwept(t *testing.T) { f := newEvFixture(t) f.bare(t) f.lane(evMoveID, "move.json") f.lane(evMoveID, "replica/0000/db") f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) obs, census := f.observe(t), f.census(t) inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census) if err != nil || inc != f.inc || !at.Equal(evAt) { t.Fatalf("VerifyRetirement = %q, %v, %v; want %q, %v", inc, at, err, f.inc, evAt) } proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, census) if err != nil || proof.Incarnation != f.inc || !proof.RetiredAt.Equal(evAt) { t.Fatalf("proof = %+v, %v", proof, err) } if err := f.worker(t, evNow).Run(t.Context()); !errors.Is(err, park.ErrPending) { t.Fatalf("the lane chore waits for its horizon: %v", err) } moves, ok := f.chores(t)["moves/"+evHost] if !ok || moves.Incarnation != f.inc || !moves.EvidenceAt.Equal(evNow) { t.Fatalf("chores = %+v; want the moves/ chore at the listing's end", f.chores(t)) } } // TestVerifyRetirementVouchesOnlyForWhatTheMoveRetired (fix round 1, // ruling 1): the close is evidence only for what the move retired. At the // live location that is a generation superseded at or before the move's // own admission, every version of it older than the close. A generation // admitted from the move's admission on was written past the close, so the // provider has no evidence for it and says so with a typed error; the // moves lane proof is unaffected. func TestVerifyRetirementVouchesOnlyForWhatTheMoveRetired(t *testing.T) { answers := func(t *testing.T, f *evFixture) { t.Helper() inc, at, err := f.provider().VerifyRetirement(t.Context(), f.observe(t), f.census(t)) if err != nil || inc != f.inc || !at.Equal(evAt) { t.Fatalf("VerifyRetirement = %q, %v, %v; want %q, %v", inc, at, err, f.inc, evAt) } } notEvidence := func(t *testing.T, f *evFixture, gen string) { t.Helper() obs, census := f.observe(t), f.census(t) inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census) var ne *NotMovesEvidence if !errors.As(err, &ne) || inc != "" || !at.IsZero() { t.Fatalf("VerifyRetirement = %q, %v, %v; want a *NotMovesEvidence and no evidence", inc, at, err) } if ne.Host != evHost || ne.Generation != gen || ne.Reason == "" { t.Fatalf("refusal = %+v, want generation %q", ne, gen) } if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err != nil { t.Fatalf("the lane proof does not depend on the generations: %v", err) } } t.Run("the current generation and the marker", func(t *testing.T) { f := newEvFixture(t) f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) f.bucket.Put("data", f.location+"/generation.json", []byte("marker")) answers(t, f) }) t.Run("a generation the move's admission superseded", func(t *testing.T) { f := newEvFixture(t) f.bucket.Put("data", f.genKey("old"), []byte("old")) f.bucket.Put("data", f.location+"/files/g-old/a.txt", []byte("old")) answers(t, f) }) t.Run("the move's generation superseded after the close", func(t *testing.T) { f := newEvFixture(t) f.setClaim(t, func(c *hostclaim.Claim) { c.Holder.Admissions, c.Holder.Generation, c.Holder.Session, c.Holder.Stamp = evAdmissions(evMoveID, "later"), "later", 3, 3 }) f.bucket.Put("data", f.genKey(evMoveID), []byte("superseded after the close")) f.bucket.Put("data", f.genKey("later"), []byte("live")) notEvidence(t, f, evMoveID) }) t.Run("a later generation superseded after the close", func(t *testing.T) { f := newEvFixture(t) f.setClaim(t, func(c *hostclaim.Claim) { c.Holder.Admissions, c.Holder.Generation, c.Holder.Session, c.Holder.Stamp = evAdmissions(evMoveID, "later", "latest"), "latest", 4, 4 }) f.bucket.Put("data", f.genKey("later"), []byte("superseded after the close")) notEvidence(t, f, "later") }) t.Run("no admission of the move in the claim", func(t *testing.T) { f := newEvFixture(t) f.setClaim(t, func(c *hostclaim.Claim) { c.Holder.Admissions, c.Holder.Generation = evAdmissions("fresh"), "fresh" }) f.bucket.Put("data", f.genKey("old"), []byte("old")) notEvidence(t, f, "old") }) t.Run("a retired generation written after the close", func(t *testing.T) { f := newEvFixture(t) f.rec.LastMove.At = time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC) f.bucket.Put("data", f.genKey("old"), []byte("late")) notEvidence(t, f, "old") }) t.Run("an object no generation owns", func(t *testing.T) { f := newEvFixture(t) f.bucket.Put("data", f.location+"/stray.bin", []byte("?")) notEvidence(t, f, "") }) } // TestMovesSweepProofRefusesALiveMove (K10, M8): no proof while a move op // is on the claim, while the record carries a live move, or while the // claim was released by a move the record has not closed. func TestMovesSweepProofRefusesALiveMove(t *testing.T) { t.Run("the record carries a live move", func(t *testing.T) { f := newEvFixture(t) f.rec.Move = &provision.MoveState{ID: "mv_next", Incarnation: f.inc, To: "uk", State: provision.MovePreparing} f.lane("mv_next", "move.json") bothRefuse(t, f, false) }) for _, op := range []hostclaim.Op{ {Kind: hostclaim.KindMoving, ID: "mv_next", Phase: hostclaim.PhaseMoving, Exec: "box"}, {Kind: hostclaim.KindLanding, ID: evMoveID, Phase: hostclaim.PhaseCommitted, Exec: "box"}, {Kind: hostclaim.KindRestoring, ID: evMoveID, Phase: hostclaim.PhaseOpened, Exec: "box"}, {Kind: hostclaim.KindReverting, ID: evMoveID, Phase: hostclaim.PhaseOpened, Exec: "box"}, } { // The last three are the record-first close's window (M4): the // record closed the move, the claim has not released it yet. t.Run("the claim holds "+op.Kind, func(t *testing.T) { f := newEvFixture(t) f.setClaim(t, func(c *hostclaim.Claim) { c.Op = op }) bothRefuse(t, f, false) }) } t.Run("the record names no closed move", func(t *testing.T) { f := newEvFixture(t) f.rec.LastMove = nil bothRefuse(t, f, false) }) } // TestMovesProofSurvivesAMetadataPlacement (fix round 1, ruling 2): the // durable proof is the record's LastMove, bound to the holder's // incarnation, with no move op on the claim. A later operation that // replaces the serving op (here CompleteMetadataPlacement, which drops the // released_move journal) leaves cleanup running, and so does a serving op // released by a move whose record never carried it. func TestMovesProofSurvivesAMetadataPlacement(t *testing.T) { t.Run("a metadata placement", func(t *testing.T) { f := newEvFixture(t) f.lane(evMoveID, "move.json") f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) f.setClaim(t, func(c *hostclaim.Claim) { if err := hostops.BeginPlacement("pl_1", "op@example", "meta", evNow, []string{"ie"}, []string{"box"})(c); err != nil { t.Fatal(err) } c.Op.Intent["app_id"] = "app_1" if err := hostops.CompleteMetadataPlacement("pl_1", "app_1")(c); err != nil { t.Fatal(err) } }) obs := f.observe(t) if obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.Journal["released_move"] != "" { t.Fatalf("the placement left op %+v", obs.Claim.Op) } census := f.census(t) proof, err := f.provider().VerifyMovesSweep(t.Context(), obs, census) if err != nil || proof.MoveID != evMoveID || !proof.RetiredAt.Equal(evAt) { t.Fatalf("proof = %+v, %v", proof, err) } if inc, at, err := f.provider().VerifyRetirement(t.Context(), obs, census); err != nil || inc != f.inc || !at.Equal(evAt) { t.Fatalf("VerifyRetirement = %q, %v, %v", inc, at, err) } if err := f.worker(t, evNow).Run(t.Context()); !errors.Is(err, park.ErrPending) { t.Fatalf("the lane chore waits for its horizon: %v", err) } if _, ok := f.chores(t)["moves/"+evHost]; !ok { t.Fatalf("chores = %+v; want the moves/ chore", f.chores(t)) } }) t.Run("a release by a move the record never carried", func(t *testing.T) { f := newEvFixture(t) f.setClaim(t, func(c *hostclaim.Claim) { c.Op = servingReleasedBy("mv_orphan") }) if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil { t.Fatalf("no lane of the orphan, so nothing unproven: %v", err) } f.bucket.Put("data", "moves/"+evHost+"/mv_orphan/move.json", []byte("orphan")) f.rec.LastMove.At = time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC) bothRefuse(t, f, true) }) } // TestMovesEvidenceRefusesAZeroTime (M7, M8): a zero time is never // evidence: not the close's, and not a lane object's. func TestMovesEvidenceRefusesAZeroTime(t *testing.T) { t.Run("a zero close", func(t *testing.T) { f := newEvFixture(t) f.rec.LastMove.At = time.Time{} bothRefuse(t, f, true) }) t.Run("a close not before the provider's clock", func(t *testing.T) { f := newEvFixture(t) f.rec.LastMove.At = evNow bothRefuse(t, f, true) }) t.Run("an older lane object with no time", func(t *testing.T) { f := newEvFixture(t) obs := f.observe(t) census := []lease.ObjectVersion{{Key: "moves/" + evHost + "/mv_older/move.json", VersionID: "v1", IsLatest: true}} if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census); err == nil { t.Fatal("a zero LastModified counted as before the close") } else { wantRefusal(t, err, true) } if _, _, err := f.provider().VerifyRetirement(t.Context(), obs, census); err == nil { t.Fatal("VerifyRetirement answered over a lane with no time") } }) } // TestMovesSweepProofRefusesAnUnknownLane (M8): every moves/// // object is the closed move's own, or older than its close. An object of // another op written after the close has an unknown writer. func TestMovesSweepProofRefusesAnUnknownLane(t *testing.T) { t.Run("an older lane is covered by the close", func(t *testing.T) { f := newEvFixture(t) f.lane("mv_older", "move.json") f.lane("rb_return", "replica/0000/db") if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil { t.Fatal(err) } }) t.Run("a lane written after the close", func(t *testing.T) { f := newEvFixture(t) f.rec.LastMove.At = time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC) // before every write f.lane(evMoveID, "move.json") if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), f.census(t)); err != nil { t.Fatalf("the closed move's own lane is its own: %v", err) } f.lane("mv_unknown", "move.json") bothRefuse(t, f, true) }) t.Run("an object outside any lane", func(t *testing.T) { f := newEvFixture(t) f.bucket.Put("data", "moves/"+evHost+"/stray.json", []byte("?")) bothRefuse(t, f, true) }) t.Run("an object outside the host's namespaces", func(t *testing.T) { f := newEvFixture(t) census := []lease.ObjectVersion{{Key: "moves/other.example/" + evMoveID + "/move.json", VersionID: "v1", LastModified: evAt.Add(-time.Hour)}} if _, err := f.provider().VerifyMovesSweep(t.Context(), f.observe(t), census); err == nil { t.Fatal("another host's object counted") } }) } // TestMovesEvidenceRefusesAnotherIdentity: the record must be the holder's // exact record, and a holderless claim has no record to prove from. func TestMovesEvidenceRefusesAnotherIdentity(t *testing.T) { t.Run("a record of another incarnation", func(t *testing.T) { f := newEvFixture(t) f.rec.CreatedAt = evCreated.Add(time.Hour) bothRefuse(t, f, true) }) t.Run("a stale observation", func(t *testing.T) { f := newEvFixture(t) obs := f.observe(t) obs.ETag = "" if _, err := f.provider().VerifyMovesSweep(t.Context(), obs, f.census(t)); err == nil { t.Fatal("answered without an etag") } }) t.Run("a free claim", func(t *testing.T) { f := newEvFixture(t) f.setClaim(t, func(c *hostclaim.Claim) { c.Holder, c.Op = nil, hostclaim.Op{} }) obs, census := f.observe(t), f.census(t) _, err := f.provider().VerifyMovesSweep(t.Context(), obs, census) wantRefusal(t, err, false) if _, _, err := f.provider().VerifyRetirement(t.Context(), obs, census); err == nil { t.Fatal("VerifyRetirement answered with no holder") } else { wantRefusal(t, err, false) } if f.reads != 0 { t.Fatalf("read %d records with no holder to name one", f.reads) } }) t.Run("no providers", func(t *testing.T) { f := newEvFixture(t) if _, err := (movesRetirementEvidence{}).VerifyMovesSweep(t.Context(), f.observe(t), nil); err == nil { t.Fatal("answered with no record reader") } if _, _, err := (movesRetirementEvidence{Records: f.provider().Records}).VerifyRetirement(t.Context(), f.observe(t), nil); err == nil { t.Fatal("answered with no clock") } }) } // The provider satisfies both slots RegisterProductionClaimMoveParks reads: // the regional evidence and, by type assertion, the sweep verifier. var ( _ hostbox.RegionalEvidence = movesRetirementEvidence{} _ park.MoveSweepVerifier = movesRetirementEvidence{} ) // --- against the claim's regional worker, with a fake store --- type evLease struct{} func (evLease) Check(context.Context) error { return nil } func (evLease) Release(context.Context) error { return nil } func (evLease) Close() {} type evLeases struct{} func (evLeases) Acquire(context.Context, string, string) (park.HeldLease, error) { return evLease{}, nil } type evEnrollment struct{} func (evEnrollment) VerifyRegional(context.Context, string, string, time.Duration) error { return nil } // worker is hostbox.RegionalWorker over park.Engine, wired as the claim's // assembly wires them (host/claimassembly.go): the provider is the worker's // Evidence and the engine's VerifyMovesSweep. This region (ie) is not the // authoritative one, so the holder's location here is residue too. func (f *evFixture) worker(t *testing.T, listing time.Time) hostbox.RegionalWorker { t.Helper() seq := 0 p := f.provider() location := f.location engine := park.Engine{ Store: f.bucket, Claims: f.claims, Bucket: "data", Region: "ie", Leases: evLeases{}, Now: func() time.Time { return listing }, NewID: func() string { seq++; return fmt.Sprintf("id%d", seq) }, View: func(_ context.Context, o hostclaim.Observed) (park.View, error) { return park.View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "uk", GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil }, Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }, VerifyMovesSweep: p.VerifyMovesSweep, } return hostbox.RegionalWorker{ Regional: hostbox.Regional{Engine: engine, Owner: func(context.Context, hostclaim.Observed) error { return nil }}, Enrollment: evEnrollment{}, Evidence: p, } } func (f *evFixture) chores(t *testing.T) map[string]hostclaim.ParkScope { t.Helper() out := map[string]hostclaim.ParkScope{} for _, o := range f.observe(t).Claim.Obligations { if o.Park == nil { t.Fatalf("an unscoped chore: %+v", o) } out[o.Park.Location] = *o.Park } return out } // TestAllowedSweepQueuesTheMovesChore: with the proof, the worker queues the // moves/ chore under the serving session, labelled with the holder's // incarnation, its evidence the listing's end, so it waits for its horizon. // The holder location's residue in this non-authoritative region takes the // close as its evidence: that horizon has passed, so the same pass parks it. func TestAllowedSweepQueuesTheMovesChore(t *testing.T) { f := newEvFixture(t) f.lane(evMoveID, "move.json") f.lane("mv_older", "replica/0000/db") left := f.genKey("old") // superseded by the move's own admission f.bucket.Put("data", left, []byte("left behind")) err := f.worker(t, evNow).Run(t.Context()) if !errors.Is(err, park.ErrPending) { t.Fatalf("a new sweep waits for its horizon: %v", err) } chores := f.chores(t) moves, ok := chores["moves/"+evHost] if !ok || len(chores) != 1 { t.Fatalf("chores = %+v; want the moves chore alone", chores) } if moves.Incarnation != f.inc || !moves.EvidenceAt.Equal(evNow) || moves.RequestLimit != hostbox.RequestLimit { t.Fatalf("moves chore = %+v; want incarnation %s, evidence %v", moves, f.inc, evNow) } if vs, err := f.bucket.ListVersions(t.Context(), "data", left); err != nil || len(vs) != 0 { t.Fatalf("the location's residue was not parked: %v, %v", vs, err) } if vs, err := f.bucket.ListVersions(t.Context(), "data", "parked/"+f.inc+"/"+evHost+"/"); err != nil || len(vs) == 0 { t.Fatalf("no archive under the holder's incarnation: %v, %v", vs, err) } for _, o := range f.observe(t).Claim.Obligations { if o.Op != "session:2" || o.Owner != "region:ie" { t.Fatalf("chore %+v is not the serving session's regional chore", o) } } if c := f.observe(t).Claim; c.Op.Kind != hostclaim.KindServing || c.Op.Journal["released_move"] != evMoveID { t.Fatalf("the sweep changed the serving op: %+v", c.Op) } } // TestSweepSkipsWhileAMoveStages (Review Focus 3, K10): while a move op is // on the claim the worker never asks for evidence; while the record carries // a live move under a serving claim the provider refuses and nothing is // queued. func TestSweepSkipsWhileAMoveStages(t *testing.T) { t.Run("a moving claim", func(t *testing.T) { f := newEvFixture(t) f.setClaim(t, func(c *hostclaim.Claim) { c.Op = hostclaim.Op{Kind: hostclaim.KindMoving, ID: "mv_next", Phase: hostclaim.PhaseMoving, Exec: "box"} }) f.rec.Move = &provision.MoveState{ID: "mv_next", Incarnation: f.inc, To: "uk", State: provision.MovePreparing} f.lane("mv_next", "replica/0000/db") if err := f.worker(t, evNow).Run(t.Context()); err != nil { t.Fatal(err) } if len(f.chores(t)) != 0 || f.reads != 0 { t.Fatalf("chores %+v, record reads %d", f.chores(t), f.reads) } }) t.Run("a live move on the record", func(t *testing.T) { f := newEvFixture(t) f.rec.Move = &provision.MoveState{ID: "mv_next", Incarnation: f.inc, To: "uk", State: provision.MovePreparing} f.lane("mv_next", "replica/0000/db") err := f.worker(t, evNow).Run(t.Context()) var r *MovesSweepRefusal if !errors.As(err, &r) { t.Fatalf("err = %v, want the provider's refusal", err) } if len(f.chores(t)) != 0 { t.Fatalf("chores %+v", f.chores(t)) } }) } // TestSweepRefusedUnlessTheCloseIsBeforeTheListing (K10): the worker's own // Check drops the moves/ candidate when the newest retired move's // close is not earlier than the listing; the rest of the decision stands. func TestSweepRefusedUnlessTheCloseIsBeforeTheListing(t *testing.T) { f := newEvFixture(t) f.lane(evMoveID, "move.json") f.bucket.Put("data", f.genKey("old"), []byte("left behind")) err := f.worker(t, evAt).Run(t.Context()) if !errors.Is(err, park.ErrPending) { t.Fatalf("the location chore still queues: %v", err) } chores := f.chores(t) if _, ok := chores["moves/"+evHost]; ok || len(chores) != 1 { t.Fatalf("chores = %+v; want only the location's", chores) } if loc := chores[f.location]; loc.Incarnation != f.inc || !loc.EvidenceAt.Equal(evAt) { t.Fatalf("location chore = %+v; want the close %v as its evidence", loc, evAt) } } // TestSweepScopeMembership (witness wSweepScopeMembership, M7): residue of // an older incarnation is never the moves/ chore's. The provider no // longer refuses the sweep for it (Q4): the engine queues it as its own // chore under its own qualified identity and parks it, and the moves/ // chore keeps the holder's. func TestSweepScopeMembership(t *testing.T) { f := newEvFixture(t) older := strings.Repeat("cd", 32) f.lane(evMoveID, "move.json") residue := "instances/" + evHost + "/" + older + "/g-old/0000/db" f.bucket.Put("data", residue, []byte("late")) if err := f.worker(t, evNow).Run(t.Context()); !errors.Is(err, park.ErrPending) { t.Fatalf("the chores wait for their horizons: %v", err) } chores := f.chores(t) moves, ok := chores["moves/"+evHost] if !ok || moves.Incarnation != f.inc { t.Fatalf("chores = %+v; want the moves/ chore under the holder's incarnation", chores) } if _, ok := chores["instances/"+evHost+"/"+older]; ok || len(chores) != 1 { t.Fatalf("chores = %+v; want only the moves/ chore pending", chores) } // The older incarnation's chore ran first, under its own identity, and // parked its residue in the same pass. if _, ok := f.bucket.Current("data", residue); ok { t.Fatal("the older incarnation's residue was not parked") } } // TestFreeHostLanesAreTheClaimsProof (fix round 1, ruling 3; Q4): a free // claim names no record, so moves can prove no closed move and the claim // stays free. The free-host lane proof is the claim's own // (claimFreeMoveProof, asked before the provider); a free claim that // reaches the provider gets the quiet sweep refusal, not a dependency. func TestFreeHostLanesAreTheClaimsProof(t *testing.T) { f := newEvFixture(t) f.setClaim(t, func(c *hostclaim.Claim) { c.Holder, c.Op, c.Touched = nil, hostclaim.Op{}, hostclaim.Touched{} }) f.lane(evMoveID, "move.json") err := f.worker(t, evNow).Run(t.Context()) wantRefusal(t, err, false) if c := f.observe(t).Claim; !c.Free() || len(c.Obligations) != 0 { t.Fatalf("claim = %+v", c) } } // authoritative makes ie the authoritative region, with the marker at the // holder's current generation and objects classified by the landing's // layout, as Task 9a's view will. func (f *evFixture) authoritative(t *testing.T, w *hostbox.RegionalWorker, current string) { t.Helper() marker := f.location + "/generation.json" ref := park.Ref{Key: marker, VersionID: f.bucket.LatestID("data", marker)} location := f.location w.Regional.Engine.View = func(_ context.Context, o hostclaim.Observed) (park.View, error) { return park.View{ClaimETag: o.ETag, Region: "ie", AuthoritativeRegion: "ie", MarkerKnown: true, MarkerGeneration: current, Marker: ref, GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil } } // TestAuthoritativeSweepStampsTheCloseOnlyOnWhatTheMoveRetired (fix round 1, // ruling 1): in the authoritative region the engine makes superseded // generations park-generation candidates and binds them to // VerifyRetirement's time. A generation the move's admission superseded // gets the close; when the move's own generation was superseded after the // close, the provider refuses and nothing is queued. func TestAuthoritativeSweepStampsTheCloseOnlyOnWhatTheMoveRetired(t *testing.T) { listing := evAt.Add(time.Minute) // before the close's horizon, so chores stay queued t.Run("superseded by the move", func(t *testing.T) { f := newEvFixture(t) f.lane(evMoveID, "move.json") f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) f.bucket.Put("data", f.location+"/generation.json", []byte("marker")) f.bucket.Put("data", f.genKey("old"), []byte("superseded by the move")) w := f.worker(t, listing) f.authoritative(t, &w, evMoveID) if err := w.Run(t.Context()); !errors.Is(err, park.ErrPending) { t.Fatalf("the chores wait for their horizons: %v", err) } var gens []hostclaim.Obligation for _, o := range f.observe(t).Claim.Obligations { if o.Kind == "park-generation" { gens = append(gens, o) } } if len(gens) != 1 || gens[0].Generation != "old" || gens[0].Park.Location != f.location || !gens[0].Park.EvidenceAt.Equal(evAt) { t.Fatalf("generation chores = %+v; want one for old at the close %v", gens, evAt) } if moves, ok := f.chores(t)["moves/"+evHost]; !ok || !moves.EvidenceAt.Equal(listing) { t.Fatalf("chores = %+v; want the moves chore at the listing", f.chores(t)) } }) t.Run("superseded after the close", func(t *testing.T) { f := newEvFixture(t) f.setClaim(t, func(c *hostclaim.Claim) { c.Holder.Admissions, c.Holder.Generation, c.Holder.Session, c.Holder.Stamp = evAdmissions(evMoveID, "later"), "later", 3, 3 }) f.lane(evMoveID, "move.json") f.bucket.Put("data", f.genKey("later"), []byte("live")) f.bucket.Put("data", f.location+"/generation.json", []byte("marker")) f.bucket.Put("data", f.genKey(evMoveID), []byte("written until after the close")) w := f.worker(t, listing) f.authoritative(t, &w, "later") err := w.Run(t.Context()) var ne *NotMovesEvidence if !errors.As(err, &ne) || ne.Generation != evMoveID { t.Fatalf("err = %v, want a *NotMovesEvidence for %s", err, evMoveID) } if len(f.observe(t).Claim.Obligations) != 0 { t.Fatalf("chores %+v", f.observe(t).Claim.Obligations) } }) } // TestMovesParkLockMatchesTheEngine (fix round 2): the provider mirrors the // park engine's unexported lock predicate (park's isLock over locationOf), // so it skips exactly the lease objects Decide skips. park.Decide is the // oracle: over a one-key census in a non-authoritative region, where every // non-lock object is a candidate, it decides nothing exactly for a lock. func TestMovesParkLockMatchesTheEngine(t *testing.T) { f := newEvFixture(t) obs := f.observe(t) view := park.View{ClaimETag: obs.ETag, Region: "ie", AuthoritativeRegion: "uk", GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(f.location, r.Key) }} other := strings.Repeat("ab", 32) for _, key := range []string{ "instances/" + evHost + "/lock.json", "instances/" + evHost + "/" + f.inc + "/lock.json", "instances/" + evHost + "/" + other + "/lock.json", "moves/" + evHost + "/lock.json", "moves/" + evHost + "/" + evMoveID + "/lock.json", "instances/" + evHost + "/g-old/lock.json", "instances/" + evHost + "/" + f.inc + "/g-old/lock.json", "instances/" + evHost + "/lock.json.old", "instances/" + evHost + "/0000/db", } { d, err := park.Decide(obs, view, []lease.ObjectVersion{{Key: key, VersionID: "v1", LastModified: evAt.Add(-time.Hour)}}, f.inc) if err != nil { t.Fatalf("%s: %v", key, err) } if engine, mirror := len(d.Candidates) == 0, movesParkLock(evHost, key); engine != mirror { t.Errorf("%s: the engine skips it %v, the provider's mirror says lock %v", key, engine, mirror) } } } // TestLiveLocationLeaseIsSweptNormally (fix round 2): the writer's lease, // /lock.json (lease.Acquire), stands at every live // location. It is the engine's to skip, never residue the close must // vouch for, so the sweep proceeds, scoped and bare alike. func TestLiveLocationLeaseIsSweptNormally(t *testing.T) { for name, bare := range map[string]bool{"scoped": false, "bare": true} { t.Run(name, func(t *testing.T) { f := newEvFixture(t) if bare { f.bare(t) } f.lane(evMoveID, "move.json") f.bucket.Put("data", f.genKey(evMoveID), []byte("live")) f.bucket.Put("data", f.location+"/generation.json", []byte("marker")) f.bucket.Put("data", f.location+"/lock.json", []byte("writer lease")) if _, _, err := f.provider().VerifyRetirement(t.Context(), f.observe(t), f.census(t)); err != nil { t.Fatalf("VerifyRetirement refused the writer's lease: %v", err) } listing := evAt.Add(time.Minute) w := f.worker(t, listing) f.authoritative(t, &w, evMoveID) if err := w.Run(t.Context()); !errors.Is(err, park.ErrPending) { t.Fatalf("the lane chore waits for its horizon: %v", err) } chores := f.chores(t) if moves, ok := chores["moves/"+evHost]; !ok || len(chores) != 1 || !moves.EvidenceAt.Equal(listing) { t.Fatalf("chores = %+v; want the moves chore alone", chores) } if vs, err := f.bucket.ListVersions(t.Context(), "data", f.location+"/lock.json"); err != nil || len(vs) != 1 { t.Fatalf("the lease was touched: %v, %v", vs, err) } }) } }