package host import ( "context" "strings" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) // Settings spec rev 11 ยง4.3a: a move carries the build state with the // replica, and a held instance never reads as new before its first wake on // the destination. const stagedStateHost = "a.example.com" func stagedStateBody(host, le string) string { return `{"schema":1,"host":"` + host + `","last_executed":"` + le + `","at":"2026-10-07T12:00:00Z"}` } func putObject(t *testing.T, s release.Store, key, body string) { t.Helper() if _, err := s.Put(context.Background(), key, []byte(body), ""); err != nil { t.Fatal(err) } } // legacyStaged is newBuildStates' staged prefix: the legacy round's // moves//, read only with host claims and Phase 1 off. func legacyStaged(h string) string { return activator.MovesPrefix(h) } // Before the first wake promotes a legacy landing's staging, the own prefix // is empty. The staged build-state.json is the state and a staged object is // a replica, so a held instance is held on its last_executed, not given the // head as a new instance. func TestAStagedMoveIsReadAsTheInstancesState(t *testing.T) { t.Parallel() b, s, _ := testBuildStates(t) b.staged = legacyStaged ctx := context.Background() putObject(t, s, "moves/"+stagedStateHost+"/generations/0000/x.ltx", "x") putObject(t, s, "moves/"+stagedStateHost+"/build-state.json", stagedStateBody(stagedStateHost, "v1")) st, replica, err := b.read(ctx, stagedStateHost) if err != nil || !replica || st == nil || st.LastExecuted != "v1" { t.Fatalf("read = %+v, %v, %v; want the staged state and a replica", st, replica, err) } // And Target therefore holds a manual instance on v1, not the head. d := release.Target(release.TargetInput{Pointer: &release.ChannelPointer{Version: "v3"}, Settings: release.UpdateSettings{Updates: release.UpdatesManual}, Build: release.BuildStateCopy{Known: true, Replica: replica, State: st}}) if d.Version != "v1" { t.Fatalf("Target = %+v", d) } // refresh records the same in the registry copy root reads. if err := b.refresh(ctx, stagedStateHost); err != nil { t.Fatal(err) } if c := copyOf(t, b.reg); !c.Known || !c.Replica || c.State == nil || c.State.LastExecuted != "v1" { t.Fatalf("copy = %+v", c) } // A staged replica with no build state is still a replica: never new. b2, s2, _ := testBuildStates(t) b2.staged = legacyStaged putObject(t, s2, "moves/"+stagedStateHost+"/generations/0000/x.ltx", "x") if st, replica, err := b2.read(ctx, stagedStateHost); err != nil || !replica || st != nil { t.Fatalf("staged replica only: read = %+v, %v, %v", st, replica, err) } } // The staged copy is read the way the wake's restore will use it // (promoteStagedReplica: staging wins when present, and its mirror deletes // what staging lacks). A host that comes back to a box it left keeps its // old lineage and build state at the own prefix; the restore replaces them // with the staging, so the staged build state is the one that counts. // With nothing staged the own prefix is read as before. func TestAStagedMoveOutranksAnOlderOwnStateAsTheRestoreDoes(t *testing.T) { b, s, _ := testBuildStates(t) b.staged = legacyStaged ctx := context.Background() own := "inst/" + stagedStateHost putObject(t, s, own+"/generations/0000/old.ltx", "old") putObject(t, s, own+"/build-state.json", stagedStateBody(stagedStateHost, "v1")) if st, replica, err := b.read(ctx, stagedStateHost); err != nil || !replica || st == nil || st.LastExecuted != "v1" { t.Fatalf("own prefix alone: read = %+v, %v, %v", st, replica, err) } putObject(t, s, "moves/"+stagedStateHost+"/generations/0000/new.ltx", "new") putObject(t, s, "moves/"+stagedStateHost+"/build-state.json", stagedStateBody(stagedStateHost, "v2")) if st, replica, err := b.read(ctx, stagedStateHost); err != nil || !replica || st == nil || st.LastExecuted != "v2" { t.Fatalf("staged over own: read = %+v, %v, %v; want the staged v2", st, replica, err) } // update still writes only the own prefix: the wake promotes first. if _, err := b.update(ctx, stagedStateHost, func(cur *release.BuildState) release.BuildState { return release.BuildState{LastExecuted: "v2"} }); err != nil { t.Fatal(err) } if raw, _, err := s.Get(ctx, "moves/"+stagedStateHost+"/build-state.json"); err != nil || !strings.Contains(string(raw), `"v2"`) { t.Fatalf("update touched the staging: %s %v", raw, err) } } // Only the legacy round's own keys are its staging. moves// also // holds Phase 1 op lanes (moves///...), which no // legacy restore promotes and which say nothing about this box's state. func TestTheStagedReadExcludesPhase1OpLanes(t *testing.T) { b, s, _ := testBuildStates(t) b.staged = legacyStaged ctx := context.Background() for _, id := range []string{p1ID, p1RB} { putObject(t, s, moves.ReplicaPrefix(stagedStateHost, id)+"0000/x.ltx", "x") putObject(t, s, moves.BuildStateKey(stagedStateHost, id), stagedStateBody(stagedStateHost, "v9")) } if st, replica, err := b.read(ctx, stagedStateHost); err != nil || replica || st != nil { t.Fatalf("read = %+v, %v, %v; want no state and no replica (op lanes only)", st, replica, err) } // A legacy key beside the op lanes is staging again. putObject(t, s, "moves/"+stagedStateHost+"/build-state.json", stagedStateBody(stagedStateHost, "v1")) if st, _, err := b.read(ctx, stagedStateHost); err != nil || st == nil || st.LastExecuted != "v1" { t.Fatalf("read = %+v, %v; want the legacy staged v1", st, err) } } // The staged read is the legacy round's: with Phase 1 on the restore never // promotes moves// (ExecBackend.Phase1), and with host claims on the // state is at the claim holder's location, so newBuildStates leaves it off. func TestTheStagedReadIsOffWithPhase1OrClaimsOn(t *testing.T) { cfg := &activator.Config{Bucket: "hib", Prefix: "inst"} t.Setenv(moves.SwitchEnv, "off") b, err := newBuildStates(cfg, nil, "") if err != nil || b == nil || b.staged == nil || b.staged(stagedStateHost) != "moves/"+stagedStateHost+"/" { t.Fatalf("Phase 1 off: staged = %v (%v)", b, err) } t.Setenv(moves.SwitchEnv, "on") if b, err := newBuildStates(cfg, nil, ""); err != nil || b == nil || b.staged != nil { t.Fatalf("Phase 1 on: the staged read is armed (%v)", err) } t.Setenv(moves.SwitchEnv, "off") read := func(context.Context, string) (hostclaim.Observed, error) { return hostclaim.Observed{}, nil } claimed := &activator.Config{Bucket: "hib", Prefix: "inst", HostWrites: hostwrite.Config{Enabled: true}, Claims: &activator.ClaimRuntime{Bucket: "data", Read: read}} if b, err := newBuildStates(claimed, nil, ""); err != nil || b == nil || b.staged != nil { t.Fatalf("claims on: the staged read is armed (%v)", err) } // With it off, a staged copy is not read. b2, s, _ := testBuildStates(t) putObject(t, s, "moves/"+stagedStateHost+"/build-state.json", stagedStateBody(stagedStateHost, "v1")) putObject(t, s, "moves/"+stagedStateHost+"/generations/0000/x.ltx", "x") if st, replica, err := b2.read(context.Background(), stagedStateHost); err != nil || replica || st != nil { t.Fatalf("read = %+v, %v, %v; want nothing staged read", st, replica, err) } } // p1StateBody is a build state for the Phase 1 harness's host. func p1StateBody(le string) string { return stagedStateBody(p1Host, le) } // storeState puts host's build state naming le at bucket/key, its ETag // unique to le as a real object's is to its bytes. func storeState(h *phase1Harness, bucket, key, le string) { body := p1StateBody(le) h.f.store(bucket, key, fakeMoveObject{lease.MirrorObject{ETag: "bs-" + le, Size: int64(len(body))}, body}) } // The Phase 1 replica lane carries the build state beside the replica in // every round: exactly build-state.json from the claim holder's location, // under the lane's state/ prefix; a later round carries a later state. func TestThePhase1LaneCopiesTheBuildState(t *testing.T) { h := newPhase1Harness(t) h.start(t, "manual") loc := strings.TrimSuffix(h.claim(t).Holder.Location, "/") storeState(h, "data-ie", loc+"/build-state.json", "v6") h.pass(t) key := moves.BuildStateKey(p1Host, p1ID) if got := h.f.buckets["data-uk"][key].body; got != p1StateBody("v6") { t.Fatalf("the lane staged %q at %s, want the holder's build state", got, key) } state := stagedKeys(h, moves.StagingPrefix(p1Host, p1ID)+"state/") if strings.Join(state, ",") != "build-state.json" { t.Fatalf("the state lane holds %v, want only build-state.json", state) } storeState(h, "data-ie", loc+"/build-state.json", "v7") h.pass(t) if got := h.f.buckets["data-uk"][key].body; got != p1StateBody("v7") { t.Fatalf("the next round staged %q, want the later state", got) } // Every write of the state lane went through the guard. var guarded bool for _, c := range h.f.stagingCalls() { if c.Key == key { guarded = !c.SignAt.IsZero() } } if !guarded { t.Fatalf("the state copy did not go through the guard: %+v", h.f.stagingCalls()) } } // The cut's byte-final round carries the final build state, not the one a // preparing round staged before the instance last ran. func TestTheCutsFinalRoundCopiesTheBuildState(t *testing.T) { h := newCutHarness(t) storeState(h.phase1Harness, "data-ie", cutRoot+"/build-state.json", "v6") h.ready(t) // the holder is at cutRoot from here if got := h.f.buckets["data-uk"][moves.BuildStateKey(p1Host, p1ID)].body; got != p1StateBody("v6") { t.Fatalf("the preparing rounds staged %q", got) } // The instance's last run before the stop wrote a later state: only the // final round, after the final checkpoint, can carry it. checkpoint := h.cfg.FinalCheckpoint h.cfg.FinalCheckpoint = func(ctx context.Context, host, path string, holder hostclaim.Holder, check func(context.Context) error) error { storeState(h.phase1Harness, "data-ie", cutRoot+"/build-state.json", "v7") return checkpoint(ctx, host, path, holder, check) } h.cut(t) h.pass(t) if m := h.move(t); m.State != provision.MoveLanding { t.Fatalf("cut did not finish: %+v", m) } key := moves.BuildStateKey(p1Host, p1ID) if got := h.f.buckets["data-uk"][key].body; got != p1StateBody("v7") { t.Fatalf("the cut staged %q, want the final state", got) } // The seal names that copy by its version (ruling R-SEAL-STATE). seal := h.seal(t) if len(seal.State) != 1 || seal.State[0].Key != key || seal.State[0].VersionID != h.f.versionOf("data-uk", key) || !lease.UsableVersionID(seal.State[0].VersionID) || seal.State[0].Size != int64(len(p1StateBody("v7"))) { t.Fatalf("seal state %+v, lane version %s", seal.State, h.f.versionOf("data-uk", key)) } } // The landing places the staged build state beside the location's lease, // holder.Location/build-state.json in the target's own bucket, before the // probe: the probe's and the first wake's record merge onto the moved // state, and a held instance never reads as new there. func TestTheLandingPlacesTheBuildStateBeforeTheProbe(t *testing.T) { t.Parallel() h := newLandingHarness(t) storeState(h.phase1Harness, "data-ie", cutRoot+"/build-state.json", "v6") h.flip(t) if got := h.f.buckets["data-uk"][moves.BuildStateKey(p1Host, p1ID)].body; got != p1StateBody("v6") { t.Fatalf("the lane holds %q", got) } dst := cutRoot + "/build-state.json" if _, ok := h.f.buckets["data-uk"][dst]; ok { t.Fatal("the build state is at the location before the landing") } h.land(t) h.installed(t) if got := h.f.buckets["data-uk"][dst].body; got != p1StateBody("v6") { t.Fatalf("after the install the location holds %q, want the moved state", got) } // The probe sees it. if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, func(rec *provision.Record) error { rec.Move.CutAttempts = 1 return nil }); err != nil { t.Fatal(err) } p := &probeLog{t: t, h: h} var seen []string p.before = func(int) { seen = append(seen, h.f.buckets["data-uk"][dst].body) } h.probe, h.probeLive = p.wake, p.liveProbe h.holdProbed = true h.land(t) if len(seen) == 0 || seen[0] != p1StateBody("v6") { t.Fatalf("the probe saw %q, want the moved state", seen) } } // A landing places exactly the sealed version: a request a stale round // signed may land a newer version in the lane until the seal's bound, and // that version is never placed (ruling R-SEAL-STATE). func TestTheLandingPlacesTheSealedVersionNotALaterOne(t *testing.T) { t.Parallel() h := newLandingHarness(t) h.f.keepHistory = true storeState(h.phase1Harness, "data-ie", cutRoot+"/build-state.json", "v6") h.flip(t) key := moves.BuildStateKey(p1Host, p1ID) sealed := h.seal(t).State if len(sealed) != 1 || sealed[0].VersionID != h.f.versionOf("data-uk", key) { t.Fatalf("seal state %+v", sealed) } if _, err := h.f.PutVersionedAt(context.Background(), h.clock.Now(), "data-uk", key, []byte(p1StateBody("v5"))); err != nil { t.Fatal(err) } if h.f.versionOf("data-uk", key) == sealed[0].VersionID { t.Fatal("the late write made no new version") } h.land(t) h.installed(t) if got := h.f.buckets["data-uk"][cutRoot+"/build-state.json"].body; got != p1StateBody("v6") { t.Fatalf("the location holds %q, want the sealed v6", got) } } // A landing whose source had no build state seals none and places none, // even if a lane object appears after the seal. func TestALandingWithNoBuildStatePlacesNone(t *testing.T) { t.Parallel() h := newLandingHarness(t) h.flip(t) if st := h.seal(t).State; len(st) != 0 { t.Fatalf("seal state %+v, want none", st) } if _, err := h.f.PutVersionedAt(context.Background(), h.clock.Now(), "data-uk", moves.BuildStateKey(p1Host, p1ID), []byte(p1StateBody("v5"))); err != nil { t.Fatal(err) } h.land(t) h.installed(t) if _, ok := h.f.buckets["data-uk"][cutRoot+"/build-state.json"]; ok { t.Fatal("a build state appeared at the location") } } // A landing whose seal carries no state removes what an earlier stay left // beside the target's lease (Task 12a): otherwise that stay's build state // would read as this instance's. The removal runs under the location's // lease, like the placement. func TestALandingWithNoBuildStateRemovesALeftoverOne(t *testing.T) { t.Parallel() h := newLandingHarness(t) key := cutRoot + "/build-state.json" storeState(h.phase1Harness, "data-uk", key, "v2") // an earlier stay's h.flip(t) if st := h.seal(t).State; len(st) != 0 { t.Fatalf("seal state %+v, want none", st) } h.land(t) h.installed(t) if o, ok := h.f.buckets["data-uk"][key]; ok { t.Fatalf("an earlier stay's build state %q is still beside the lease", o.body) } if calls := h.vs.callsOn("delete-version data-uk/" + key); len(calls) == 0 { t.Fatal("the leftover was not removed by version") } } // A committed move's source location park acknowledges with a build state // beside the source's lease (the 8b reviewer's regression): the state is the // marker's twin, never an object of no generation that would hold the // park-replica open for ever. func TestTheSourceLocationParkAcknowledgesWithABuildState(t *testing.T) { t.Parallel() lh := newLandingHarness(t) storeState(lh.phase1Harness, "data-ie", cutRoot+"/build-state.json", "v6") h := newFinalHarnessFrom(t, lh) // One admission on the holder: with none the engine names no parked // generations at all, and the build state is never classified. rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { c.Holder.Admissions = append(c.Holder.Admissions, hostclaim.AdmissionRecord{Region: "ie", Session: 1, Box: "ie-box", Generation: "cur", Mode: "cold"}) }) h.commit(t) h.clock.Advance(park.SignatureAge + cutRequestLimit + time.Minute) h.sourcePass(t) for _, o := range h.claim(t).Obligations { if o.Kind == "park-replica" && o.Region == "ie" { t.Fatalf("the ie park-replica is still owed past its horizon: %+v", o) } } } // returnSealState is the return seal's state class, checked against the // lane object it names. func returnSealState(t *testing.T, h *rollbackHarness, rbID string) (moves.SealedObject, string) { t.Helper() seal, err := moves.DecodeSeal(h.claim(t).Op.Journal[moves.JournalReturnSeal]) if err != nil { t.Fatal(err) } key := moves.BuildStateKey(p1Host, rbID) if len(seal.State) != 1 || seal.State[0].Key != key || seal.State[0].VersionID != h.f.versionOf("data-ie", key) { t.Fatalf("return seal state %+v, lane version %s", seal.State, h.f.versionOf("data-ie", key)) } return seal.State[0], h.f.buckets["data-ie"][key].body } // The rollback's return lane carries the target's build state back, under // the lane's state/ prefix in the source's bucket, sealed by version. Once // the target's install has run, the state beside its lease is the one that // goes back: here the probe recorded v8 over the moved v6. func TestTheReturnLaneCarriesTheBuildState(t *testing.T) { t.Parallel() h := &rollbackHarness{landingHarness: newLandingHarness(t)} storeState(h.phase1Harness, "data-ie", cutRoot+"/build-state.json", "v6") h.flip(t) h.land(t) h.installed(t) h.step(t, provision.LandRestored, provision.LandProbing) h.tpos = h.n storeState(h.phase1Harness, "data-uk", cutRoot+"/build-state.json", "v8") h.uncover(t) h.rollBack(t) h.rollbackPass(t) rb := h.rb(t) if !rb.Dirty || rb.TargetPhase != provision.RbTargetReturned { t.Fatalf("rollback %+v", rb) } if _, body := returnSealState(t, h, rb.ID); body != p1StateBody("v8") { t.Fatalf("the return lane holds %q, want the target's state", body) } } // A return before the target's install built anything: the target's // location holds no state of this move, so the forward lane's sealed state // goes back as given (by its exact version, not the lane's latest). func TestAReturnBeforeTheInstallCarriesTheSealedState(t *testing.T) { t.Parallel() h := &rollbackHarness{landingHarness: newLandingHarness(t)} h.f.keepHistory = true storeState(h.phase1Harness, "data-ie", cutRoot+"/build-state.json", "v6") h.flip(t) if _, err := h.f.PutVersionedAt(context.Background(), h.clock.Now(), "data-uk", moves.BuildStateKey(p1Host, p1ID), []byte(p1StateBody("v5"))); err != nil { t.Fatal(err) } h.landAt = "after-record-" + provision.LandBuilding h.crashLand(t) if m := h.read(t, p1Host).Move; m.LandingPhase != provision.LandBuilding { t.Fatalf("landing phase %s", m.LandingPhase) } h.uncover(t) h.rollBack(t) h.rollbackPass(t) rb := h.rb(t) if !rb.Dirty || rb.TargetPhase != provision.RbTargetReturned { t.Fatalf("rollback %+v", rb) } if _, body := returnSealState(t, h, rb.ID); body != p1StateBody("v6") { t.Fatalf("the return lane holds %q, want the forward lane's sealed v6", body) } } // The dirty return's install places the returned build state beside the // source's location lease, as the landing does. func TestTheReturnInstallPlacesTheBuildState(t *testing.T) { t.Parallel() h := newReturnHarness(t, true) h.f.keepHistory = true key := moves.BuildStateKey(p1Host, p1RB) ver, err := h.vs.PutIfAbsent(context.Background(), "data-ie", key, []byte(p1StateBody("v8"))) if err != nil { t.Fatal(err) } h.seal.State = []moves.SealedObject{{Key: key, VersionID: ver, Size: int64(len(p1StateBody("v8")))}} if err := h.seal.Validate(); err != nil { t.Fatal(err) } // A later lane version is not the sealed one and is not placed. if _, err := h.f.PutVersionedAt(context.Background(), h.clock.Now(), "data-ie", key, []byte(p1StateBody("v5"))); err != nil { t.Fatal(err) } pinned := h.read(t, p1Host).Move.Inputs if refused, err := h.install(t); err != nil || refused != "" { t.Fatalf("install: %q %v", refused, err) } h.installedReturn(t, pinned) if got := h.f.buckets["data-ie"][cutRoot+"/build-state.json"].body; got != p1StateBody("v8") { t.Fatalf("the source's location holds %q, want the returned state", got) } } // A wake that promoted the staging and then failed to drop it // (dropStagedReplica is best-effort) leaves the pre-move staged state // behind its own newer record: when both hold a state, the later At wins. func TestALaterOwnStateOutranksALeftoverStaging(t *testing.T) { b, s, _ := testBuildStates(t) b.staged = legacyStaged ctx := context.Background() putObject(t, s, "moves/"+stagedStateHost+"/generations/0000/x.ltx", "x") putObject(t, s, "moves/"+stagedStateHost+"/build-state.json", stagedStateBody(stagedStateHost, "v1")) own := "inst/" + stagedStateHost putObject(t, s, own+"/generations/0000/x.ltx", "x") putObject(t, s, own+"/build-state.json", `{"schema":1,"host":"`+stagedStateHost+`","last_executed":"v2","at":"2026-10-08T12:00:00Z"}`) if st, replica, err := b.read(ctx, stagedStateHost); err != nil || !replica || st == nil || st.LastExecuted != "v2" { t.Fatalf("read = %+v, %v, %v; want the own, later v2", st, replica, err) } // An older own state (an earlier stay's) loses to the staging. _, etag, err := s.Get(ctx, own+"/build-state.json") if err != nil { t.Fatal(err) } if _, err := s.Put(ctx, own+"/build-state.json", []byte(`{"schema":1,"host":"`+stagedStateHost+`","last_executed":"v0","at":"2026-10-01T12:00:00Z"}`), etag); err != nil { t.Fatal(err) } if st, _, err := b.read(ctx, stagedStateHost); err != nil || st == nil || st.LastExecuted != "v1" { t.Fatalf("read = %+v, %v; want the staged v1", st, err) } } // An abort retires the build-state lane's receipts with the replica lane's. func TestAbortRetiresTheBuildStateReceipts(t *testing.T) { h := newAbortHarness(t) loc := strings.TrimSuffix(h.claim(t).Holder.Location, "/") storeState(h.phase1Harness, "data-ie", loc+"/build-state.json", "v6") h.pass(t) key := moves.BuildStateReceiptsKey(p1Host, p1ID) if _, ok := h.f.buckets["data-ie"][key]; !ok { t.Fatalf("the state lane wrote no receipts at %s", key) } h.abort(t) h.pass(t) h.waiting(t) h.ownStateGone(t) if _, ok := h.f.buckets["data-ie"][key]; ok { t.Fatalf("%s survived the abort", key) } }