package host import ( "context" "encoding/json" "errors" "os" "path/filepath" "slices" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" execctl "github.com/carlosframework/platform/internal/hostexec" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/release" ) type recordingDrain struct{ calls *[]string } func (d recordingDrain) DrainClaim(context.Context, StopRequest, string) (DrainAcknowledgement, error) { *d.calls = append(*d.calls, "producers") return DrainAcknowledgement{FenceID: "fence"}, nil } type memPositions struct { calls *[]string p ReleasePosition ok bool } func (m *memPositions) Record(_ context.Context, p ReleasePosition) error { *m.calls = append(*m.calls, "record") m.p, m.ok = p, true return nil } func (m *memPositions) Recorded(_ context.Context, r StopRequest) (ReleasePosition, bool, error) { if !m.ok || !sameStop(m.p.Request, r) { return ReleasePosition{}, false, nil } return m.p, true, nil } // writersStoppedFor is a complete StopWriters witness and root observation // for o's writer session, by "src" (main's ClaimAssembly.WritersStopped). func writersStoppedFor(o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { h := o.Claim.Holder at := time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC) target := hostguard.FenceTarget{Host: o.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: "src", Enrollment: "enr", BootID: "boot", ControllerID: "ctl"} drained := ClaimWritersDrained{Target: target, Generation: h.Generation, WriterBox: h.WriterBox, FenceClosed: true, LaunchJoined: true, WorkDrained: true, SpawnsSettled: true, CgroupEmpty: true, Cgroup: "carlos/shop", DBPaths: []string{"/data/shop.db"}, CheckedAt: at} seen := WritersStoppedEvidence{Host: o.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, WriterSession: h.Session, Box: "src", Enrollment: "enr", BootID: "boot", ControllerID: "ctl", ClaimETag: o.ETag, CheckedAt: at} return drained, seen, nil } // fakeReleaseObligation records main's relocation-release CASes: Queue // before root's stop, Complete with its receipt. type fakeReleaseObligation struct { calls *[]string queueErr error completeErr error completed []string } func (f *fakeReleaseObligation) Queue(_ context.Context, o hostclaim.Observed) (hostclaim.Observed, error) { *f.calls = append(*f.calls, "queue") return o, f.queueErr } func (f *fakeReleaseObligation) Verify(context.Context, hostclaim.Observed, string, string) error { *f.calls = append(*f.calls, "verify") return nil } func (f *fakeReleaseObligation) Complete(_ context.Context, o hostclaim.Observed, key, digest string) (hostclaim.Observed, error) { *f.calls = append(*f.calls, "complete") f.completed = append(f.completed, key+"@"+digest) return o, f.completeErr } func effectsFixture(t *testing.T) (*RelocationEffects, hostclaim.Observed, *[]string, *StopRequest) { t.Helper() calls := &[]string{} seen := &StopRequest{} exp := time.Date(2026, 10, 7, 10, 0, 0, 0, time.UTC) e := &RelocationEffects{ Claims: release.NewDirStore(t.TempDir()), Participation: func(context.Context, hostclaim.Observed) (execctl.State, error) { return execctl.State{Binding: execctl.Binding{Host: "shop.example", Incarnation: "inc", Op: "rel-1", Enrollment: "enr", BootID: "boot", ControllerID: "c9"}, LeaseETag: "lease-1", Expires: exp}, nil }, Identity: func(context.Context) (RelocationIdentity, error) { return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 4, Binding: "4"}, BootID: "boot", ControllerID: "ctl", PID: 100, ProcessStart: "5000"}, nil }, Producers: func(context.Context, hostclaim.Observed) (StopDrainer, error) { return recordingDrain{calls}, nil }, WritersStopped: func(_ context.Context, o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { *calls = append(*calls, "writers-stopped") return writersStoppedFor(o) }, Releases: &fakeReleaseObligation{calls: calls}, FlushDB: func(context.Context, hostclaim.Observed) (uint64, error) { *calls = append(*calls, "db") return 41, nil }, FlushFiles: func(context.Context, hostclaim.Observed) (int64, error) { *calls = append(*calls, "files") return 8, nil }, Positions: &memPositions{calls: calls}, Collect: func(ctx context.Context, r StopRequest, d StopDrainer) (StopResult, error) { *seen = r if _, err := d.DrainClaim(ctx, r, "fence"); err != nil { return StopResult{}, err } *calls = append(*calls, "receipt") return StopResult{Key: "control/hosts/shop.example/evidence/rel-1/src/r.json", Digest: strings.Repeat("a", 64)}, nil }, } obs := hostclaim.Observed{Host: "shop.example", ETag: "etag-7", Claim: &hostclaim.Claim{Host: "shop.example", Holder: &hostclaim.Holder{Incarnation: "inc", Stamp: 5, Session: 6, Generation: "g6", WriterBox: "src"}, Op: hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseReleasing, Exec: "src"}}} return e, obs, calls, seen } func TestReleaseStopsWritersAndFlushesBeforeItQueues(t *testing.T) { e, obs, calls, seen := effectsFixture(t) r, err := e.Release(context.Background(), obs) if err != nil { t.Fatal(err) } if want := []string{"writers-stopped", "db", "files", "record", "queue", "producers", "receipt", "complete"}; !slices.Equal(*calls, want) { t.Fatalf("order %v, want %v", *calls, want) } if r.FinalTXID != 41 || r.ManifestSeq != 8 || r.Session != 6 || r.Generation != "g6" || r.ReleasedBy != "src" || r.Receipt == "" { t.Fatalf("release: %+v", r) } if seen.ObligationID != RelocationReleaseObligation || seen.Phase != hostclaim.PhaseReleasing || seen.ExpectedClaimETag != "etag-7" || seen.ExpectedDirectoryRevision != "4" { t.Fatalf("stop request: %+v", *seen) } } func TestRelocationReleaseStopsAtTheFirstFailedStep(t *testing.T) { failure := errors.New("sync failed") for _, tc := range []struct { name string change func(*RelocationEffects) want []string }{ {"the obligation cannot be queued", func(e *RelocationEffects) { e.Releases.(*fakeReleaseObligation).queueErr = failure }, []string{"writers-stopped", "db", "files", "record", "queue"}}, {"the writers cannot be stopped", func(e *RelocationEffects) { e.WritersStopped = func(context.Context, hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { return ClaimWritersDrained{}, WritersStoppedEvidence{}, failure } }, nil}, {"writers still running", func(e *RelocationEffects) { e.WritersStopped = func(_ context.Context, o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { *e.Positions.(*memPositions).calls = append(*e.Positions.(*memPositions).calls, "writers-stopped") d, ev, err := writersStoppedFor(o) d.WorkDrained = false return d, ev, err } }, []string{"writers-stopped"}}, {"root observed another session", func(e *RelocationEffects) { e.WritersStopped = func(_ context.Context, o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { *e.Positions.(*memPositions).calls = append(*e.Positions.(*memPositions).calls, "writers-stopped") d, ev, err := writersStoppedFor(o) ev.WriterSession++ return d, ev, err } }, []string{"writers-stopped"}}, {"the database flush", func(e *RelocationEffects) { e.FlushDB = func(context.Context, hostclaim.Observed) (uint64, error) { return 0, failure } }, []string{"writers-stopped"}}, {"the files flush", func(e *RelocationEffects) { e.FlushFiles = func(context.Context, hostclaim.Observed) (int64, error) { return 0, failure } }, []string{"writers-stopped", "db"}}, {"the receipt does not discharge the obligation", func(e *RelocationEffects) { e.Releases.(*fakeReleaseObligation).completeErr = failure }, []string{"writers-stopped", "db", "files", "record", "queue", "producers", "receipt", "complete"}}, } { t.Run(tc.name, func(t *testing.T) { e, obs, calls, _ := effectsFixture(t) tc.change(e) if _, err := e.Release(context.Background(), obs); err == nil { t.Fatal("released") } if !slices.Equal(*calls, tc.want) { t.Fatalf("ran %v, want %v: nothing after a failed step", *calls, tc.want) } }) } } func TestRelocationReleaseReplaysOnlyRecordedPositions(t *testing.T) { replayed := func(context.Context, StopRequest, StopDrainer) (StopResult, error) { return StopResult{Key: "k", Digest: strings.Repeat("a", 64)}, nil // root replayed; no drain ran here } t.Run("no record", func(t *testing.T) { // Ruling 28: without a recorded flush the release flushes first. // A process with no admitted attempt cannot stop or flush the // writers, so it never reaches the obligation or root. e, obs, calls, _ := effectsFixture(t) e.Collect = replayed e.WritersStopped = func(context.Context, hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { return ClaimWritersDrained{}, WritersStoppedEvidence{}, ErrClaimStopUnavailable } if _, err := e.Release(context.Background(), obs); !errors.Is(err, ErrClaimStopUnavailable) || len(*calls) != 0 { t.Fatalf("got %v, ran %v", err, *calls) } }) t.Run("the lost response", func(t *testing.T) { // The first call flushed and recorded; root completed its receipt; // the response was lost. The retry pairs the replayed receipt with // the recorded positions. e, obs, _, _ := effectsFixture(t) collect := e.Collect e.Collect = func(ctx context.Context, r StopRequest, d StopDrainer) (StopResult, error) { _, err := collect(ctx, r, d) return StopResult{}, errors.Join(err, errors.New("connection reset")) } if _, err := e.Release(context.Background(), obs); err == nil { t.Fatal("the lost response released") } e.Collect = replayed r, err := e.Release(context.Background(), obs) if err != nil || r.FinalTXID != 41 || r.ManifestSeq != 8 { t.Fatalf("replay: %+v %v", r, err) } }) t.Run("an identity without its process", func(t *testing.T) { e, obs, calls, _ := effectsFixture(t) e.Identity = func(context.Context) (RelocationIdentity, error) { return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 4, Binding: "4"}, BootID: "boot", ControllerID: "ctl"}, nil } if _, err := e.Release(context.Background(), obs); !errors.Is(err, ErrClaimStopUnavailable) || len(*calls) != 0 { t.Fatalf("released without a process identity: %v ran %v", err, *calls) } }) t.Run("a record of another session", func(t *testing.T) { e, obs, _, _ := effectsFixture(t) if _, err := e.Release(context.Background(), obs); err != nil { t.Fatal(err) } e.Collect = replayed obs.Claim.Holder.Session = 7 *e.Positions.(*memPositions).calls = nil if _, err := e.Release(context.Background(), obs); err != nil || !slices.Contains(*e.Positions.(*memPositions).calls, "writers-stopped") { t.Fatalf("another session's positions were used: %v ran %v", err, *e.Positions.(*memPositions).calls) } }) } // Review Focus 3: a release never returns before root's receipt. Root // fails after the producers closed (kernel verification or CompleteFence), // so no receipt is durable: the release returns an error with its // positions recorded, and the retry, same controller, no restart, resumes // at the producers' drain with the recorded positions. No step of the // release runs twice. func TestReleaseNeverReturnsBeforeRootsReceipt(t *testing.T) { // Round 7: the drain flushed, recorded and closed the producers; // root then failed before its receipt was durable (kernel // verification or CompleteFence). The retry, same controller, no // restart, resumes at the producers' drain with the recorded // positions: no step of the release runs again. e, obs, calls, seen := effectsFixture(t) collect := e.Collect e.Collect = func(ctx context.Context, r StopRequest, d StopDrainer) (StopResult, error) { if _, err := d.DrainClaim(ctx, r, "fence"); err != nil { return StopResult{}, err } return StopResult{}, errors.New("verify stopped: cgroup not yet empty") // no receipt persisted } if _, err := e.Release(context.Background(), obs); err == nil { t.Fatal("a release without a receipt succeeded") } if want := []string{"writers-stopped", "db", "files", "record", "queue", "producers"}; !slices.Equal(*calls, want) { t.Fatalf("the first call ran %v, want %v", *calls, want) } *calls = nil e.Collect = collect // Round 8 finding 2: the roster moved on between the attempts // (same boot, same controller); the retry still names the // revision the stop was flushed under. e.Identity = func(context.Context) (RelocationIdentity, error) { return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 5, Binding: "5"}, BootID: "boot", ControllerID: "ctl", PID: 100, ProcessStart: "5000"}, nil } r, err := e.Release(context.Background(), obs) if err != nil || r.FinalTXID != 41 || r.ManifestSeq != 8 { t.Fatalf("retry: %+v %v", r, err) } if want := []string{"queue", "producers", "receipt", "complete"}; !slices.Equal(*calls, want) { t.Fatalf("the retry ran %v, want %v (no flush, no record, no writers stop again)", *calls, want) } if seen.ExpectedDirectoryRevision != "4" || seen.BootID != "boot" || seen.ControllerID != "ctl" { t.Fatalf("the retry asked root under another stop: %+v", *seen) } if p := e.Positions.(*memPositions); !p.ok || p.p.TXID != 41 || p.p.Seq != 8 { t.Fatalf("the positions were not kept for the retry: %+v", p) } } // A restarted agent asks root as the recorded owner's recovery: the // request names the boot and controller the flush was made under, offers // no drain, and pairs root's replayed receipt with the recorded positions. // Both restarts count: new ids, and the same ids in a new process (main's // claimLaunchOwner.local). func TestReleaseAfterARestartRecoversTheRecordedOwner(t *testing.T) { t.Run("after a restart the recorded identity asks root", func(t *testing.T) { // The agent flushed and recorded under controller "ctl", root // completed its receipt, the agent restarted. The new process asks // root under the recorded boot and controller (so root finds that // stop's durable receipt), offers no drain of its own, and pairs // the replayed receipt with the recorded positions. e, obs, _, _ := effectsFixture(t) if _, err := e.Release(context.Background(), obs); err != nil { t.Fatal(err) } e.Identity = func(context.Context) (RelocationIdentity, error) { return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 5, Binding: "5"}, BootID: "boot-2", ControllerID: "ctl-2", PID: 200, ProcessStart: "9000"}, nil } var asked StopRequest recovery := false e.Collect = func(_ context.Context, r StopRequest, d StopDrainer) (StopResult, error) { asked = r _, recovery = d.(RecoveryReleaseDrain) // Task 8a sets req.Recover from this if _, err := d.DrainClaim(context.Background(), r, "fence"); !errors.Is(err, ErrClaimStopIdentity) { return StopResult{}, errors.New("a restarted agent offered a drain for another process's stop") } return StopResult{Key: "k", Digest: strings.Repeat("a", 64)}, nil } r, err := e.Release(context.Background(), obs) if err != nil || r.FinalTXID != 41 || r.ManifestSeq != 8 { t.Fatalf("after restart: %+v %v", r, err) } if asked.BootID != "boot" || asked.ControllerID != "ctl" || asked.ExpectedDirectoryRevision != "4" || !recovery { t.Fatalf("root was not asked as the recorded owner's recovery: %+v recovery=%v", asked, recovery) } }) t.Run("a restarted process with the same boot and controller ids recovers", func(t *testing.T) { // Fix round 1 (I-2), main's claimLaunchOwner.local: the same boot // and controller ids in a restarted process are not this process. // Its producers ended with the old one, so the request is a // recovery under the recorded owner and no drain is offered. e, obs, calls, _ := effectsFixture(t) if _, err := e.Release(context.Background(), obs); err != nil { t.Fatal(err) } *calls = nil e.Identity = func(context.Context) (RelocationIdentity, error) { return RelocationIdentity{Actor: hostbox.Actor{Box: "src", Enrollment: "enr", Revision: 4, Binding: "4"}, BootID: "boot", ControllerID: "ctl", PID: 101, ProcessStart: "6000"}, nil } var asked StopRequest recovery := false e.Collect = func(_ context.Context, r StopRequest, d StopDrainer) (StopResult, error) { asked = r _, recovery = d.(RecoveryReleaseDrain) if _, err := d.DrainClaim(context.Background(), r, "fence"); !errors.Is(err, ErrClaimStopIdentity) { return StopResult{}, errors.New("a restarted process offered its producers for the old process's stop") } return StopResult{Key: "k", Digest: strings.Repeat("a", 64)}, nil } r, err := e.Release(context.Background(), obs) if err != nil || r.FinalTXID != 41 || r.ManifestSeq != 8 { t.Fatalf("same ids, new process: %+v %v", r, err) } if !recovery || asked.BootID != "boot" || asked.ControllerID != "ctl" || !slices.Equal(*calls, []string{"queue", "complete"}) { t.Fatalf("not a recovery: recovery=%v asked=%+v ran %v", recovery, asked, *calls) } }) } func TestReleasePositionsFileBindsTheExactStop(t *testing.T) { ctx := context.Background() f := ReleasePositionsFile{Dir: filepath.Join(t.TempDir(), "relocate")} r := StopRequest{Host: "shop.example", Incarnation: "inc", InstallationStamp: 5, WriterSession: 6, OperationID: "rel-1", Phase: "releasing", ObligationID: RelocationReleaseObligation, Box: "src", Enrollment: "enr", BootID: "boot", ControllerID: "ctl", ExpectedClaimETag: "e1", ExpectedDirectoryRevision: "4"} if _, ok, err := f.Recorded(ctx, r); ok || err != nil { t.Fatalf("before any record: %v %v", ok, err) } if err := f.Record(ctx, ReleasePosition{Request: r, TXID: 41, Seq: 8}); !errors.Is(err, ErrClaimStopUnavailable) { t.Fatalf("a record without its process: %v", err) } if err := f.Record(ctx, ReleasePosition{Request: r, PID: 100, ProcessStart: "5000", TXID: 41, Seq: 8}); err != nil { t.Fatal(err) } moved := r moved.ExpectedClaimETag, moved.ExpectedDirectoryRevision = "e2", "5" // a target replacement moved the claim if rec, ok, err := f.Recorded(ctx, moved); !ok || err != nil || rec.TXID != 41 || rec.Seq != 8 || rec.Request.ControllerID != "ctl" || rec.PID != 100 || rec.ProcessStart != "5000" { t.Fatalf("recorded: %+v %v %v", rec, ok, err) } // A restarted controller of the same installation finds the record, // and gets the identity the flush was made under. restarted := r restarted.BootID, restarted.ControllerID = "boot-2", "ctl-2" if rec, ok, err := f.Recorded(ctx, restarted); !ok || err != nil || rec.Request.BootID != "boot" || rec.Request.ControllerID != "ctl" { t.Fatalf("after a restart: %+v %v %v", rec, ok, err) } for name, change := range map[string]func(*StopRequest){ "session": func(r *StopRequest) { r.WriterSession = 7 }, "stamp": func(r *StopRequest) { r.InstallationStamp = 6 }, "operation": func(r *StopRequest) { r.OperationID = "rel-2" }, "phase": func(r *StopRequest) { r.Phase = "probe-failed" }, "box": func(r *StopRequest) { r.Box = "dst" }, } { other := r change(&other) if _, ok, _ := f.Recorded(ctx, other); ok { t.Fatalf("%s: another stop's positions were returned", name) } } if err := os.WriteFile(filepath.Join(f.Dir, "rel-1.json"), []byte("{"), 0o600); err != nil { t.Fatal(err) } if _, _, err := f.Recorded(ctx, r); !errors.Is(err, ErrClaimStopReceipt) { t.Fatalf("a damaged record: %v", err) } // A record that names no process would read as this process's own // under claimLaunchOwner.local: it is damaged, never trusted. if err := os.WriteFile(filepath.Join(f.Dir, "rel-1.json"), []byte(`{"v":1,"request":{"Host":"shop.example"},"txid":41,"seq":8}`), 0o600); err != nil { t.Fatal(err) } if _, _, err := f.Recorded(ctx, r); !errors.Is(err, ErrClaimStopReceipt) { t.Fatalf("a record without its process: %v", err) } } // Round 14 finding 2: EnsureLocal verifies the receipt's install, restores // a retired one under the same stamp and verifies it again, refuses a // restore that is not the receipt's, and never touches an opened P. func TestEnsureLocalRepairsOnlyUnderTheActiveStampBeforePOpens(t *testing.T) { ctx := context.Background() e, obs, _, _ := effectsFixture(t) receipt := hostclaim.LocalInstall{Box: "dst", Enrollment: "enr-d", InstallStamp: 9, FilesManifestVersion: 1, JournalSHA256: strings.Repeat("e", 64)} raw, _ := json.Marshal(receipt) obs.Claim.Op.Phase = hostclaim.PhaseRelocatingProbing obs.Claim.Op.Journal = map[string]string{"local_install": string(raw), "prepared": "P1", "install_stamp": "9"} present, installs := true, 0 e.VerifyLocal = func(context.Context, hostclaim.Observed, hostclaim.LocalInstall) (bool, error) { return present, nil } restored := receipt e.Install = func(context.Context, hostclaim.Observed) (hostclaim.LocalInstall, error) { installs++ present = true return restored, nil } if err := e.EnsureLocal(ctx, obs); err != nil || installs != 0 { t.Fatalf("a complete install: %v installs=%d", err, installs) } present = false if err := e.EnsureLocal(ctx, obs); err != nil || installs != 1 || !present { t.Fatalf("a damaged install under its active stamp: %v installs=%d", err, installs) } present, restored.InstallStamp = false, 10 if err := e.EnsureLocal(ctx, obs); !errors.Is(err, ErrClaimStopReceipt) { t.Fatalf("a restore under another stamp: %v", err) } present, installs = false, 0 obs.Claim.Op.Journal["prepared_opened"] = "P1" if err := e.EnsureLocal(ctx, obs); err != nil || installs != 0 { t.Fatalf("an opened P was reinstalled: %v installs=%d", err, installs) } } // Round 15 finding 1: after P opens, VerifyReceipt compares the box's // durable receipt with the journaled one and nothing else: a missing or // different receipt is false (the participant marks the operation stuck); // changed files never trigger a restore. func TestVerifyReceiptChecksTheReceiptOnly(t *testing.T) { ctx := context.Background() e, obs, _, _ := effectsFixture(t) receipt := hostclaim.LocalInstall{Box: "dst", Enrollment: "enr-d", InstallStamp: 9, FilesManifestVersion: 1, JournalSHA256: strings.Repeat("e", 64)} raw, _ := json.Marshal(receipt) obs.Claim.Op.Journal = map[string]string{"local_install": string(raw), "prepared": "P1", "prepared_opened": "P1", "install_stamp": "9"} stored, present := receipt, true e.Receipt = func(context.Context, hostclaim.Observed) (hostclaim.LocalInstall, bool, error) { return stored, present, nil } touched := 0 e.VerifyLocal = func(context.Context, hostclaim.Observed, hostclaim.LocalInstall) (bool, error) { touched++ return false, nil } e.Install = func(context.Context, hostclaim.Observed) (hostclaim.LocalInstall, error) { touched++ return receipt, nil } if ok, err := e.VerifyReceipt(ctx, obs); !ok || err != nil || touched != 0 { t.Fatalf("an intact receipt over changed files: %v %v touched=%d", ok, err, touched) } present = false if ok, err := e.VerifyReceipt(ctx, obs); ok || err != nil { t.Fatalf("a missing receipt: %v %v", ok, err) } present, stored.InstallStamp = true, 8 if ok, err := e.VerifyReceipt(ctx, obs); ok || err != nil { t.Fatalf("another installation's receipt: %v %v", ok, err) } }