package host import ( "context" "crypto/ed25519" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostevidence" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/release" ) // releaseWorld is a relocating claim in releasing on the source "box", // a root-signed roster naming the box's census key, and the box's key to // sign root's receipt with (main's plan11_release_test fixture). type releaseWorld struct { s release.Store dir hostactors.Directory boxKey ed25519.PrivateKey en string obs hostclaim.Observed } func newReleaseWorld(t *testing.T) releaseWorld { t.Helper() return newReleaseWorldWith(t, nil) } // newReleaseWorldWith is newReleaseWorld with the roster's boxes changed // by roster before it is signed (nil leaves them). func newReleaseWorldWith(t *testing.T, roster func(map[string]hostactors.Box)) releaseWorld { t.Helper() ctx := context.Background() s := release.NewDirStore(t.TempDir()) root := ed25519.NewKeyFromSeed(make([]byte, 32)) boxKey := ed25519.NewKeyFromSeed([]byte(strings.Repeat("b", 32))) en := strings.Repeat("a", 32) box := func(name, enrollment string, key ed25519.PrivateKey) hostactors.Box { return hostactors.Box{Enrollment: enrollment, BoxID: name, Label: name, Namespace: "platform", Region: "ie", Pool: "default", State: "active", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-" + name}, EvidenceSHA256: strings.Repeat("d", 64), CensusPublicKey: key.Public().(ed25519.PublicKey)} } boxes := map[string]hostactors.Box{"box": box("box", en, boxKey), "other": box("other", strings.Repeat("c", 32), otherBoxKey)} if roster != nil { roster(boxes) } signed := hostactors.Roster{V: 1, Revision: 1, DeploymentBucket: "bucket", Boxes: boxes, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} wire, err := release.SignRootEnvelope(release.RootSigner(root), signed) if err != nil { t.Fatal(err) } if _, err := s.Put(ctx, hostactors.Key, wire, ""); err != nil { t.Fatal(err) } host := "shop.example" created := time.Date(2026, 10, 9, 0, 0, 0, 0, time.UTC) inc, _ := hostclaim.Incarnation("bab", "jam", host, created) claim := hostclaim.Claim{V: 2, Gen: 1, Host: host, Holder: &hostclaim.Holder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", Incarnation: inc, CreatedAt: created, Location: "instances/" + host + "/" + inc, Stamp: 7, Session: 7, WriterBox: "box"}, Op: hostclaim.Op{ID: "reloc", Kind: hostclaim.KindRelocating, Phase: hostclaim.PhaseReleasing, Exec: "box", Intent: map[string]string{"source_box": "box", "source_enrollment": en, "region": "ie"}}} raw, _ := json.Marshal(claim) key, _ := hostclaim.Key(host) tag, err := s.Put(ctx, key, raw, "") if err != nil { t.Fatal(err) } return releaseWorld{s: s, dir: hostactors.Directory{Store: s, PublicKey: root.Public().(ed25519.PublicKey), Bucket: "bucket"}, boxKey: boxKey, en: en, obs: hostclaim.Observed{Host: host, Claim: &claim, ETag: tag}} } // otherBoxKey is another roster box's census key. var otherBoxKey = ed25519.NewKeyFromSeed([]byte(strings.Repeat("o", 32))) // receipt publishes root's signed full stop of the claim's session. func (w releaseWorld) receipt(t *testing.T) (string, string) { t.Helper() return w.receiptWith(t, nil, w.boxKey) } // receiptWith publishes the receipt changed by mutate, signed by signer // (nil: the body alone, unsigned). func (w releaseWorld) receiptWith(t *testing.T, mutate func(*StopReceipt), signer ed25519.PrivateKey) (string, string) { t.Helper() c := w.obs.Claim r := StopReceipt{V: 1, ID: "stop-1", Kind: "stopped", Host: c.Host, Incarnation: c.Holder.Incarnation, InstallationStamp: c.Holder.Stamp, WriterSession: c.Holder.Session, OperationID: c.Op.ID, Phase: hostclaim.PhaseReleasing, ObligationID: RelocationReleaseObligation, Box: "box", Enrollment: w.en, Region: "ie", EvidenceAt: time.Now(), RequestLimitNS: int64(hostbox.RequestLimit), ClaimGen: 1, FenceID: "fence", InventorySHA256: strings.Repeat("e", 64), LocalJournalSHA256: strings.Repeat("f", 64), ProducerDrainDigest: strings.Repeat("a", 64), CgroupIdentities: []string{"root-role"}} if mutate != nil { mutate(&r) } body, _ := json.Marshal(r) key := stopEvidenceKey(c.Host, c.Op.ID, "box", r.ID) if signer == nil { if _, err := w.s.Put(context.Background(), key, body, ""); err != nil { t.Fatal(err) } } else if err := hostevidence.Publish(context.Background(), w.s, signer, key, body); err != nil { t.Fatal(err) } sum := sha256.Sum256(body) return key, hex.EncodeToString(sum[:]) } // A release runs main's relocation-release lifecycle: the obligation is // queued once before root's stop, root's exact receipt discharges it, and // only then does the journaled release count for the pin. func TestReleaseObligationQueuesOnceAndDischargesWithRootsReceipt(t *testing.T) { ctx := context.Background() w := newReleaseWorld(t) o := ReleaseObligation{Receipts: RelocationReleaseReceipts{Store: w.s, Directory: w.dir}} queued, err := o.Queue(ctx, w.obs) if err != nil { t.Fatal(err) } if len(queued.Claim.Obligations) != 1 || queued.Claim.Obligations[0].Kind != RelocationReleaseObligation || queued.Claim.Obligations[0].Retirement != "reloc/7/7" { t.Fatalf("queued: %+v", queued.Claim.Obligations) } again, err := o.Queue(ctx, queued) if err != nil || again.ETag != queued.ETag { t.Fatalf("a second queue wrote the claim: %v", err) } key, digest := w.receipt(t) rel := hostclaim.Release{ReleasedBy: "box", Enrollment: w.en, Generation: "", Session: 7, Receipt: key, ReceiptSHA256: digest} if err := (StopReceiptReleases{Receipts: o.Receipts}).VerifyRelease(ctx, *queued.Claim, rel); !errors.Is(err, ErrClaimStopReceipt) { t.Fatalf("a release counted before its receipt discharged the obligation: %v", err) } done, err := o.Complete(ctx, queued, key, digest) if err != nil { t.Fatal(err) } if len(done.Claim.Obligations) != 0 || done.Claim.Op.Journal["release_stop_receipt"] != key { t.Fatalf("not discharged: %+v %v", done.Claim.Obligations, done.Claim.Op.Journal) } if err := (StopReceiptReleases{Receipts: o.Receipts}).VerifyRelease(ctx, *done.Claim, rel); err != nil { t.Fatalf("the discharged release: %v", err) } // A replay: the discharged release queues nothing and completes again. if replayed, err := o.Queue(ctx, done); err != nil || replayed.ETag != done.ETag { t.Fatalf("a replay queued again: %v", err) } if _, err := o.Complete(ctx, done, key, digest); err != nil { t.Fatalf("a replayed completion: %v", err) } for name, change := range map[string]func(*hostclaim.Release){ "another receipt": func(r *hostclaim.Release) { r.Receipt = key + "x" }, "another digest": func(r *hostclaim.Release) { r.ReceiptSHA256 = strings.Repeat("0", 64) }, "no receipt": func(r *hostclaim.Release) { r.Receipt = "" }, } { other := rel change(&other) if err := (StopReceiptReleases{Receipts: o.Receipts}).VerifyRelease(ctx, *done.Claim, other); !errors.Is(err, ErrClaimStopReceipt) { t.Fatalf("%s counted: %v", name, err) } } // Main discharges a relocation release only for the source in // releasing: a failed target's release in probe-failed never counts. failed := *done.Claim failed.Op.Phase = hostclaim.PhaseProbeFailed if err := (StopReceiptReleases{Receipts: o.Receipts}).VerifyRelease(ctx, failed, rel); !errors.Is(err, ErrClaimStopReceipt) { t.Fatalf("a probe-failed release counted: %v", err) } } func TestReleaseObligationRefusesAnotherRelease(t *testing.T) { ctx := context.Background() w := newReleaseWorld(t) o := ReleaseObligation{Receipts: RelocationReleaseReceipts{Store: w.s, Directory: w.dir}} queued, err := o.Queue(ctx, w.obs) if err != nil { t.Fatal(err) } // The same operation's release of a later session is another // retirement identity: the owed obligation is not it. later := *queued.Claim h := *later.Holder h.Session++ later.Holder = &h if _, err := o.Queue(ctx, hostclaim.Observed{Host: queued.Host, Claim: &later, ETag: queued.ETag}); !errors.Is(err, hostclaim.ErrWrongState) { t.Fatalf("another session's release reused the obligation: %v", err) } // probe-failed has no relocation-release in main's claim. w2 := newReleaseWorld(t) w2.obs.Claim.Op.Phase = hostclaim.PhaseProbeFailed if _, err := (ReleaseObligation{Receipts: RelocationReleaseReceipts{Store: w2.s, Directory: w2.dir}}).Queue(ctx, w2.obs); !errors.Is(err, hostclaim.ErrWrongState) { t.Fatalf("a probe-failed target queued a relocation release: %v", err) } if _, err := (ReleaseObligation{}).Queue(ctx, w.obs); !errors.Is(err, ErrClaimStopUnavailable) { t.Fatalf("no store: %v", err) } } // discharged is the world's claim as Complete leaves it: the receipt's key // and digest journaled and no obligation owed. func (w releaseWorld) discharged(key, digest string) hostclaim.Claim { c := *w.obs.Claim c.Op.Journal = map[string]string{"release_stop_receipt": key, "release_stop_sha256": digest} c.Obligations = nil return c } // The pin's verifier consumes root's signed receipt itself (Plan 11 Task // E), never the journaled digest alone: every receipt main's Complete // would refuse is refused at the pin, though the claim journals it as // discharged. (The deleted branch verifier's signature tests, against // main's receipt.) func TestStopReceiptReleasesVerifiesRootsSignedReceiptAtThePin(t *testing.T) { ctx := context.Background() verify := func(t *testing.T, w releaseWorld, key, digest string, dir hostactors.Directory) error { t.Helper() rel := hostclaim.Release{ReleasedBy: "box", Enrollment: w.en, Session: 7, Receipt: key, ReceiptSHA256: digest} return (StopReceiptReleases{Receipts: RelocationReleaseReceipts{Store: w.s, Directory: dir}}).VerifyRelease(ctx, w.discharged(key, digest), rel) } w := newReleaseWorld(t) key, digest := w.receipt(t) if err := verify(t, w, key, digest, w.dir); err != nil { t.Fatalf("root's signed receipt: %v", err) } if now, err := hostclaim.Read(ctx, w.s, w.obs.Host); err != nil || now.ETag != w.obs.ETag { t.Fatalf("verifying wrote the claim: %v", err) } t.Run("an unsigned receipt", func(t *testing.T) { w := newReleaseWorld(t) key, digest := w.receiptWith(t, nil, nil) if err := verify(t, w, key, digest, w.dir); err == nil { t.Fatal("an unsigned receipt counted") } }) t.Run("signed by another box's key", func(t *testing.T) { w := newReleaseWorld(t) key, digest := w.receiptWith(t, nil, otherBoxKey) if err := verify(t, w, key, digest, w.dir); err == nil { t.Fatal("another box's signature counted") } }) t.Run("a roster entry of another enrollment", func(t *testing.T) { w := newReleaseWorldWith(t, func(b map[string]hostactors.Box) { box := b["box"] box.Enrollment = strings.Repeat("d", 32) b["box"] = box }) key, digest := w.receipt(t) if err := verify(t, w, key, digest, w.dir); !errors.Is(err, ErrClaimStopIdentity) { t.Fatalf("another enrollment's key counted: %v", err) } }) t.Run("a box the roster does not name", func(t *testing.T) { w := newReleaseWorldWith(t, func(b map[string]hostactors.Box) { delete(b, "box") }) key, digest := w.receipt(t) if err := verify(t, w, key, digest, w.dir); !errors.Is(err, ErrClaimStopIdentity) { t.Fatalf("a box missing from the roster counted: %v", err) } }) t.Run("no roster", func(t *testing.T) { w := newReleaseWorld(t) key, digest := w.receipt(t) if err := verify(t, w, key, digest, hostactors.Directory{Store: w.s, Bucket: "bucket"}); err == nil { t.Fatal("a release counted without a roster to name its key") } }) t.Run("a digest that is not the bytes", func(t *testing.T) { w := newReleaseWorld(t) key, _ := w.receipt(t) if err := verify(t, w, key, strings.Repeat("0", 64), w.dir); !errors.Is(err, ErrClaimStopReceipt) { t.Fatalf("another digest counted: %v", err) } }) for name, mutate := range map[string]func(*StopReceipt){ "another session": func(r *StopReceipt) { r.WriterSession++ }, "another installation stamp": func(r *StopReceipt) { r.InstallationStamp++ }, "another operation's phase": func(r *StopReceipt) { r.Phase = hostclaim.PhaseProbeFailed }, "a retirement's stop": func(r *StopReceipt) { r.ObligationID = "park-local" }, "another enrollment": func(r *StopReceipt) { r.Enrollment = strings.Repeat("f", 32) }, "no stopped cgroups": func(r *StopReceipt) { r.CgroupIdentities = nil }, "a malformed producer digest": func(r *StopReceipt) { r.ProducerDrainDigest = "drain" }, } { t.Run(name, func(t *testing.T) { w := newReleaseWorld(t) key, digest := w.receiptWith(t, mutate, w.boxKey) if err := verify(t, w, key, digest, w.dir); !errors.Is(err, ErrClaimStopReceipt) { t.Fatalf("accepted: %v", err) } }) } t.Run("no receipts store", func(t *testing.T) { w := newReleaseWorld(t) key, digest := w.receipt(t) rel := hostclaim.Release{ReleasedBy: "box", Enrollment: w.en, Session: 7, Receipt: key, ReceiptSHA256: digest} if err := (StopReceiptReleases{}).VerifyRelease(ctx, w.discharged(key, digest), rel); !errors.Is(err, ErrClaimStopUnavailable) { t.Fatalf("verified with no store: %v", err) } }) } // rootCollect models root's FenceAndStop for a relocation release: it // refuses unless the claim in the store owes the obligation at exactly the // request's ETag (hostguard.StopAuthorityMatches, the expected ETag), runs // the agent's drain, then publishes its signed receipt. calls counts the // requests that reached it. func (w releaseWorld) rootCollect(t *testing.T, calls *int) func(context.Context, StopRequest, StopDrainer) (StopResult, error) { return func(ctx context.Context, r StopRequest, d StopDrainer) (StopResult, error) { *calls++ o, err := hostclaim.Read(ctx, w.s, r.Host) if err != nil { return StopResult{}, err } owed := false for _, ob := range o.Claim.Obligations { owed = owed || (ob.Kind == RelocationReleaseObligation && ob.Op == r.OperationID && ob.Box == r.Box) } if !owed || o.ETag != r.ExpectedClaimETag { return StopResult{}, ErrClaimStopIdentity } if _, err := d.DrainClaim(ctx, r, "fence"); err != nil { return StopResult{}, err } key, digest := w.receipt(t) return StopResult{Key: key, Digest: digest}, nil } } // releaseEffects is one agent process's RelocationEffects over the world: // the real obligation (main's Queue and Complete), durable positions in // dir, root modelled by rootCollect, and this process's identity. func (w releaseWorld) releaseEffects(t *testing.T, dir string, pid int, calls *int) *RelocationEffects { steps := &[]string{} return &RelocationEffects{ Identity: func(context.Context) (RelocationIdentity, error) { return RelocationIdentity{Actor: hostbox.Actor{Box: "box", Enrollment: w.en, Region: "ie", Revision: 1, Binding: "1"}, BootID: "boot", ControllerID: fmt.Sprint("ctl-", pid), PID: pid, ProcessStart: fmt.Sprint(pid * 10)}, nil }, Producers: func(context.Context, hostclaim.Observed) (StopDrainer, error) { return recordingDrain{steps}, nil }, Collect: w.rootCollect(t, calls), WritersStopped: func(_ context.Context, o hostclaim.Observed) (ClaimWritersDrained, WritersStoppedEvidence, error) { h := o.Claim.Holder at := time.Date(2026, 10, 9, 1, 0, 0, 0, time.UTC) target := hostguard.FenceTarget{Host: o.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: "box", Enrollment: w.en, BootID: "boot", ControllerID: fmt.Sprint("ctl-", pid)} return ClaimWritersDrained{Target: target, Generation: h.Generation, WriterBox: h.WriterBox, FenceClosed: true, LaunchJoined: true, WorkDrained: true, SpawnsSettled: true, CgroupEmpty: true, Cgroup: "carlos/shop", DBPaths: []string{"/data/shop.db"}, CheckedAt: at}, WritersStoppedEvidence{Host: o.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, WriterSession: h.Session, Box: "box", Enrollment: w.en, CheckedAt: at}, nil }, Releases: ReleaseObligation{Receipts: RelocationReleaseReceipts{Store: w.s, Directory: w.dir}}, FlushDB: func(context.Context, hostclaim.Observed) (uint64, error) { return 41, nil }, FlushFiles: func(context.Context, hostclaim.Observed) (int64, error) { return 8, nil }, Positions: ReleasePositionsFile{Dir: dir}, } } // Review finding 4: once root's receipt has discharged the obligation the // claim has moved on and owes root nothing, so a second stop is refused. // A release whose journal CAS was lost, or whose agent restarted after // Complete, must still be returned: from the journaled receipt, verified // again, and the recorded positions, never by asking root. func TestADischargedReleaseReplaysAfterALostJournalOrARestart(t *testing.T) { ctx := context.Background() w := newReleaseWorld(t) dir := filepath.Join(t.TempDir(), "relocate") calls := 0 first, err := w.releaseEffects(t, dir, 100, &calls).Release(ctx, w.obs) if err != nil { t.Fatal(err) } if calls != 1 || first.FinalTXID != 41 || first.ManifestSeq != 8 { t.Fatalf("first release: %+v, root asked %d times", first, calls) } fresh := func() hostclaim.Observed { o, err := hostclaim.Read(ctx, w.s, w.obs.Host) if err != nil { t.Fatal(err) } return o } discharged := fresh() if len(discharged.Claim.Obligations) != 0 || discharged.Claim.Op.Journal["release_stop_receipt"] != first.Receipt { t.Fatalf("not discharged: %+v", discharged.Claim) } // The journal CAS lost: the same process retries at the claim as it is. again, err := w.releaseEffects(t, dir, 100, &calls).Release(ctx, discharged) if err != nil || again != first { t.Fatalf("same process retry: %+v %v", again, err) } // The agent restarted after Complete: a new process, new controller. restarted, err := w.releaseEffects(t, dir, 200, &calls).Release(ctx, fresh()) if err != nil || restarted != first { t.Fatalf("after a restart: %+v %v", restarted, err) } if calls != 1 { t.Fatalf("root was asked again for a discharged release: %d calls", calls) } // The release journals, and nothing was written by the replays. if now := fresh(); now.ETag != discharged.ETag { t.Fatal("a replay wrote the claim") } if _, err := hostclaim.Update(ctx, w.s, fresh(), hostclaim.JournalRelease("reloc", restarted)); err != nil { t.Fatalf("the replayed release does not journal: %v", err) } // Without a recorded flush the replay refuses rather than guess. w2 := newReleaseWorld(t) calls2 := 0 if _, err := w2.releaseEffects(t, filepath.Join(t.TempDir(), "relocate"), 100, &calls2).Release(ctx, w2.obs); err != nil { t.Fatal(err) } o2, err := hostclaim.Read(ctx, w2.s, w2.obs.Host) if err != nil { t.Fatal(err) } if _, err := w2.releaseEffects(t, filepath.Join(t.TempDir(), "elsewhere"), 300, &calls2).Release(ctx, o2); !errors.Is(err, ErrClaimStopReceipt) || calls2 != 1 { t.Fatalf("a discharged release with no recorded positions: %v, root asked %d times", err, calls2) } }