package hostops import ( "context" "encoding/json" "errors" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/termfence" ) const completionHost = "shop.example" // completionWorld is ruling 86's schedule (the two-box model, seed 2224): // a relocation completed on box-a, which is terminated before it // discharges, so the serving op's completed relocation stays undischarged // and box-a is not on lost. The signed roster holds box-a and the worker // box-w that writes box-a's fence; the authority is production's // (ProductionAuthority over that roster, ClaimFenceAuthority). type completionWorld struct { s release.Store a ProductionAuthority } func newCompletionWorld(t *testing.T, change func(*hostclaim.Claim)) *completionWorld { t.Helper() ctx := context.Background() x, s, _, _ := chooserFixture(t, "box-a", "box-w") w := &completionWorld{s: s, a: ProductionAuthority{Store: s, Directory: x.Directory, Namespace: "platform", Now: x.Now}} obs, err := hostclaim.Read(ctx, s, completionHost) if err != nil { t.Fatal(err) } obs, err = hostclaim.Update(ctx, s, obs, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: "apps/acct/app/instances/" + completionHost + ".json", Account: "acct", App: "app", Host: completionHost, Region: "ie"}, "box-a", nil, nil, "adopt", func() string { return "id" })) if err != nil { t.Fatal(err) } done, err := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: "box-a", Session: 7, Receipt: "probe-fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, Phase: hostclaim.PhaseRelocatingProbing}) if err != nil { t.Fatal(err) } if _, err = hostclaim.Update(ctx, s, obs, func(c *hostclaim.Claim) error { c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box-a", Participants: []string{"box-a"}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}} if change != nil { change(c) } return nil }); err != nil { t.Fatal(err) } return w } // fence writes raw at box-a's fence key (its roster enrollment). func (w *completionWorld) fence(t *testing.T, raw []byte) { t.Helper() if _, err := w.s.Put(context.Background(), "control/box-fences/"+strings.Repeat("a", 32)+".json", raw, ""); err != nil { t.Fatal(err) } } // terminated is box-w's termination fence of box-a, valid under the // roster; edit changes it before it is written. func (w *completionWorld) terminated(t *testing.T, edit func(*termfence.Record)) { t.Helper() f := termfence.Record{V: 1, Box: "box-a", Enrollment: strings.Repeat("a", 32), State: "terminated", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-box-a"}, ObservedAt: "2026-10-07T08:00:00Z", Via: termfence.ViaDescribe, RequestID: "req", WrittenBy: termfence.WrittenBy{Box: "box-w", Enrollment: strings.Repeat("b", 32)}} if edit != nil { edit(&f) } raw, err := json.Marshal(f) if err != nil { t.Fatal(err) } w.fence(t, raw) } // purge is the purge start as startAppLifecycle makes it: the verified // completion fence, then BeginPurge's CAS with it. delete is the same // for StartDeleteClaim and BeginDelete. func (w *completionWorld) purge(t *testing.T) (hostclaim.Observed, error) { t.Helper() ctx := context.Background() obs, err := hostclaim.Read(ctx, w.s, completionHost) if err != nil { t.Fatal(err) } fenced, err := w.a.completionFence(ctx, obs.Claim) if err != nil { return obs, err } return hostclaim.Update(ctx, w.s, obs, BeginPurge("purge-1", "console", time.Unix(200, 0), []string{"ie"}, []string{"box-a", "box-w"}, fenced)) } func (w *completionWorld) delete(t *testing.T) (hostclaim.Observed, error) { t.Helper() ctx := context.Background() obs, err := hostclaim.Read(ctx, w.s, completionHost) if err != nil { t.Fatal(err) } fenced, err := w.a.completionFence(ctx, obs.Claim) if err != nil { return obs, err } return hostclaim.Update(ctx, w.s, obs, hostclaim.BeginDelete("delete-1", "console", "record-etag", time.Unix(200, 0), []string{"ie"}, []string{"box-a", "box-w"}, fenced)) } func (w *completionWorld) unchanged(t *testing.T, what string) { t.Helper() obs, err := hostclaim.Read(context.Background(), w.s, completionHost) if err != nil { t.Fatal(err) } if obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.ID != "" { t.Fatalf("%s: the claim was started: %+v", what, obs.Claim.Op) } } // Ruling 86, the deadlock schedule: the completion box is terminated before // it discharged, and a purge (or a delete) starts. With box-a's termination // fence verified under the roster, the start passes the completion barrier // as a failover's would; without the exemption StartGuard's barrier // refuses it, which is the deadlock the model found. func TestPurgeStartsOverATerminatedCompletionBox(t *testing.T) { w := newCompletionWorld(t, nil) w.terminated(t, nil) ctx := context.Background() obs, err := hostclaim.Read(ctx, w.s, completionHost) if err != nil { t.Fatal(err) } if _, err := hostclaim.Update(ctx, w.s, obs, BeginPurge("purge-1", "console", time.Unix(200, 0), []string{"ie"}, []string{"box-a"}, "")); !errors.Is(err, hostclaim.ErrRelocationFinishing) { t.Fatalf("the barrier without the exemption: %v (the deadlock's refusal)", err) } if fenced, err := w.a.completionFence(ctx, obs.Claim); err != nil || fenced != "box-a" { t.Fatalf("the verified completion fence: %q %v", fenced, err) } started, err := w.purge(t) if err != nil || started.Claim.Op.Kind != hostclaim.KindPurging || started.Claim.Op.ID != "purge-1" { t.Fatalf("the purge over a terminated completion box: %v %+v", err, started.Claim) } d := newCompletionWorld(t, nil) d.terminated(t, nil) if started, err := d.delete(t); err != nil || started.Claim.Op.Kind != hostclaim.KindDeleting { t.Fatalf("the delete over a terminated completion box: %v", err) } } // Ruling 86's guards: only a verified fence exempts. A live completion box // (no fence), a fence that does not verify (another instance, written by // the box itself, not terminated, another box's record at this key), an // unreadable fence and a completion box the roster does not hold all leave // the purge refused; an unreadable or unverifiable fence is its error. A // completion already discharged, or whose box is on lost, needs no fence: // the barrier passes it and an unreadable fence is never read. func TestPurgeOverACompletionNeedsAVerifiedFence(t *testing.T) { ctx := context.Background() t.Run("a live completion box", func(t *testing.T) { w := newCompletionWorld(t, nil) if _, err := w.purge(t); !errors.Is(err, hostclaim.ErrRelocationFinishing) { t.Fatalf("a live completion box: %v", err) } if _, err := w.delete(t); !errors.Is(err, hostclaim.ErrRelocationFinishing) { t.Fatalf("a live completion box (delete): %v", err) } w.unchanged(t, "a live completion box") }) for name, edit := range map[string]func(*termfence.Record){ "another instance": func(f *termfence.Record) { f.Cloud.InstanceID = "i-other" }, "written by the box": func(f *termfence.Record) { f.WrittenBy = termfence.WrittenBy{Box: "box-a", Enrollment: strings.Repeat("a", 32)} }, "not terminated": func(f *termfence.Record) { f.State = "running" }, "another box's record": func(f *termfence.Record) { f.Box = "box-w" }, "written by an unenrolled": func(f *termfence.Record) { f.WrittenBy.Box = "box-x" }, } { t.Run("a fence that does not verify: "+name, func(t *testing.T) { w := newCompletionWorld(t, nil) w.terminated(t, edit) if _, err := w.purge(t); err == nil { t.Fatalf("an unverified fence exempted the purge") } if _, err := w.delete(t); err == nil { t.Fatalf("an unverified fence exempted the delete") } w.unchanged(t, name) }) } t.Run("an unreadable fence", func(t *testing.T) { w := newCompletionWorld(t, nil) w.fence(t, []byte("{")) if _, err := w.purge(t); err == nil || errors.Is(err, hostclaim.ErrRelocationFinishing) { t.Fatalf("an unreadable fence: %v (want its error)", err) } w.unchanged(t, "an unreadable fence") }) t.Run("a fence read that fails", func(t *testing.T) { w := newCompletionWorld(t, nil) w.terminated(t, nil) w.a.Store = &faultStore{Store: w.s, failGet: "control/box-fences/" + strings.Repeat("a", 32) + ".json"} if fenced, err := w.a.completionFence(ctx, mustClaim(t, w)); fenced != "" || err == nil { t.Fatalf("a failed fence read: %q %v (want its error)", fenced, err) } }) t.Run("a completion box the roster does not hold", func(t *testing.T) { w := newCompletionWorld(t, func(c *hostclaim.Claim) { done, _, _ := hostclaim.ReadCompletedRelocation(c) done.Stop.Box = "box-gone" raw, _ := json.Marshal(done) c.Op.Exec, c.Op.Participants, c.Op.Journal[hostclaim.JournalCompleted] = "box-gone", []string{"box-gone"}, string(raw) }) w.terminated(t, nil) if fenced, err := w.a.completionFence(ctx, mustClaim(t, w)); fenced != "" || err != nil { t.Fatalf("an unrostered box: %q %v", fenced, err) } if _, err := w.purge(t); !errors.Is(err, hostclaim.ErrRelocationFinishing) { t.Fatalf("an unrostered box: %v", err) } }) t.Run("a completion record that cannot be read", func(t *testing.T) { w := newCompletionWorld(t, func(c *hostclaim.Claim) { c.Op.Journal[hostclaim.JournalCompleted] = "{" }) w.terminated(t, nil) if _, err := w.purge(t); !errors.Is(err, hostclaim.ErrRelocation) { t.Fatalf("an unreadable completion record: %v", err) } w.unchanged(t, "an unreadable completion record") }) for name, change := range map[string]func(*hostclaim.Claim){ "already discharged": func(c *hostclaim.Claim) { done, _, _ := hostclaim.ReadCompletedRelocation(c) done.Discharged = true raw, _ := json.Marshal(done) c.Op.Journal[hostclaim.JournalCompleted] = string(raw) }, "its box on lost": func(c *hostclaim.Claim) { c.Lost = append(c.Lost, "box-a") }, "no completion": func(c *hostclaim.Claim) { c.Op.Journal = nil }, } { t.Run("nothing to exempt: "+name, func(t *testing.T) { w := newCompletionWorld(t, change) w.fence(t, []byte("{")) if fenced, err := w.a.completionFence(ctx, mustClaim(t, w)); fenced != "" || err != nil { t.Fatalf("%s: %q %v (the fence must not be read)", name, fenced, err) } if started, err := w.purge(t); err != nil || started.Claim.Op.Kind != hostclaim.KindPurging { t.Fatalf("%s: %v", name, err) } }) } } func mustClaim(t *testing.T, w *completionWorld) *hostclaim.Claim { t.Helper() obs, err := hostclaim.Read(context.Background(), w.s, completionHost) if err != nil { t.Fatal(err) } return obs.Claim }