package host import ( "context" "encoding/json" "errors" "fmt" "os/exec" "strings" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/genselect" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/registry" "github.com/superfly/ltx" ) type plan9D5Owner struct { name string calls *[]string } func (o plan9D5Owner) note(call string) { *o.calls = append(*o.calls, o.name+"."+call) } func (o plan9D5Owner) Select(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) (ClaimLaunchInputs, error) { o.note("select") return ClaimLaunchInputs{}, nil } func (o plan9D5Owner) Materialize(context.Context, hostclaim.Observed, ClaimLaunchInputs, hostwrite.Work) error { o.note("materialize") return nil } func (o plan9D5Owner) EmptyAuthority(context.Context, hostclaim.Observed, ClaimLaunchInputs) error { o.note("empty") return nil } func (o plan9D5Owner) Publish(context.Context, hostclaim.Observed, ClaimLaunchInputs) error { o.note("publish") return nil } func (o plan9D5Owner) owners() ClaimKindOwners { return ClaimKindOwners{Inputs: o, AdmissionEvidence: func(context.Context, hostclaim.Observed, hostclaim.Admission, *genselect.Plan) error { o.note("admission") return nil }, PublicationEvidence: func(context.Context, hostclaim.Observed) error { o.note("publication"); return nil }} } func plan9D5Kind(kind string) hostclaim.Observed { id := "op" if kind == hostclaim.KindServing { id = "" } return hostclaim.Observed{Host: recoveryHost, Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{}, Op: hostclaim.Op{Kind: kind, ID: id}}} } func plan9D5Call(d ClaimOwnerDispatch, o hostclaim.Observed) error { ctx := context.Background() _, err := d.Select(ctx, registry.Route{}, o, activator.StartPolicy{}) return errors.Join(err, d.Materialize(ctx, o, ClaimLaunchInputs{}, nil), d.EmptyAuthority(ctx, o, ClaimLaunchInputs{}), d.Publish(ctx, o, ClaimLaunchInputs{}), d.AdmissionEvidence(ctx, o, hostclaim.Admission{}, nil), d.PublicationEvidence(ctx, o)) } func TestPlan9D5OwnerDispatchRoutesByOpKind(t *testing.T) { var calls []string moves := plan9D5Owner{"moves", &calls}.owners() d := ClaimOwnerDispatch{Serving: plan9D5Owner{"serving", &calls}.owners(), Moves: &moves} all := "select,materialize,empty,publish,admission,publication" for kind, owner := range map[string]string{hostclaim.KindServing: "serving", hostclaim.KindMoving: "moves", hostclaim.KindLanding: "moves", hostclaim.KindReverting: "moves", hostclaim.KindRestoring: "moves"} { calls = nil if err := plan9D5Call(d, plan9D5Kind(kind)); err != nil { t.Fatalf("%s: %v", kind, err) } if got, want := strings.Join(calls, ","), owner+"."+strings.ReplaceAll(all, ",", ","+owner+"."); got != want { t.Fatalf("%s routed %s, want %s", kind, got, want) } } for _, kind := range []string{hostclaim.KindCreating, hostclaim.KindDeleting, hostclaim.KindParking, hostclaim.KindPurging, hostclaim.KindPlacing, hostclaim.KindRelocating, hostclaim.KindPushing} { calls = nil if err := plan9D5Call(d, plan9D5Kind(kind)); !errors.Is(err, hostbox.ErrUnsupported) || len(calls) != 0 { t.Fatalf("%s: %v %v", kind, err, calls) } } // Moves off or unowned: the move kinds keep main's typed refusal and // serving is unaffected. d.Moves = nil var refusal *ClaimDependencyError calls = nil if err := plan9D5Call(d, plan9D5Kind(hostclaim.KindLanding)); !errors.As(err, &refusal) || len(calls) != 0 { t.Fatalf("unowned landing: %v %v", err, calls) } if err := plan9D5Call(d, plan9D5Kind(hostclaim.KindServing)); err != nil { t.Fatal(err) } if err := plan9D5Call(ClaimOwnerDispatch{}, plan9D5Kind(hostclaim.KindServing)); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("missing serving owners: %v", err) } } func TestPlan9D5MovesIntegrationNeedsBothFlagsAndACompleteOwnerSet(t *testing.T) { var calls []string complete := ClaimMovesOwners{ClaimKindOwners: plan9D5Owner{"moves", &calls}.owners(), Operations: &plan9D2Attempts{}} env := func(v string) func(string) string { return func(k string) string { if k == "CARLOS_HOST_CLAIM" { return "1" } if k == "CARLOS_MOVES_PHASE1" { return v } return "" } } for name, tc := range map[string]struct { getenv func(string) string owners ClaimMovesOwners want bool err error }{ "claim-off": {func(k string) string { if k == "CARLOS_MOVES_PHASE1" { return "on" } return "" }, complete, false, nil}, "moves-off": {env(""), complete, false, nil}, "moves-zero": {env("0"), complete, false, nil}, "no-owners": {env("on"), ClaimMovesOwners{}, false, nil}, "complete": {env("on"), complete, true, nil}, "no-attempts": {env("on"), ClaimMovesOwners{ClaimKindOwners: complete.ClaimKindOwners}, false, hostwrite.ErrUnavailable}, "no-publication": {env("1"), ClaimMovesOwners{ClaimKindOwners: ClaimKindOwners{Inputs: complete.Inputs, AdmissionEvidence: complete.AdmissionEvidence}, Operations: complete.Operations}, false, hostwrite.ErrUnavailable}, "probe-check-only": {env("true"), ClaimMovesOwners{ProbeReady: func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil }}, false, hostwrite.ErrUnavailable}, } { got, err := claimMovesC1(tc.getenv, tc.owners) if got != tc.want || !errors.Is(err, tc.err) { t.Fatalf("%s: %v %v", name, got, err) } } } // plan9D5Markers is an If-Match marker store over the versioned fixture. type plan9D5Markers struct{ b *leasetest.VersionedBucket } func (m plan9D5Markers) PutMarker(ctx context.Context, bucket, key string, body []byte, expected string, check func(context.Context) error) (string, error) { if err := check(ctx); err != nil { return "", err } versions, err := m.b.ListVersions(ctx, bucket, key) if err != nil { return "", err } current := "" for _, v := range versions { if v.Key == key && v.IsLatest && !v.DeleteMarker { current = v.ETag } } if current != expected { return "", hostbox.ErrChanged } return m.b.Put(bucket, key, body), nil } type plan9D5Serving struct { owners *claimServingOwners bucket *leasetest.VersionedBucket launch ClaimServingLaunch seq int files []string filesErr error synced int } // plan9D5Files is a C8 files provider that records each establishment. type plan9D5Files struct{ f *plan9D5Serving } func (p plan9D5Files) Establish(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error { if err := work.With(ctx, in.Identity, func() error { return nil }); err != nil { return err } p.f.files = append(p.f.files, o.Claim.Holder.Generation+"/"+in.Admission.NewGeneration) return p.f.filesErr } // publication is a fenced publication whose sync is counted and whose // fence always passes. func (f *plan9D5Serving) publication() ClaimPublication { return ClaimPublication{Sync: func(context.Context) error { f.synced++; return nil }, Check: func(context.Context) error { return nil }} } func plan9D5Fixture(t *testing.T) *plan9D5Serving { t.Helper() b := leasetest.NewVersionedBucket() b.AddBucket("data") f := &plan9D5Serving{bucket: b, launch: ClaimServingLaunch{Identity: hostwrite.Identity{Config: "sha256:config", Release: "sha256:release"}, Executable: "/opt/carlos/live/" + recoveryHost, ExecutableSHA256: strings.Repeat("b", 64), Args: []string{"--socket", "/run/app.sock"}, Env: []string{"A=1"}, Version: "v7"}} f.owners = &claimServingOwners{Data: b, Bucket: "data", Region: "ie", Box: "box-1", Now: func() time.Time { return time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) }, NewID: func() string { f.seq++; return fmt.Sprintf("fresh-%d", f.seq) }, Sources: ClaimServingSources{Launch: func(context.Context, registry.Route) (ClaimServingLaunch, error) { return f.launch, nil }, Markers: plan9D5Markers{b}, Files: plan9D5Files{f}}} return f } func (f *plan9D5Serving) file(t *testing.T, gen string, level int, min, max uint64) { t.Helper() h := ltx.Header{Version: ltx.Version, Flags: ltx.HeaderFlagNoChecksum, PageSize: 4096, MinTXID: ltx.TXID(min), MaxTXID: ltx.TXID(max)} body, err := h.MarshalBinary() if err != nil { t.Fatal(err) } f.bucket.Put("data", fmt.Sprintf("instances/%s/g-%s/%04x/%s", recoveryHost, gen, level, ltx.FormatFilename(ltx.TXID(min), ltx.TXID(max))), body) } func (f *plan9D5Serving) marker(t *testing.T, gen string) { t.Helper() body, err := json.Marshal(genselect.Marker{Op: gen, Incarnation: "inc", Host: recoveryHost, Box: "box-0", InstalledAt: time.Unix(1, 0).UTC()}) if err != nil { t.Fatal(err) } f.bucket.Put("data", "instances/"+recoveryHost+"/generation.json", body) } func plan9D5Observed(h hostclaim.Holder) hostclaim.Observed { h.Incarnation, h.Location = "inc", "instances/"+recoveryHost return hostclaim.Observed{Host: recoveryHost, ETag: "etag", Claim: &hostclaim.Claim{Host: recoveryHost, Holder: &h, Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box-1"}}} } var plan9D5Route = registry.Route{Host: recoveryHost, DBPath: "/data/" + recoveryHost + "/data.db"} func TestPlan9D5ServingEmptyBootstrap(t *testing.T) { f := plan9D5Fixture(t) ctx := t.Context() created := plan9D5Observed(hostclaim.Holder{Stamp: 3}) in, err := f.owners.Select(ctx, plan9D5Route, created, activator.StartPolicy{MaxStarts: 5, Window: 30 * time.Second}) if err != nil { t.Fatal(err) } a := in.Admission if !in.Empty || in.Baseline != nil || a.Mode != "cold" || a.NewGeneration != "fresh-1" || a.RemoteMaxTXID != 0 || a.PredecessorRegion != "" || a.KeepStamp || a.Box != "box-1" || a.Region != "ie" || in.Identity != f.launch.Identity || in.DBPath != plan9D5Route.DBPath || len(in.Roles) != 1 || in.Roles[0] != "main" { t.Fatalf("empty bootstrap inputs: %+v", in) } if err := f.owners.admissionEvidence(ctx, created, a, nil); err != nil { t.Fatal(err) } for name, mutate := range map[string]func(*hostclaim.Admission){ "keep-stamp": func(a *hostclaim.Admission) { a.KeepStamp = true }, "certified": func(a *hostclaim.Admission) { a.PredecessorCertified = true }, "coverage": func(a *hostclaim.Admission) { a.Coverage = "sealed" }, "region": func(a *hostclaim.Admission) { a.PredecessorRegion = "ie" }, "op": func(a *hostclaim.Admission) { a.ObservedOpID = "m1" }, "generation": func(a *hostclaim.Admission) { a.NewGeneration = hostclaim.RootGeneration }, "other-box": func(a *hostclaim.Admission) { a.Box = "box-2" }, } { changed := a mutate(&changed) if err := f.owners.admissionEvidence(ctx, created, changed, nil); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("%s: %v", name, err) } } admitted := plan9D5Observed(hostclaim.Holder{Stamp: 5, Session: 5, Generation: "fresh-1", WriterBox: "box-1", Admissions: []hostclaim.AdmissionRecord{{Session: 5, Box: "box-1", Generation: "fresh-1", Region: "ie", Mode: "cold"}}}) if err := f.owners.EmptyAuthority(ctx, admitted, in); err != nil { t.Fatal(err) } f.file(t, "elsewhere", 0, 1, 1) if err := f.owners.EmptyAuthority(ctx, admitted, in); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a nonempty location passed as empty: %v", err) } f.bucket.FailOn = func(op, _ string) error { if op == "list" { return errors.New("listing failed") } return nil } if err := f.owners.EmptyAuthority(ctx, admitted, in); err == nil { t.Fatal("a listing error passed as empty") } } func TestPlan9D5ServingSelectRefusals(t *testing.T) { f := plan9D5Fixture(t) ctx := t.Context() created := plan9D5Observed(hostclaim.Holder{Stamp: 3}) if _, err := f.owners.Select(ctx, plan9D5Route, created, activator.StartPolicy{Probe: true}); !errors.Is(err, hostbox.ErrUnsupported) { t.Fatalf("probe: %v", err) } if _, err := f.owners.Select(ctx, registry.Route{Host: recoveryHost, DBPath: "relative.db"}, created, activator.StartPolicy{}); !errors.Is(err, hostbox.ErrUnsupported) { t.Fatalf("relative database: %v", err) } landing := created landing.Claim = &hostclaim.Claim{Host: recoveryHost, Holder: created.Claim.Holder, Op: hostclaim.Op{Kind: hostclaim.KindLanding, ID: "m1", Phase: "committed", Exec: "box-1"}} if _, err := f.owners.Select(ctx, plan9D5Route, landing, activator.StartPolicy{}); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("serving owners answered a landing claim: %v", err) } f.launch.ExecutableSHA256 = "short" if _, err := f.owners.Select(ctx, plan9D5Route, created, activator.StartPolicy{}); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("bad digest: %v", err) } if _, err := f.owners.launchDigest(ctx, plan9D5Route); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("bad digest in the retry inputs: %v", err) } } func TestPlan9D5ServingBaselinePublicationAndEvidence(t *testing.T) { f := plan9D5Fixture(t) ctx := t.Context() f.file(t, "g0", 9, 1, 11) f.marker(t, "g0") serving := plan9D5Observed(hostclaim.Holder{Stamp: 4, Session: 4, Generation: "g0", WriterBox: "box-1"}) in, err := f.owners.Select(ctx, plan9D5Route, serving, activator.StartPolicy{}) if err != nil { t.Fatal(err) } if in.Empty || in.Baseline == nil || in.Baseline.Generation != "g0" || in.Baseline.MaxTXID != 11 || in.Admission.RemoteMaxTXID != 11 || in.Admission.PredecessorRegion != "ie" || in.Admission.ObservedGeneration != "g0" || in.Admission.ObservedSession != 4 { t.Fatalf("serving inputs: %+v", in) } if err := f.owners.admissionEvidence(ctx, serving, in.Admission, in.Baseline); err != nil { t.Fatal(err) } stale := *in.Baseline stale.MaxTXID = 10 if err := f.owners.admissionEvidence(ctx, serving, in.Admission, &stale); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a baseline that is not the certified remote: %v", err) } if err := f.owners.admissionEvidence(ctx, serving, in.Admission, nil); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("no baseline: %v", err) } // After the admission CAS, publication waits for the generation's root. admitted := plan9D5Observed(hostclaim.Holder{Stamp: 7, Session: 7, Generation: in.Admission.NewGeneration, WriterBox: "box-1", Admissions: []hostclaim.AdmissionRecord{{Session: 7, Box: "box-1", Generation: in.Admission.NewGeneration, Predecessor: "g0", Region: "ie", PredecessorRegion: "ie", Mode: "cold"}}}) before := f.bucket.LatestID("data", "instances/"+recoveryHost+"/generation.json") if _, err := f.owners.PublishFenced(ctx, admitted, in, f.publication()); !errors.Is(err, hostbox.ErrChanged) || f.synced != 1 { t.Fatalf("a marker before the root: %v synced=%d", err, f.synced) } // The unfenced entry never writes a serving marker. if err := f.owners.Publish(ctx, admitted, in); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("unfenced publication: %v", err) } if f.bucket.LatestID("data", "instances/"+recoveryHost+"/generation.json") != before { t.Fatal("the marker moved before the root existed") } if err := f.owners.publicationEvidence(ctx, admitted); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("publication evidence without a published marker: %v", err) } f.file(t, in.Admission.NewGeneration, 0, 1, 1) receipt, err := f.owners.PublishFenced(ctx, admitted, in, f.publication()) if err != nil { t.Fatal(err) } root := fmt.Sprintf("instances/%s/g-%s/0000/%s", recoveryHost, in.Admission.NewGeneration, ltx.FormatFilename(1, 1)) if receipt.MarkerVersion == "" || receipt.Root.Key != root || receipt.Root.VersionID != f.bucket.LatestID("data", root) { t.Fatalf("receipt: %+v", receipt) } if err := f.owners.publicationEvidence(ctx, admitted); err != nil { t.Fatal(err) } body, ok := f.bucket.Current("data", "instances/"+recoveryHost+"/generation.json") var m genselect.Marker if !ok || json.Unmarshal(body, &m) != nil || m.Op != in.Admission.NewGeneration || m.Box != "box-1" || m.Incarnation != "inc" || m.SnapshotTXID != 0 { t.Fatalf("published marker: %s", body) } // A replay of the same session's publication writes nothing new. written := f.bucket.LatestID("data", "instances/"+recoveryHost+"/generation.json") if replay, err := f.owners.PublishFenced(ctx, admitted, in, f.publication()); err != nil || replay != receipt || f.bucket.LatestID("data", "instances/"+recoveryHost+"/generation.json") != written { t.Fatalf("replay: %+v %v", replay, err) } // Another writer's marker replaces ours: the evidence no longer holds. f.marker(t, in.Admission.NewGeneration) if err := f.owners.publicationEvidence(ctx, admitted); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("evidence for a marker this session did not write: %v", err) } } func TestPlan9D5ServingPublishSupersedesOnlyTheObservedMarker(t *testing.T) { f := plan9D5Fixture(t) ctx := t.Context() f.file(t, "g0", 9, 1, 11) f.marker(t, "g0") in := ClaimLaunchInputs{Admission: hostclaim.Admission{NewGeneration: "g1"}} admitted := plan9D5Observed(hostclaim.Holder{Stamp: 7, Session: 7, Generation: "g1", WriterBox: "box-1", Admissions: []hostclaim.AdmissionRecord{{Session: 7, Box: "box-1", Generation: "g1", Predecessor: "g0", Region: "ie", PredecessorRegion: "ie", Mode: "cold"}}}) f.file(t, "g1", 0, 1, 1) // The conditional write loses to a marker written after its read. f.owners.Sources.Markers = plan9D5Racing{plan9D5Markers{f.bucket}, func() { f.bucket.Put("data", "instances/"+recoveryHost+"/generation.json", []byte(`{"op":"other"}`)) }} if _, err := f.owners.PublishFenced(ctx, admitted, in, f.publication()); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a raced marker write: %v", err) } } type plan9D5Racing struct { plan9D5Markers race func() } func (r plan9D5Racing) PutMarker(ctx context.Context, bucket, key string, body []byte, expected string, check func(context.Context) error) (string, error) { r.race() return r.plan9D5Markers.PutMarker(ctx, bucket, key, body, expected, check) } type plan9D5Work struct { identities *[]hostwrite.Identity err error } func (w plan9D5Work) With(_ context.Context, id hostwrite.Identity, fn func() error) error { *w.identities = append(*w.identities, id) if w.err != nil { return w.err } return fn() } func TestPlan9D5ServingMaterializeChecksIdentityThenFiles(t *testing.T) { f := plan9D5Fixture(t) ctx := t.Context() admitted := plan9D5Observed(hostclaim.Holder{Stamp: 7, Session: 7, Generation: "g1", WriterBox: "box-1"}) in := ClaimLaunchInputs{Identity: f.launch.Identity, Admission: hostclaim.Admission{NewGeneration: "g1"}} var seen []hostwrite.Identity if err := f.owners.Materialize(ctx, admitted, in, plan9D5Work{identities: &seen}); err != nil { t.Fatal(err) } if len(seen) != 2 || seen[0] != f.launch.Identity || seen[1] != f.launch.Identity || len(f.files) != 1 || f.files[0] != "g1/g1" { t.Fatalf("identity checks %v, files %v", seen, f.files) } // A refused identity establishes no files manifest. refused := errors.New("held identity changed") if err := f.owners.Materialize(ctx, admitted, in, plan9D5Work{identities: &seen, err: refused}); !errors.Is(err, refused) || len(f.files) != 1 { t.Fatalf("refused identity: %v files=%v", err, f.files) } f.filesErr = errors.New("no valid predecessor manifest") if err := f.owners.Materialize(ctx, admitted, in, plan9D5Work{identities: &seen}); !errors.Is(err, f.filesErr) { t.Fatalf("files failure: %v", err) } if err := f.owners.Materialize(ctx, admitted, ClaimLaunchInputs{}, plan9D5Work{identities: &seen}); !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("no identity: %v", err) } f.owners.Sources.Files = nil var refusal *ClaimDependencyError if err := f.owners.Materialize(ctx, admitted, in, plan9D5Work{identities: &seen}); !errors.As(err, &refusal) || !errors.Is(err, hostwrite.ErrUnavailable) { t.Fatalf("no files owner: %v", err) } }