package host import ( "context" "encoding/json" "os" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootparticipation" ) type restartConsumptionFixture struct { p rootRestartRequests e *proofRootEnv u *plan14RestartUnits claim *hostclaim.Claim marks string } func newRestartConsumptionFixture(t *testing.T) restartConsumptionFixture { t.Helper() e, rt, u, checks, _ := plan14AdoptionFixture(t) rec, err := provision.GetRecord(t.Context(), e.store, rt.Account, rt.App, rt.Host) if err != nil { t.Fatal(err) } rec.Environment = "relay" rec.CreatedAt = time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) plan14PutRecord(t, e.store, *rec) inc, err := hostclaim.Incarnation(rec.Account, rec.App, rec.Host, rec.CreatedAt) if err != nil { t.Fatal(err) } c := &hostclaim.Claim{Host: rt.Host, Holder: &hostclaim.Holder{Key: provision.RecordKey(rt.Account, rt.App, rt.Host), Account: rt.Account, App: rt.App, Incarnation: inc, Stamp: 1, Session: 1, WriterBox: "box"}} c.Op.Kind, c.Op.Exec = hostclaim.KindServing, "box" state := &rollbackDisk{dir: t.TempDir()} g := hostguard.Guard{DataDir: t.TempDir(), LockDir: t.TempDir(), Claims: rootViewClaimReader{obs: hostclaim.Observed{Host: rt.Host, ETag: "claim", Claim: c}}, Root: hostguard.NewRootAuthority(state)} in := hostguard.RootInstallation{Host: rt.Host, Incarnation: inc, Stamp: 1, Account: rt.Account, App: rt.App, Record: c.Holder.Key, Box: "box", Enrollment: "en", Migration: 1, InventoryRevision: 1} if err := g.EstablishRoot(t.Context(), in, func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } writes := hostwrite.Config{Enabled: true, Selector: hostbox.WorkSource{Guard: g, Box: "box", Region: "ie", Now: time.Now, Policy: func(_ context.Context, c *hostclaim.Claim, s hostbox.Selection, _ hostwrite.Identity) error { return hostbox.Serving(c, s) }, Local: func(context.Context, *hostclaim.Claim, hostbox.Selection) error { return nil }, }} p := rootRestartRequests{journal: &release.RootRestarts{Store: &rollbackDisk{dir: t.TempDir()}}, source: rootRestartRequestSource{state: state, store: e.store, census: func() (rootparticipation.Census, error) { return rootparticipation.Census{Schema: 1, Box: "box", Enrollment: "en", Migration: 1, Revision: 1, Complete: true}, nil }}, writes: writes, sys: u, checks: checks, getenv: func(string) string { return "5s" }, optDir: u.opt, } return restartConsumptionFixture{p: p, e: e, u: u, claim: c, marks: restartMarksPath(t.TempDir())} } func (f restartConsumptionFixture) request(t *testing.T, kind, value string) { t.Helper() if _, err := release.AppendRestartScope(t.Context(), f.e.store, "bab", "console", release.RestartScope{Kind: kind, Value: value}); err != nil { t.Fatal(err) } } func (f restartConsumptionFixture) run(t *testing.T) error { t.Helper() return restartRequestPass(func() (bool, error) { return false, nil }, func() error { routes, err := f.e.root.reg.List() if err != nil { return err } restartPass(t.Context(), routes, f.e.store, f.marks, func(host string) error { return f.u.TryRestart("console.service") }) return nil }, func() error { return f.p.run(t.Context()) }) } func TestPlan14RootRestartProductionConsumption(t *testing.T) { t.Parallel() for _, attack := range []string{"none", "delete-marks", "rename-parent", "forge-marks", "delete-row", "forge-selector", "forge-unit"} { t.Run(attack, func(t *testing.T) { f := newRestartConsumptionFixture(t) if err := f.run(t); err != nil { t.Fatal(err) } f.request(t, release.RestartScopeEnvironment, "relay") if err := f.run(t); err != nil || len(f.u.effects) != 1 { t.Fatalf("first request: %v %v", err, f.u.effects) } if err := os.WriteFile(f.marks, []byte(`{"schema":1,"apps":{},"scopes":{}}`), 0600); err != nil { t.Fatal(err) } switch attack { case "delete-marks": if err := os.Remove(f.marks); err != nil { t.Fatal(err) } case "rename-parent": if err := os.Rename(filepath.Dir(f.marks), filepath.Dir(f.marks)+".old"); err != nil { t.Fatal(err) } t.Cleanup(func() { _ = os.RemoveAll(filepath.Dir(f.marks) + ".old") }) case "forge-marks": if err := os.WriteFile(f.marks, []byte(`{"schema":1,"apps":{"bab|console":"forged"},"scopes":{"bab|console":"forged"}}`), 0600); err != nil { t.Fatal(err) } case "delete-row": if err := f.e.root.reg.Delete(observedHost); err != nil { t.Fatal(err) } case "forge-selector", "forge-unit": r, err := f.e.root.reg.Get(observedHost) if err != nil { t.Fatal(err) } r.Environment, r.Unit, r.Hibernate = "forged", "unrelated.service", true if err := f.e.root.reg.Upsert(*r); err != nil { t.Fatal(err) } } if err := f.run(t); err != nil || len(f.u.effects) != 1 { t.Fatalf("replay: %v %v", err, f.u.effects) } f.request(t, release.RestartScopeEnvironment, "forged") if err := f.run(t); err != nil || len(f.u.effects) != 1 { t.Fatalf("widened: %v %v", err, f.u.effects) } f.request(t, release.RestartScopeHost, observedHost) if err := f.run(t); err != nil || len(f.u.effects) != 2 { t.Fatalf("fresh request: %v %v", err, f.u.effects) } }) } } func TestPlan14RootRestartProductionHeldRollover(t *testing.T) { t.Parallel() f := newRestartConsumptionFixture(t) if err := f.run(t); err != nil { t.Fatal(err) } f.claim.Op.Kind = hostclaim.KindLanding f.request(t, release.RestartScopeHost, observedHost) if err := f.e.root.reg.Delete(observedHost); err != nil { t.Fatal(err) } if err := f.run(t); err == nil || len(f.u.effects) != 0 { t.Fatalf("held: %v %v", err, f.u.effects) } for i := 0; i < 34; i++ { f.request(t, release.RestartScopeHost, "other.example.com") } f.claim.Op.Kind = hostclaim.KindServing if err := f.run(t); err != nil || len(f.u.effects) != 1 { t.Fatalf("lost pending: %v %v", err, f.u.effects) } if err := f.run(t); err != nil || len(f.u.effects) != 1 { t.Fatalf("replayed pending: %v %v", err, f.u.effects) } } func TestPlan14RootRestartProductionBindingChange(t *testing.T) { t.Parallel() f := newRestartConsumptionFixture(t) if err := f.run(t); err != nil { t.Fatal(err) } f.claim.Op.Kind = hostclaim.KindLanding f.request(t, release.RestartScopeHost, observedHost) _ = f.run(t) rec, err := provision.GetRecord(t.Context(), f.e.store, "bab", "console", observedHost) if err != nil { t.Fatal(err) } rec.Environment = "replacement" plan14PutRecord(t, f.e.store, *rec) f.claim.Op.Kind = hostclaim.KindServing if err := f.run(t); err == nil || len(f.u.effects) != 0 { t.Fatalf("changed binding: %v %v", err, f.u.effects) } } func TestPlan14RootRestartProductionAdmissionAfterIntent(t *testing.T) { t.Parallel() f := newRestartConsumptionFixture(t) if err := f.run(t); err != nil { t.Fatal(err) } f.request(t, release.RestartScopeHost, observedHost) disk := f.p.journal.Store.(*rollbackDisk) disk.afterWrite = func(_ string, raw []byte) { var r struct{ Pending *release.RestartIntent } if json.Unmarshal(raw, &r) == nil && r.Pending != nil && r.Pending.Targets[0].Before != "" { f.claim.Op.Kind = hostclaim.KindLanding } } if err := f.run(t); err == nil || len(f.u.effects) != 0 { t.Fatalf("move crossed dispatch intent: %v %v", err, f.u.effects) } disk.afterWrite = nil f.claim.Op.Kind = hostclaim.KindServing if err := f.run(t); err == nil || len(f.u.effects) != 0 { t.Fatalf("ambiguous intent replayed: %v %v", err, f.u.effects) } // A verified successor completes the captured request without another call. f.u.props.InvocationID += "successor" f.u.props.MainPID++ if err := f.run(t); err != nil || len(f.u.effects) != 0 { t.Fatalf("successor: %v %v", err, f.u.effects) } } func TestPlan14RootRestartProductionUnknown(t *testing.T) { t.Parallel() for _, attack := range []string{"census-error", "census-incomplete", "root-corrupt", "record-missing", "retired-pending", "flag-off", "foreign-census", "inactive"} { t.Run(attack, func(t *testing.T) { f := newRestartConsumptionFixture(t) if err := f.run(t); err != nil { t.Fatal(err) } f.request(t, release.RestartScopeHost, observedHost) switch attack { case "census-error": f.p.source.census = func() (rootparticipation.Census, error) { return rootparticipation.Census{}, os.ErrPermission } case "census-incomplete": f.p.source.census = func() (rootparticipation.Census, error) { return rootparticipation.Census{Box: "box"}, nil } case "foreign-census": f.p.source.census = func() (rootparticipation.Census, error) { return rootparticipation.Census{Box: "other", Complete: true}, nil } case "root-corrupt": if err := f.p.source.state.Write(observedHost+".json", []byte(`{}`)); err != nil { t.Fatal(err) } case "record-missing": if err := f.e.store.Delete(t.Context(), provision.RecordKey("bab", "console", observedHost)); err != nil { t.Fatal(err) } case "retired-pending": f.claim.Op.Kind = hostclaim.KindLanding _ = f.run(t) f.claim.Op.Kind = hostclaim.KindServing if err := f.p.source.state.Remove(observedHost + ".json"); err != nil { t.Fatal(err) } case "flag-off": f.p.writes.Enabled = false case "inactive": f.u.props.ActiveState = "inactive" } if err := f.run(t); err == nil || len(f.u.effects) != 0 { t.Fatalf("unknown granted: %v %v", err, f.u.effects) } }) } } func TestPlan14RootRestartRequestCensusGate(t *testing.T) { for _, mode := range []string{"absent", "present", "unreadable"} { t.Run(mode, func(t *testing.T) { old, root := 0, 0 err := restartRequestPass(func() (bool, error) { if mode == "unreadable" { return false, os.ErrPermission } return mode == "absent", nil }, func() error { old++; return nil }, func() error { root++; return nil }) if mode == "unreadable" { if err == nil || old+root != 0 { t.Fatal(err, old, root) } return } if err != nil || (mode == "absent" && (old != 1 || root != 0)) || (mode == "present" && (old != 0 || root != 1)) { t.Fatal(err, old, root) } }) } } type restartRequestSyncFault struct { *rollbackDisk stage string } func (s *restartRequestSyncFault) Write(name string, raw []byte) error { var r struct { Pending *release.RestartIntent ScopeID string `json:"scope_id"` } if err := json.Unmarshal(raw, &r); err != nil { return err } stage := "baseline" if r.Pending != nil { stage = "pending" if r.Pending.Targets[0].Before != "" { stage = "intent" } if r.Pending.Targets[0].After != "" { stage = "completed" } } else if r.ScopeID != release.RestartScopeNone { stage = "consumed" } if stage != s.stage { return s.rollbackDisk.Write(name, raw) } f, err := os.CreateTemp(s.dir, "unsynced-") if err != nil { return err } defer os.Remove(f.Name()) if _, err = f.Write(raw); err != nil { f.Close() return err } if err = f.Sync(); err != nil { f.Close() return err } if err = f.Close(); err != nil { return err } if err = os.Rename(f.Name(), filepath.Join(s.dir, name)); err != nil { return err } return os.ErrPermission } func TestPlan14RootRestartProductionSyncFailure(t *testing.T) { t.Parallel() for _, stage := range []string{"baseline", "pending", "intent", "completed", "consumed"} { t.Run(stage, func(t *testing.T) { f := newRestartConsumptionFixture(t) if err := f.run(t); err != nil { t.Fatal(err) } f.request(t, release.RestartScopeHost, observedHost) fault := &restartRequestSyncFault{rollbackDisk: f.p.journal.Store.(*rollbackDisk), stage: stage} f.p.journal.Store = fault if err := f.run(t); err == nil { t.Fatal("failed sync accepted") } want := 0 if stage == "completed" || stage == "consumed" { want = 1 } if len(f.u.effects) != want { t.Fatal(f.u.effects) } // The read must repair durability before dispatch or completion. _ = f.run(t) if len(f.u.effects) != want { t.Fatal("repeat under failed sync", f.u.effects) } fault.stage = "" err := f.run(t) if stage == "intent" { if err == nil || len(f.u.effects) != 0 { t.Fatalf("ambiguous repeat: %v %v", err, f.u.effects) } f.u.props.InvocationID += "verified" f.u.props.MainPID++ if err = f.run(t); err != nil || len(f.u.effects) != 0 { t.Fatalf("repair: %v %v", err, f.u.effects) } } else if err != nil || len(f.u.effects) != 1 { t.Fatalf("repair: %v %v", err, f.u.effects) } }) } } func TestPlan14RootRestartProductionCanonicalSelection(t *testing.T) { t.Parallel() for _, kind := range []string{"hibernate", "exec", "sidecar"} { t.Run(kind, func(t *testing.T) { f := newRestartConsumptionFixture(t) if kind == "sidecar" { // A valid root sidecar installation cannot hide another app's units. in, err := f.p.source.installation(t.Context(), observedHost) if err != nil { t.Fatal(err) } in.Host = provision.SidecarHost("worker", in.App, in.Account) in.Record = provision.SidecarRecordKey(in.Account, in.App, "worker") claim := *f.claim holder := *claim.Holder claim.Holder = &holder claim.Host = in.Host holder.Key = in.Record g := hostguard.Guard{DataDir: t.TempDir(), LockDir: t.TempDir(), Root: hostguard.NewRootAuthority(f.p.source.state), Claims: rootViewClaimReader{obs: hostclaim.Observed{Host: in.Host, ETag: "sidecar", Claim: &claim}}} if err = g.EstablishRoot(t.Context(), in, func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } } else { rec, err := provision.GetRecord(t.Context(), f.e.store, "bab", "console", observedHost) if err != nil { t.Fatal(err) } if kind == "exec" { rec.Unit = "" } else { yes := true rec.Hibernate = &yes rec.Addr = "" rec.Unit = "carlos-app@" + rec.Host + ".service" } plan14PutRecord(t, f.e.store, *rec) } if err := f.run(t); err != nil { t.Fatal(err) } f.request(t, release.RestartScopeHost, observedHost) if err := f.run(t); err != nil { t.Fatal(err) } want := 0 if kind == "sidecar" { want = 1 } if len(f.u.effects) != want { t.Fatal(kind, f.u.effects) } }) } } func TestPlan14RootRestartProductionReplacedInstallation(t *testing.T) { t.Parallel() f := newRestartConsumptionFixture(t) if err := f.run(t); err != nil { t.Fatal(err) } f.request(t, release.RestartScopeHost, observedHost) f.claim.Op.Kind = hostclaim.KindLanding _ = f.run(t) in, err := f.p.source.installation(t.Context(), observedHost) if err != nil { t.Fatal(err) } f.claim.Holder.Stamp++ in.Stamp++ w := f.p.writes.Selector.(hostbox.WorkSource) if err = w.Guard.EstablishRoot(t.Context(), in, func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } f.claim.Op.Kind = hostclaim.KindServing if err = f.run(t); err == nil || len(f.u.effects) != 0 { t.Fatalf("captured request crossed installation: %v %v", err, f.u.effects) } } type restartRequestErrorUnits struct { *plan14RestartUnits after bool } func (u restartRequestErrorUnits) TryRestart(unit string) error { if u.after { if err := u.plan14RestartUnits.TryRestart(unit); err != nil { return err } } return os.ErrPermission } func TestPlan14RootRestartProductionDispatchError(t *testing.T) { t.Parallel() for _, after := range []bool{false, true} { t.Run(map[bool]string{false: "no-effect", true: "effect-then-error"}[after], func(t *testing.T) { f := newRestartConsumptionFixture(t) if err := f.run(t); err != nil { t.Fatal(err) } f.request(t, release.RestartScopeHost, observedHost) f.p.sys = restartRequestErrorUnits{f.u, after} if err := f.run(t); err == nil { t.Fatal("lost dispatch error") } b := release.RestartBinding{Box: "box", Account: "bab", App: "console"} pending, err := f.p.journal.Observe(b, release.RestartSnapshot{}) if err != nil || pending == nil { t.Fatalf("ambiguous consumed: %+v %v", pending, err) } f.p.sys = f.u err = f.run(t) if after { if err != nil || len(f.u.effects) != 1 { t.Fatal(err, f.u.effects) } } else { if err == nil || len(f.u.effects) != 0 { t.Fatal(err, f.u.effects) } } }) } } func TestPlan14RootRestartProductionDefiniteFailure(t *testing.T) { t.Parallel() f := newRestartConsumptionFixture(t) if err := f.run(t); err != nil { t.Fatal(err) } f.request(t, release.RestartScopeHost, observedHost) f.u.afterEffect = func() { f.u.props.ActiveState = "failed"; f.u.props.MainPID = 0 } if err := f.run(t); err == nil || len(f.u.effects) != 1 { t.Fatal(err, f.u.effects) } if err := f.run(t); err != nil || len(f.u.effects) != 1 { t.Fatalf("failed request repeated: %v %v", err, f.u.effects) } f.u.afterEffect = nil f.u.props.ActiveState = "active" f.u.props.MainPID = 8 f.u.props.InvocationID += "recovered" f.request(t, release.RestartScopeHost, observedHost) if err := f.run(t); err != nil || len(f.u.effects) != 2 { t.Fatalf("new request lost: %v %v", err, f.u.effects) } } type restartRequestShowError struct{ *plan14RestartUnits } func (u restartRequestShowError) Show(unit string) (unitProps, error) { if len(u.effects) > 0 { return unitProps{}, os.ErrPermission } return u.plan14RestartUnits.Show(unit) } func TestPlan14RootRestartProductionObservationError(t *testing.T) { t.Parallel() f := newRestartConsumptionFixture(t) if err := f.run(t); err != nil { t.Fatal(err) } f.request(t, release.RestartScopeHost, observedHost) f.p.sys = restartRequestShowError{f.u} if err := f.run(t); err == nil || len(f.u.effects) != 1 { t.Fatal(err, f.u.effects) } b := release.RestartBinding{Box: "box", Account: "bab", App: "console"} pending, err := f.p.journal.Observe(b, release.RestartSnapshot{}) if err != nil || pending == nil { t.Fatalf("observation failure consumed: %+v %v", pending, err) } f.p.sys = f.u if err = f.run(t); err != nil || len(f.u.effects) != 1 { t.Fatalf("observed successor replayed: %v %v", err, f.u.effects) } }