package host import ( "bytes" "context" "crypto/ed25519" "encoding/json" "errors" "log" "os" "path/filepath" "strings" "sync" "testing" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" ) // Model try-restart's no-op for a stopped service, and reset-failed's // transition to inactive. Restart represents the operator's subsequent start. type r2Units struct { *plan14RestartUnits showErr error tryErr error tries []string starts []string } func (u *r2Units) Show(unit string) (unitProps, error) { return u.props, u.showErr } func (u *r2Units) TryRestart(unit string) error { u.tries = append(u.tries, unit) if u.tryErr != nil { return u.tryErr } if u.props.ActiveState == "inactive" || u.props.ActiveState == "failed" { return nil } return u.Restart(unit) } func (u *r2Units) ResetFailed(unit string) error { u.effects = append(u.effects, "reset "+unit) if u.props.MainPID == 0 { u.props.ActiveState, u.props.SubState = "inactive", "dead" } return nil } func (u *r2Units) Start(unit string) error { u.starts = append(u.starts, unit) if u.props.ActiveState == "active" { return nil } return u.Restart(unit) } type r2Live struct { e *proofRootEnv rt registry.Route u *r2Units c rootRestartChecks source *proofBindingSource pub ed25519.PublicKey key ed25519.PrivateKey } func r2Fixture(t *testing.T, sourceKind string) *r2Live { t.Helper() e, rt, base, c, _ := plan14AdoptionFixture(t) u := &r2Units{plan14RestartUnits: base} if err := e.store.Delete(t.Context(), provision.RecordKey(rt.Account, rt.App, rt.Host)); err != nil { t.Fatal(err) } rt.Account = "" if err := e.root.reg.Upsert(rt); err != nil { t.Fatal(err) } s := productionProofBindingSource(e.store, u, u.opt) s.namedPath = filepath.Join(t.TempDir(), "named-units.json") s.unitDirs = []string{filepath.Dir(u.props.FragmentPath)} s.healthy = nil put := func(path string, body []byte) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { t.Fatal(err) } if err := os.WriteFile(path, body, 0644); err != nil { t.Fatal(err) } makeProofAncestorsPublic(t, path) } makeProofAncestorsPublic(t, s.namedPath) makeProofAncestorsPublic(t, u.exe) makeProofAncestorsPublic(t, u.props.FragmentPath) switch sourceKind { case "guard": put(filepath.Join(u.opt, "system/guard/console-ready.json"), []byte(release.LiveLink(u.opt, rt.Host))) drop := filepath.Join(s.unitDirs[0], "console.service.d/90-host-floor.conf") put(drop, []byte(hostfloor.ConsoleGuardDropIn(release.LiveLink(u.opt, rt.Host)))) u.props.DropInPaths = []string{drop} u.props.ExecPath = "/opt/carlos/system/guard/carlos" put(u.props.ExecPath, []byte("guard")) case "list": body, err := json.Marshal(struct { Schema int `json:"schema"` Units []namedUnit `json:"units"` }{1, []namedUnit{{Host: rt.Host, App: rt.App, Unit: rt.Unit, Addr: rt.Addr, Health: &provision.Health{Path: "/ready"}}}}) if err != nil { t.Fatal(err) } put(s.namedPath, body) } c.resolve, c.host, c.rollback, c.prepareRollback, c.dispatch, c.targetFile = s.resolve, s.forHost, s.rollback, s.prepareRollback, s.needsRestart, rootstate.OpenTrustedFile c.legacy, c.legacyLog, c.legacyReady = legacyRootRestart, &sync.Once{}, c.ready pub, key, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } if err := e.store.Delete(t.Context(), release.ChannelKey("bab", rt.App, rt.Channel)); err != nil { t.Fatal(err) } f := &r2Live{e: e, rt: rt, u: u, c: c, source: s, pub: pub, key: key} f.promote(t, "v1") f.promote(t, "v2") return f } func (f *r2Live) promote(t *testing.T, v string) { t.Helper() if err := release.Promote(t.Context(), f.e.store, f.key, "bab", f.rt.App, f.rt.Channel, v); err != nil { t.Fatal(err) } } func (f *r2Live) restart() func(string) error { return rootRestartRouteWith(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, nil, f.c) } func (f *r2Live) pass(t *testing.T, extra ...release.PassOption) error { start := f.c start.legacyStart = f.u.Start opts := []release.PassOption{release.WithRunningVersion(f.c.runningVersion), release.WithRestartFailurePropagation(), release.WithStartUnit(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, start)), release.WithRollbackRestart(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c))} return release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, f.restart(), append(opts, extra...)...) } func (f *r2Live) wantLive(t *testing.T, v string) { t.Helper() got, err := release.LiveVersion(f.u.opt, f.rt.Host) if err != nil || got != v { t.Fatalf("live=%s, err=%v; want %s", got, err, v) } } func (f *r2Live) wantRunning(t *testing.T, v string) { t.Helper() if f.u.props.MainPID <= 0 || f.u.exe != release.VersionPath(f.u.opt, "", v, f.rt.App) { t.Fatalf("not running %s: %+v", v, f.u) } } func TestR2UnguardedConsoleUpdate(t *testing.T) { if !plan14RootFixture(t) { return } for _, kind := range []string{"none", "list"} { t.Run(kind, func(t *testing.T) { f := r2Fixture(t, kind) var logged bytes.Buffer previousLog := log.Writer() log.SetOutput(&logged) defer log.SetOutput(previousLog) if err := f.pass(t); err != nil { t.Fatal(err) } f.wantLive(t, "v2") f.wantRunning(t, "v2") if len(f.u.effects) != 1 { t.Fatal(f.u.effects) } // The same callback serves config sync and restart-only requests. rt := f.rt rt.Backing = registry.BackingUnit if err := f.e.root.reg.Upsert(rt); err != nil { t.Fatal(err) } if err := appconfig.SetEnv(t.Context(), f.e.store, "bab", rt.App, map[string]string{"LOG_LEVEL": "debug"}); err != nil { t.Fatal(err) } dir := t.TempDir() for i := 0; i < 2; i++ { if err := appconfig.Sync(t.Context(), f.e.root.reg, f.e.store, nil, dir, f.restart(), nil); err != nil { t.Fatal(err) } } if len(f.u.effects) != 2 { t.Fatal("config sync missed or repeated restart", f.u.effects) } routes, err := f.e.root.reg.List() if err != nil { t.Fatal(err) } marks := filepath.Join(t.TempDir(), "restart.json") restartPass(t.Context(), routes, f.e.store, marks, f.restart()) askRestart(t, t.Context(), f.e.store, "bab", rt.App) restartPass(t.Context(), routes, f.e.store, marks, f.restart()) restartPass(t.Context(), routes, f.e.store, marks, f.restart()) if len(f.u.effects) != 3 { t.Fatal("restart-only missed or repeated restart", f.u.effects) } if kind == "none" && strings.Count(logged.String(), "legacy restart (no root census)") != 1 { t.Fatal("legacy restart diagnostic was not once per pass", logged.String()) } }) } } func TestR2UnguardedIncidentRecipe(t *testing.T) { if !plan14RootFixture(t) { return } for _, kind := range []string{"none", "guard", "list"} { for _, order := range []string{"reset before adoption", "reset after adoption", "never failed"} { t.Run(kind+"/"+order, func(t *testing.T) { f := r2Fixture(t, kind) if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil { t.Fatal(err) } if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil { t.Fatal(err) } if err := f.e.root.reg.SetVersions(f.rt.Host, "v2", "v1"); err != nil { t.Fatal(err) } f.u.exe = release.VersionPath(f.u.opt, "", "v2", f.rt.App) f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = "failed", "failed", 0 if order == "never failed" { f.u.props.ActiveState, f.u.props.SubState = "inactive", "dead" } if order == "reset before adoption" { if err := f.u.ResetFailed(f.rt.Unit); err != nil { t.Fatal(err) } } before := len(f.u.effects) if kind != "none" { if _, err := f.source.forHost(t.Context(), f.rt.Host); err != nil { t.Fatal("PID-zero binding rejected", err) } } f.promote(t, "v1") if err := f.pass(t); err != nil { t.Fatal(err) } f.wantLive(t, "v1") if order != "reset after adoption" && (f.u.props.MainPID != 0 || len(f.u.effects) != before) { t.Fatal("woke inactive unit", f.u.effects) } if order == "reset after adoption" { if err := f.u.ResetFailed(f.rt.Unit); err != nil { t.Fatal(err) } } if err := f.u.Restart(f.rt.Unit); err != nil { t.Fatal(err) } f.wantRunning(t, "v1") }) } } } func TestR2GuardedAutoRestartFixForward(t *testing.T) { if !plan14RootFixture(t) { return } for _, state := range []unitProps{{ActiveState: "activating", SubState: "auto-restart"}, {ActiveState: "activating", SubState: "start"}} { t.Run(state.SubState, func(t *testing.T) { f := r2Fixture(t, "guard") f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = state.ActiveState, state.SubState, 0 if err := f.pass(t); err != nil { t.Fatal(err) } f.wantLive(t, "v2") f.wantRunning(t, "v2") // Without a census, main try-restarts the activating unit once. if len(f.u.effects) != 1 || len(f.u.tries) != 1 || f.u.tries[0] != "console.service" { t.Fatal(f.u.effects, f.u.tries) } }) } } func TestR2BoxLocalUnitStamp(t *testing.T) { if !plan14RootFixture(t) { return } for _, recordUnit := range []string{"", "custom.service"} { t.Run(recordUnit, func(t *testing.T) { f := r2Fixture(t, "none") // A socket-derived record cannot authorize a custom named unit. The // no-census fallback still uses main's box-local stamp. plan14PutRecord(t, f.e.store, provision.Record{Schema: 1, Host: f.rt.Host, Account: "bab", App: f.rt.App, Channel: f.rt.Channel, Unit: recordUnit}) f.rt.Unit = "titogo-canary@t.service" f.u.props.Name = f.rt.Unit if err := f.e.root.reg.Upsert(f.rt); err != nil { t.Fatal(err) } if err := f.pass(t); err != nil { t.Fatal(err) } f.wantRunning(t, "v2") if len(f.u.tries) != 1 || f.u.tries[0] != f.rt.Unit { t.Fatal(f.u.tries) } }) } } type r2ListFailure struct { release.Store lists int } func (s *r2ListFailure) List(context.Context, string) ([]string, error) { s.lists++ return nil, os.ErrPermission } func TestR2OrdinaryRestartWithListFailure(t *testing.T) { if !plan14RootFixture(t) { return } for _, kind := range []string{"unit", "exec", "derived stamp"} { t.Run(kind, func(t *testing.T) { f := r2Fixture(t, "none") st := &r2ListFailure{Store: f.e.store} f.source.store = st f.rt.Unit = "" f.rt.Backing = registry.BackingUnit if kind == "exec" { f.rt.Backing = "exec" } if kind == "derived stamp" { f.rt.Unit = activator.UnitName(f.rt.Host) } if err := f.e.root.reg.Delete(f.rt.Host); err != nil { t.Fatal(err) } if err := f.e.root.reg.Upsert(f.rt); err != nil { t.Fatal(err) } if err := f.pass(t); err != nil { t.Fatal(err) } f.wantLive(t, "v2") f.wantRunning(t, "v2") if st.lists != 0 || len(f.u.tries) != 1 || f.u.tries[0] != activator.UnitName(f.rt.Host) { t.Fatal(st.lists, f.u.tries) } }) } } func TestR2NamedRestartWithListFailure(t *testing.T) { if !plan14RootFixture(t) { return } f := r2Fixture(t, "none") f.source.store = &r2ListFailure{Store: f.e.store} if err := f.pass(t); err != nil { t.Fatal(err) } f.wantLive(t, "v2") f.wantRunning(t, "v2") } func TestR2OrdinaryRestartMatchesMain(t *testing.T) { t.Parallel() for _, scenario := range []string{"show error", "exec mismatch", "activating", "inactive", "missing row", "registry error", "try-restart error"} { t.Run(scenario, func(t *testing.T) { e, rt, base, c, _ := plan14AdoptionFixture(t) u := &r2Units{plan14RestartUnits: base} rt.Unit = "" rt.Backing = registry.BackingUnit if err := e.root.reg.Delete(rt.Host); err != nil { t.Fatal(err) } if err := e.root.reg.Upsert(rt); err != nil { t.Fatal(err) } c.legacy = func() (bool, error) { return true, nil } c.dispatch = func(context.Context, registry.Route) (bool, error) { t.Fatal("ordinary restart consulted records") return false, nil } switch scenario { case "show error": u.showErr = os.ErrPermission case "exec mismatch": u.props.ExecPath = "/different" case "activating": u.props.ActiveState = "activating" case "inactive": u.props.ActiveState = "inactive" case "missing row": if err := e.root.reg.Delete(rt.Host); err != nil { t.Fatal(err) } case "registry error": e.root.reg.Close() case "try-restart error": u.tryErr = os.ErrPermission } if err := rootRestartRouteWith(e.root.reg, u, u.opt, nil, nil, c)(rt.Host); err != nil { t.Fatal(err) } if len(u.tries) != 1 || u.tries[0] != activator.UnitName(rt.Host) { t.Fatal(u.tries) } if scenario == "inactive" && len(u.effects) != 0 { t.Fatal("woke inactive ordinary unit") } if err := rollbackUnitRestart(e.root.reg, u, u.opt, nil, c)(rt.Host); err != nil { t.Fatal("ordinary rollback reports outage", err) } }) } } func TestR2BindingRefusalKeepsPromotedFix(t *testing.T) { if !plan14RootFixture(t) { return } for _, during := range []bool{false, true} { t.Run(map[bool]string{false: "before restart", true: "after restart"}[during], func(t *testing.T) { f := r2Fixture(t, "list") if during { f.u.afterRestart = func() { f.source.namedPath = filepath.Join(t.TempDir(), "missing") } } else { f.source.unitDirs = []string{"/wrong"} } if err := f.pass(t); err != nil { t.Fatal(err) } f.wantLive(t, "v2") f.wantRunning(t, "v2") }) } } func TestR2CrashBeforeRestartCapture(t *testing.T) { if !plan14RootFixture(t) { return } // Merge ruling 6: without a census the rollback target is main's cur // (the interrupted repoint's v2); only a census-backed box retains the // verified running build (v1) as its rollback target. for _, mode := range []string{"no census", "census"} { for _, fail := range []bool{false, true} { t.Run(mode+"/"+map[bool]string{false: "forward", true: "rollback"}[fail], func(t *testing.T) { f := r2Fixture(t, "list") if mode == "census" { f.c.legacy = func() (bool, error) { return false, nil } } // Main's rollback target, and the retained one with a census. target := map[string]string{"no census": "v2", "census": "v1"}[mode] crashed := errors.New("crash after repoint before restart capture") func() { defer func() { if recover() != crashed { t.Fatal("missing crash") } }() _ = release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, func(string) error { panic(crashed) }, release.WithRunningVersion(f.c.runningVersion)) }() f.wantLive(t, "v2") f.wantRunning(t, "v1") state, err := f.c.attempts() if err != nil { t.Fatal(err) } if _, err := readRollbackAttempt(state, f.rt.Host); !os.IsNotExist(err) { t.Fatal("attempt already captured", err) } m := release.Manifest{App: f.rt.App, Version: "v3", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} if err := release.Ship(t.Context(), f.e.store, "bab", m, map[string][]byte{release.AdoptPlatform(): []byte("third build")}); err != nil { t.Fatal(err) } f.promote(t, "v3") // A new resolver and restart callback model the next root process. f.source.index = nil if fail { f.u.afterEffect = func() { if f.u.exe == release.VersionPath(f.u.opt, "", "v3", f.rt.App) { f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = "failed", "failed", 0 } } } err = f.pass(t) if fail { if err == nil { t.Fatal("failed build accepted") } f.wantLive(t, target) f.wantRunning(t, target) } else { if err != nil { t.Fatal(err) } f.wantLive(t, "v3") f.wantRunning(t, "v3") prior, err := os.Readlink(filepath.Join(f.u.opt, "previous", f.rt.Host)) if err != nil || prior != release.VersionPath(f.u.opt, "", target, f.rt.App) { t.Fatal("previous pointer", prior, err) } pins, err := release.LivePins(f.u.opt) if err != nil || !pins[target] { t.Fatal("rollback build not pinned", pins, err) } } }) } } } func TestR2RetainedCensusKeepsStrictRestart(t *testing.T) { if !plan14RootFixture(t) { return } for _, body := range []string{"{}", "corrupt"} { t.Run(body, func(t *testing.T) { f := r2Fixture(t, "none") s, err := rootstate.Open(rootparticipation.Namespace, true, false) if err != nil { t.Fatal(err) } if err := s.Write("census.json", []byte(body)); err != nil { t.Fatal(err) } defer func() { s.Remove("census.json"); s.Close() }() if legacy, err := legacyRootRestart(); err != nil || legacy { t.Fatal("retained census selected legacy", legacy, err) } if err := f.restart()(f.rt.Host); err == nil || len(f.u.effects) != 0 { t.Fatal("unbound restart passed retained census", err, f.u.effects) } }) } } func TestR2UnguardedCrashRollback(t *testing.T) { if !plan14RootFixture(t) { return } f := r2Fixture(t, "none") f.u.afterEffect = func() { if f.u.exe == release.VersionPath(f.u.opt, "", "v2", f.rt.App) { f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = "failed", "failed", 0 } } if err := f.pass(t); err == nil { t.Fatal("crashed build accepted") } f.wantLive(t, "v1") f.wantRunning(t, "v1") if len(f.u.effects) != 3 { t.Fatal("rollback did not reset and restart", f.u.effects) } } func TestR2NamedFallbackMatchesMain(t *testing.T) { if !plan14RootFixture(t) { return } for _, scenario := range []string{"show error", "exec mismatch", "inactive", "failed", "activating", "try-restart error"} { t.Run(scenario, func(t *testing.T) { f := r2Fixture(t, "none") switch scenario { case "show error": f.u.showErr = os.ErrPermission case "exec mismatch": f.u.props.ExecPath = "/other" case "inactive", "failed", "activating": f.u.props.ActiveState = scenario f.u.props.MainPID = 0 case "try-restart error": f.u.tryErr = os.ErrPermission } err := f.restart()(f.rt.Host) if (scenario == "try-restart error") != errors.Is(err, os.ErrPermission) { t.Fatal("fallback changed try-restart result", err) } if len(f.u.tries) != 1 || f.u.tries[0] != f.rt.Unit { t.Fatal(f.u.tries) } if (scenario == "inactive" || scenario == "failed") && len(f.u.effects) != 0 { t.Fatal("try-restart woke stopped unit") } if scenario == "try-restart error" { starts := 0 f.c.legacyStart = func(unit string) error { if unit != f.rt.Unit { t.Fatal(unit) } starts++ return nil } if err := rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, nil, f.c)(f.rt.Host); err != nil || starts != 1 { t.Fatal("non-NotUp legacy recovery did not use main start", err, starts) } } }) } }