package host import ( "context" "crypto/ed25519" "errors" "os" "path/filepath" "strconv" "sync" "testing" "time" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootstate" ) type plan14RestartUnits struct { props unitProps exe string opt string effects []string crashNew bool badRollback bool failNew bool afterEffect func() afterRestart func() } func (u *plan14RestartUnits) StartTime(pid int) (string, error) { return strconv.Itoa(pid), nil } func (u *plan14RestartUnits) Show(string) (unitProps, error) { return u.props, nil } func (u *plan14RestartUnits) Exe(int) (string, error) { return u.exe, nil } func (u *plan14RestartUnits) JournalTail(string, int) string { return "" } func (u *plan14RestartUnits) ResetFailed(unit string) error { u.effects = append(u.effects, "reset "+unit) return nil } func (u *plan14RestartUnits) TryRestart(unit string) error { return u.Restart(unit) } func (u *plan14RestartUnits) Restart(unit string) error { u.effects = append(u.effects, "restart "+unit) if u.afterRestart != nil { u.afterRestart() } u.props.InvocationID += "x" u.props.MainPID++ u.props.ActiveState, u.props.SubState = "active", "running" var err error u.exe, err = filepath.EvalSymlinks(release.LiveLink(u.opt, observedHost)) if u.crashNew && u.exe == release.VersionPath(u.opt, "bab", "v2", "console") { u.props.ActiveState, u.props.SubState, u.props.MainPID = "failed", "failed", 0 } if u.badRollback && u.exe == release.VersionPath(u.opt, "bab", "v1", "console") { u.exe = release.VersionPath(u.opt, "bab", "v2", "console") } if u.failNew && u.exe == release.VersionPath(u.opt, "bab", "v2", "console") { return os.ErrPermission } if u.afterEffect != nil { u.afterEffect() } return err } func TestPlan14CrashedNamedUnitRollsBack(t *testing.T) { t.Parallel() e, _, units, c, pub := plan14AdoptionFixture(t) units.crashNew = true forward := rootRestartRouteWith(e.root.reg, units, units.opt, func(string) string { return "5s" }, e.root.obs.recordSettled(e.bake), c) var rollbackErr error rollback := rollbackUnitRestart(e.root.reg, units, units.opt, func(string) string { return "5s" }, c) err := release.Pass(context.Background(), e.root.reg, e.store, pub, units.opt, forward, release.BakeDir(e.bake), release.WithRestartFailurePropagation(), release.WithRollbackRestart(func(host string) error { rollbackErr = rollback(host); return rollbackErr })) if rollbackErr != nil || err == nil { t.Fatalf("crash recovery: %v / %v", err, rollbackErr) } if len(units.effects) != 3 || units.props.MainPID <= 0 || units.exe != release.VersionPath(units.opt, "bab", "v1", "console") { t.Fatalf("rollback was not verified: %+v", units) } if version, err := release.LiveVersion(units.opt, observedHost); err != nil || version != "v1" { t.Fatalf("restored link = %q, %v", version, err) } if got := release.ReadHealthy(e.bake, "bab", "console"); len(got) != 1 { t.Fatalf("crash recorded health: %+v", got) } } func plan14AdoptionFixture(t *testing.T) (*proofRootEnv, registry.Route, *plan14RestartUnits, rootRestartChecks, ed25519.PublicKey) { t.Helper() e, rt, rec, source := plan14ProofFixture(t) if got := e.root.observeAndRecord(rt, 15*time.Minute); got.err != nil { t.Fatal(got.err) } source.healthy = func(acct, app, version, digest string) *release.HealthyBuild { for _, b := range release.ReadHealthy(e.bake, acct, app) { if b.SHA256 == digest { return &b } } return nil } u := &plan14RestartUnits{props: source.sys.(*fakeUnits).shows[0], exe: release.VersionPath(source.optDir, "bab", "v1", "console"), opt: source.optDir} source.sys, e.root.obs.sys = u, u e.root.obs.procExe = func(int) string { return u.exe } rec.Pin = "v2" plan14PutRecord(t, e.store, rec) ctx := context.Background() pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } for ver, body := range map[string]string{"v1": "x", "v2": "new build"} { m := release.Manifest{App: "console", Version: ver, Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} if err := release.Ship(ctx, e.store, "bab", m, map[string][]byte{release.AdoptPlatform(): []byte(body)}); err != nil { t.Fatal(err) } } if err := release.Promote(ctx, e.store, priv, "bab", "console", rt.Channel, "v2"); err != nil { t.Fatal(err) } now, sleep := settleClock() mem := &rollbackMemory{files: map[string][]byte{}} c := rootRestartChecks{targetFile: os.Open, started: &sync.Map{}, attempts: func() (rootstate.Store, error) { return mem, nil }, rollback: source.rollback, prepareRollback: source.prepareRollback, resolve: source.resolve, host: source.forHost, ready: e.root.obs.ready, now: now, sleep: sleep} return e, rt, u, c, pub } func TestPlan14AdoptionCanonicalReadiness(t *testing.T) { t.Parallel() mutations := map[string]func(*registry.Route){ "unchanged": func(*registry.Route) {}, "socket": func(r *registry.Route) { r.Socket = "/run/forged.sock" }, "addr": func(r *registry.Route) { r.Addr = "" }, "addrs": func(r *registry.Route) { r.Addrs = "127.0.0.1:1" }, "health": func(r *registry.Route) { r.Health = "" }, "unit": func(r *registry.Route) { r.Unit = "" }, "backing": func(r *registry.Route) { r.Backing = "exec" }, "empty upstream": func(r *registry.Route) { r.Socket, r.Addr, r.Addrs, r.Health = "", "", "", "" }, } for name, mutate := range mutations { t.Run(name, func(t *testing.T) { e, rt, units, c, pub := plan14AdoptionFixture(t) mutate(&rt) if err := e.root.reg.Upsert(rt); err != nil { t.Fatal(err) } c.ready = func(rt registry.Route) func() error { if rt.Addr == "" && rt.Socket == "" { return nil } return func() error { return os.ErrPermission } } e.root.obs.ready = c.ready restart := rootRestartRouteWith(e.root.reg, units, units.opt, func(string) string { return "5s" }, e.root.obs.recordSettled(e.bake), c) err := release.Pass(context.Background(), e.root.reg, e.store, pub, units.opt, restart, release.BakeDir(e.bake), release.WithRestartFailurePropagation(), release.WithRollbackRestart(rollbackUnitRestart(e.root.reg, units, units.opt, func(string) string { return "5s" }, c))) if err == nil { t.Fatal("adoption swallowed the canonical readiness refusal") } got, getErr := e.root.reg.Get(rt.Host) if getErr != nil || got.Version == "v2" { t.Fatalf("stamped adoption: %+v, %v", got, getErr) } if history := release.ReadHealthy(e.bake, "bab", "console"); len(history) != 1 || history[0].Version != "v1" { t.Fatalf("new health: %+v", history) } }) } t.Run("health write cannot block restart", func(t *testing.T) { e, _, units, c, pub := plan14AdoptionFixture(t) sentinel := errors.New("settled witness") restart := rootRestartRouteWith(e.root.reg, units, units.opt, func(string) string { return "5s" }, func(registry.Route) error { return sentinel }, c) err := release.Pass(context.Background(), e.root.reg, e.store, pub, units.opt, restart, release.WithRestartFailurePropagation()) if err != nil { t.Fatalf("health write blocked restart: %v", err) } }) t.Run("positive adoption", func(t *testing.T) { e, _, units, c, pub := plan14AdoptionFixture(t) restart := rootRestartRouteWith(e.root.reg, units, units.opt, func(string) string { return "5s" }, e.root.obs.recordSettled(e.bake), c) if err := release.Pass(context.Background(), e.root.reg, e.store, pub, units.opt, restart, release.WithRestartFailurePropagation()); err != nil { t.Fatal(err) } rt, err := e.root.reg.Get(observedHost) if err != nil || rt.Version != "v2" || len(release.ReadHealthy(e.bake, "bab", "console")) != 2 { t.Fatalf("canonical adoption failed: %+v, %v", rt, err) } }) } func TestPlan14RollbackCanonicalReadiness(t *testing.T) { t.Parallel() for _, attack := range []string{"missing", "unchanged", "account", "app", "unit", "backing", "socket", "addr", "addrs", "health", "all empty"} { for _, ready := range []bool{false, true} { name := attack + "/" + strconv.FormatBool(ready) t.Run(name, func(t *testing.T) { e, rt, units, c, pub := plan14AdoptionFixture(t) c.ready = func(r registry.Route) func() error { if r.Addr != "127.0.0.1:4000" || r.Health != rt.Health { t.Fatalf("noncanonical rollback readiness: %+v", r) } return func() error { if !ready || units.exe == release.VersionPath(units.opt, "bab", "v2", "console") { return os.ErrPermission } return nil } } forward := rootRestartRouteWith(e.root.reg, units, units.opt, func(string) string { return "5s" }, e.root.obs.recordSettled(e.bake), c) restart := func(host string) error { err := forward(host) if attack == "missing" { if e := e.root.reg.Delete(host); e != nil { t.Fatal(e) } } else { forged := rt switch attack { case "account": forged.Account = "bap" case "app": forged.App = "other" case "unit": forged.Unit = "other.service" case "backing": forged.Backing = registry.BackingUnit case "socket": forged.Socket = "/run/forged.sock" case "addr": forged.Addr = "" case "addrs": forged.Addrs = "127.0.0.1:1" case "health": forged.Health = `{"path":"/forged"}` case "all empty": forged.Unit, forged.Backing, forged.Socket, forged.Addr, forged.Addrs, forged.Health = "", "", "", "", "", "" } if e := e.root.reg.Upsert(forged); e != nil { t.Fatal(e) } } return err } var rollbackErr error rollback := rollbackUnitRestart(e.root.reg, units, units.opt, func(string) string { return "5s" }, c) err := release.Pass(context.Background(), e.root.reg, e.store, pub, units.opt, restart, release.BakeDir(e.bake), release.WithRestartFailurePropagation(), release.WithRollbackRestart(func(host string) error { rollbackErr = rollback(host); return rollbackErr })) if err == nil { t.Fatal("forward readiness failure disappeared") } if ready != (rollbackErr == nil) { t.Fatalf("rollback error = %v, readiness = %v", rollbackErr, ready) } if len(units.effects) != 3 { t.Fatalf("rollback did not use the authoritative unit: %v", units.effects) } if !ready && !errors.Is(err, rollbackErr) { t.Fatalf("rollback failure not reported: %v / %v", err, rollbackErr) } if history := release.ReadHealthy(e.bake, "bab", "console"); len(history) != 1 { t.Fatalf("rollback wrote health: %+v", history) } }) } } } func TestPlan14BindingChangesDuringRestart(t *testing.T) { t.Parallel() e, _, u, c, pub := plan14AdoptionFixture(t) u.afterRestart = func() { rec, _, err := provision.FindRecordByHost(context.Background(), e.store, observedHost) if err != nil { t.Fatal(err) } rec.RecordSeq++ plan14PutRecord(t, e.store, *rec) } forward := rootRestartRouteWith(e.root.reg, u, u.opt, func(string) string { return "5s" }, e.root.obs.recordSettled(e.bake), c) err := release.Pass(context.Background(), e.root.reg, e.store, pub, u.opt, forward, release.BakeDir(e.bake), release.WithRestartFailurePropagation(), release.WithRollbackRestart(rollbackUnitRestart(e.root.reg, u, u.opt, func(string) string { return "5s" }, c))) if err == nil || len(u.effects) != 1 || len(release.ReadHealthy(e.bake, "bab", "console")) != 1 { t.Fatalf("changed binding settled: %v, %v", err, u.effects) } } func TestPlan14RollbackInvocationChangesInProbe(t *testing.T) { t.Parallel() e, _, u, c, pub := plan14AdoptionFixture(t) u.crashNew = true c.ready = func(registry.Route) func() error { return func() error { u.props.InvocationID += "raced"; return nil } } forward := rootRestartRouteWith(e.root.reg, u, u.opt, func(string) string { return "5s" }, nil, c) var recovered error err := release.Pass(context.Background(), e.root.reg, e.store, pub, u.opt, forward, release.BakeDir(e.bake), release.WithRestartFailurePropagation(), release.WithRollbackRestart(func(host string) error { recovered = rollbackUnitRestart(e.root.reg, u, u.opt, func(string) string { return "5s" }, c)(host) return recovered })) if err == nil || recovered == nil { t.Fatal("changed invocation accepted as recovery") } }