package host import ( "context" "errors" "os" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootparticipation" ) func (f *r2Live) productionPass(t *testing.T) error { t.Helper() start := f.c start.legacyStart = f.u.Start observer := f.e.root.obs observer.resolve, observer.sys = f.c.resolve, f.u observer.procExe = func(int) string { return f.u.exe } pr := f.e.root pr.obs = observer return pr.around(t.Context(), func() error { restart := rootRestartRouteWith(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, observer.recordSettled(f.e.bake), f.c) return release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, restart, release.WithRunningVersion(f.c.runningVersion), release.WithRestartAttemptSettled(f.c.finish), release.WithStartUnit(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, start)), release.WithRollbackRestart(rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c))) }) } func r3RestartCount(u *r2Units) int { n := 0 for _, effect := range u.effects { if strings.HasPrefix(effect, "restart ") { n++ } } return n } func TestR3StickyBindingErrorMasksCrash(t *testing.T) { if !plan14RootFixture(t) { return } for _, kind := range []string{"none", "list"} { for _, state := range []string{"failed", "auto-restart", "healthy"} { t.Run(kind+"/"+state, func(t *testing.T) { f := r2Fixture(t, kind) original := f.c.host f.c.host = func(ctx context.Context, host string) (proofBinding, error) { if r3RestartCount(f.u) > 0 && strings.HasSuffix(f.u.exe, "/v2/console") { return proofBinding{}, os.ErrPermission } return original(ctx, host) } ready := func(registry.Route) func() error { return func() error { if f.u.exe == release.VersionPath(f.u.opt, "", "v2", f.rt.App) && state != "healthy" { return os.ErrDeadlineExceeded } return nil } } f.c.ready, f.c.legacyReady = ready, ready sleep := f.c.sleep f.c.sleep = func(d time.Duration) { sleep(d) if f.u.exe != release.VersionPath(f.u.opt, "", "v2", f.rt.App) || state == "healthy" { return } f.u.props.MainPID = 0 if state == "failed" { f.u.props.ActiveState, f.u.props.SubState = "failed", "failed" } else { f.u.props.ActiveState, f.u.props.SubState = "activating", "auto-restart" } } if err := f.productionPass(t); err != nil { t.Fatal("per-host failure changed main's pass status", err) } want, restarts := "v1", 2 if state == "healthy" { want, restarts = "v2", 1 } f.wantLive(t, want) f.wantRunning(t, want) if got := r3RestartCount(f.u); got != restarts { t.Fatalf("restarts=%d want=%d: %v", got, restarts, f.u.effects) } }) } } } func TestR3FallbackDoubleRestart(t *testing.T) { if !plan14RootFixture(t) { return } for _, rollback := range []bool{false, true} { for _, journalFailure := range []bool{false, true} { t.Run(map[bool]string{false: "restart", true: "rollback"}[rollback]+map[bool]string{false: "/binding", true: "/journal"}[journalFailure], func(t *testing.T) { f := r2Fixture(t, "list") initial, err := f.c.resolve(t.Context(), f.rt) if err != nil { t.Fatal(err) } if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil { t.Fatal(err) } if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil { t.Fatal(err) } if rollback { f.u.crashNew = true } original := f.c.host f.c.host = func(ctx context.Context, host string) (proofBinding, error) { if r3RestartCount(f.u) > 0 { return proofBinding{}, os.ErrPermission } return original(ctx, host) } if journalFailure { f.c.host = original store, err := f.c.attempts() if err != nil { t.Fatal(err) } memory := store.(*rollbackMemory) f.u.afterEffect = func() { memory.failWrite = true } } err = f.restart()(f.rt.Host) if !rollback { if err != nil { t.Fatal(err) } f.wantRunning(t, "v2") if r3RestartCount(f.u) != 1 { t.Fatal(f.u.effects) } return } var notUp *release.UnitNotUpError if !errors.As(err, ¬Up) { t.Fatal("missing crash verdict", err) } if err := release.PointLive(f.u.opt, f.rt.Host, "", "v1", f.rt.App); err != nil { t.Fatal(err) } if journalFailure { store, _ := f.c.attempts() store.(*rollbackMemory).failWrite = false } f.c.host = original f.c.rollback = func(ctx context.Context, b proofBinding) (proofBinding, error) { return f.source.rollback(ctx, initial) } f.u.afterEffect = nil if journalFailure { store, _ := f.c.attempts() f.u.afterEffect = func() { store.(*rollbackMemory).failWrite = true } } // Refuse identity after the rollback effect, inside the callback's copy. host := f.c.host f.c.host = func(ctx context.Context, h string) (proofBinding, error) { if r3RestartCount(f.u) >= 2 { return proofBinding{}, os.ErrPermission } return host(ctx, h) } before := r3RestartCount(f.u) if err := rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c)(f.rt.Host); err != nil { t.Fatal(err) } if r3RestartCount(f.u)-before != 1 { t.Fatal(f.u.effects) } f.wantLive(t, "v1") f.wantRunning(t, "v1") }) } } } type r3ExeUnits struct { *r2Units denied bool } func (u *r3ExeUnits) Exe(pid int) (string, error) { if u.denied { return "", os.ErrPermission } return u.r2Units.Exe(pid) } func TestPlan14Fix4ExeDeniedAfterRestart(t *testing.T) { if !plan14RootFixture(t) { return } f := r2Fixture(t, "list") u := &r3ExeUnits{r2Units: f.u} f.source.sys = u f.u.afterEffect = func() { u.denied = true } if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil { t.Fatal(err) } if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil { t.Fatal(err) } if err := rootRestartRouteWith(f.e.root.reg, u, f.u.opt, func(string) string { return "5s" }, nil, f.c)(f.rt.Host); err != nil { t.Fatal(err) } f.wantLive(t, "v2") f.wantRunning(t, "v2") if r3RestartCount(f.u) != 1 { t.Fatal(f.u.effects) } } func TestR3CensusProbeErrorBlocksOrdinary(t *testing.T) { if !plan14RootFixture(t) { return } if err := os.MkdirAll(rootparticipation.Namespace, 0770); err != nil { t.Fatal(err) } if err := os.Chmod(rootparticipation.Namespace, 0770); err != nil { t.Fatal(err) } if _, err := legacyRootRestart(); err == nil { t.Fatal("unreadable census fixture did not refuse") } for _, stamped := range []bool{false, true} { t.Run(map[bool]string{false: "unstamped", true: "derived"}[stamped], func(t *testing.T) { f := r2Fixture(t, "none") f.rt.Unit = "" if stamped { f.rt.Unit = activator.UnitName(f.rt.Host) } if err := f.e.root.reg.Delete(f.rt.Host); err != nil { t.Fatal(err) } if err := f.e.root.reg.Upsert(f.rt); err != nil { t.Fatal(err) } f.c.legacy = func() (bool, error) { t.Fatal("ordinary route probed census"); return false, os.ErrPermission } if err := f.productionPass(t); err != nil { t.Fatal(err) } f.wantLive(t, "v2") f.wantRunning(t, "v2") if err := rollbackUnitRestart(f.e.root.reg, f.u, f.u.opt, func(string) string { return "5s" }, f.c)(f.rt.Host); err != nil { t.Fatal(err) } }) } } func TestR3PreviousPins(t *testing.T) { if !plan14RootFixture(t) { return } t.Run("relative live link", func(t *testing.T) { f := r2Fixture(t, "list") link := release.LiveLink(f.u.opt, f.rt.Host) relative, err := filepath.Rel(filepath.Dir(link), f.u.exe) if err != nil { t.Fatal(err) } if err := os.Remove(link); err != nil { t.Fatal(err) } if err := os.Symlink(relative, link); err != nil { t.Fatal(err) } if err := f.productionPass(t); err != nil { t.Fatal(err) } f.wantLive(t, "v2") f.wantRunning(t, "v2") prior, err := filepath.EvalSymlinks(filepath.Join(f.u.opt, "previous", f.rt.Host)) if err != nil || prior != release.VersionPath(f.u.opt, "", "v1", f.rt.App) { t.Fatal(prior, err) } }) } type r3DiscoveryStore struct { release.Store lists, gets int blocked <-chan struct{} } func (s *r3DiscoveryStore) List(ctx context.Context, prefix string) ([]string, error) { s.lists++ if s.blocked != nil { <-s.blocked } return nil, os.ErrPermission } func (s *r3DiscoveryStore) Get(ctx context.Context, key string) ([]byte, string, error) { s.gets++ return s.Store.Get(ctx, key) } func TestPlan14Fix4OrdinaryProductionWrapper(t *testing.T) { if !plan14RootFixture(t) { return } for _, kind := range []string{"unit", "exec", "derived", "hibernate"} { for _, stall := range []bool{false, true} { t.Run(kind+map[bool]string{false: "/error", true: "/hang"}[stall], func(t *testing.T) { f := r2Fixture(t, "none") f.rt.Unit = "" f.rt.Backing = registry.BackingUnit f.rt.Socket = "/run/ordinary.sock" f.rt.Addr, f.rt.Health = "", "" if kind == "exec" { f.rt.Backing = "exec" } if kind == "derived" { f.rt.Unit = activator.UnitName(f.rt.Host) } if kind == "hibernate" { f.rt.Hibernate = true } if err := f.e.root.reg.Delete(f.rt.Host); err != nil { t.Fatal(err) } if err := f.e.root.reg.Upsert(f.rt); err != nil { t.Fatal(err) } store := &r3DiscoveryStore{Store: f.e.store} if stall { unblock := make(chan struct{}) store.blocked = unblock timer := time.AfterFunc(time.Second, func() { close(unblock) }) defer timer.Stop() } f.source.store = store if err := f.productionPass(t); err != nil { t.Fatal(err) } if store.lists != 0 || store.gets != 0 { t.Fatalf("ordinary proof wrapper read records: lists=%d gets=%d", store.lists, store.gets) } f.wantLive(t, "v2") if kind == "hibernate" { if r3RestartCount(f.u) != 0 { t.Fatal("hibernating route restarted", f.u.effects) } } else { f.wantRunning(t, "v2") } }) } } } func TestPlan14Fix4IndexFailureCachedForPass(t *testing.T) { if !plan14RootFixture(t) { return } f := r2Fixture(t, "none") store := &r3DiscoveryStore{Store: f.e.store} f.source.store = store other := f.rt other.Host = "second.example.com" if err := f.e.root.reg.Upsert(other); err != nil { t.Fatal(err) } // Observation, adoption, readiness recording and minting share one resolver. if err := f.productionPass(t); err != nil { t.Fatal(err) } if store.lists != 1 { t.Fatalf("index attempts=%d want=1", store.lists) } next := productionProofBindingSource(store, f.u, f.u.opt) if _, err := next.resolve(t.Context(), f.rt); err == nil { t.Fatal("failed discovery accepted") } if store.lists != 2 { t.Fatalf("new pass did not retry: %d", store.lists) } } func TestPlan14Fix4HibernateStartGuard(t *testing.T) { t.Parallel() e, rt, u, c, _ := plan14AdoptionFixture(t) rt.Hibernate = true if err := e.root.reg.Upsert(rt); err != nil { t.Fatal(err) } c.legacyStart = func(string) error { t.Fatal("woke hibernating route"); return nil } c.legacy = func() (bool, error) { t.Fatal("hibernating start read census"); return false, os.ErrPermission } if err := rollbackUnitRestart(e.root.reg, u, u.opt, nil, c)(rt.Host); err != nil { t.Fatal(err) } if len(u.effects) != 0 { t.Fatal(u.effects) } } func TestPlan14Fix4NotUpWinsOverBinding(t *testing.T) { if !restartEffectFailed(errors.Join(errRestartBinding, &release.UnitNotUpError{Unit: "console.service"})) { t.Fatal("binding masked D1 failure") } } func TestPlan14Fix4IncidentProductionWiring(t *testing.T) { if !plan14RootFixture(t) { return } for _, kind := range []string{"none", "guard", "list"} { for _, order := range []string{"reset before adoption", "reset after adoption", "never failed"} { t.Run(kind+"/"+order, func(t *testing.T) { f := r2Fixture(t, kind) if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil { t.Fatal(err) } if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil { t.Fatal(err) } if err := f.e.root.reg.SetVersions(f.rt.Host, "v2", "v1"); err != nil { t.Fatal(err) } f.u.exe = release.VersionPath(f.u.opt, "", "v2", f.rt.App) f.u.props.ActiveState, f.u.props.SubState, f.u.props.MainPID = "failed", "failed", 0 if order == "never failed" { f.u.props.ActiveState, f.u.props.SubState = "inactive", "dead" } if order == "reset before adoption" { if err := f.u.ResetFailed(f.rt.Unit); err != nil { t.Fatal(err) } } before := len(f.u.effects) if kind != "none" { if _, err := f.source.forHost(t.Context(), f.rt.Host); err != nil { t.Fatal("PID-zero binding rejected", err) } } f.promote(t, "v1") if err := f.productionPass(t); err != nil { t.Fatal(err) } f.wantLive(t, "v1") if len(f.u.starts) != 0 { t.Fatal("adoption called start", f.u.starts) } if order != "reset after adoption" && (f.u.props.MainPID != 0 || len(f.u.effects) != before) { t.Fatal("woke inactive unit", f.u.effects) } if order == "reset after adoption" { if err := f.u.ResetFailed(f.rt.Unit); err != nil { t.Fatal(err) } } if err := f.u.Restart(f.rt.Unit); err != nil { t.Fatal(err) } f.wantRunning(t, "v1") }) } } } func TestPlan14Fix4RepointFailureStillLogged(t *testing.T) { if !plan14RootFixture(t) { return } f := r2Fixture(t, "list") if err := os.MkdirAll(filepath.Join(f.u.opt, "previous", f.rt.Host), 0755); err != nil { t.Fatal(err) } output, err := os.CreateTemp(t.TempDir(), "pass-output") if err != nil { t.Fatal(err) } original := os.Stdout os.Stdout = output defer func() { os.Stdout = original; output.Close() }() err = f.productionPass(t) os.Stdout = original if err != nil { t.Fatal("repoint failure changed pass status", err) } if err := output.Close(); err != nil { t.Fatal(err) } raw, err := os.ReadFile(output.Name()) if err != nil { t.Fatal(err) } if !strings.Contains(string(raw), "repoint failed:") { t.Fatal("missing repoint diagnostic", string(raw)) } f.wantLive(t, "v1") f.wantRunning(t, "v1") if r3RestartCount(f.u) != 0 { t.Fatal(f.u.effects) } } type r3ResetErrorUnits struct { *r2Units resets int } func (u *r3ResetErrorUnits) ResetFailed(string) error { u.resets++; return os.ErrPermission } func TestPlan14Fix4RollbackResetFailureMatchesMain(t *testing.T) { if !plan14RootFixture(t) { return } f := r2Fixture(t, "list") u := &r3ResetErrorUnits{r2Units: f.u} f.source.sys = u if err := release.StageArtifact(u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil { t.Fatal(err) } if err := release.PointLive(u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil { t.Fatal(err) } u.crashNew = true err := rootRestartRouteWith(f.e.root.reg, u, u.opt, func(string) string { return "5s" }, nil, f.c)(f.rt.Host) var notUp *release.UnitNotUpError if !errors.As(err, ¬Up) { t.Fatal("missing crash verdict", err) } if err := release.PointLive(u.opt, f.rt.Host, "", "v1", f.rt.App); err != nil { t.Fatal(err) } if err := rollbackUnitRestart(f.e.root.reg, u, u.opt, func(string) string { return "5s" }, f.c)(f.rt.Host); err != nil { t.Fatal(err) } if u.resets != 1 || r3RestartCount(f.u) != 2 { t.Fatal("reset failure repeated or prevented restart", u.resets, u.effects) } f.wantLive(t, "v1") f.wantRunning(t, "v1") }