package host import ( "context" "errors" "os" "path/filepath" "reflect" "strings" "testing" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootstate" ) // Each unit has separate process state. In particular, restarting the local // binding must never look like a restart of a different box-local stamp. type fix5Units struct { units map[string]*r2Units events []string resetError, tryError bool } func (u *fix5Units) Show(name string) (unitProps, error) { return u.units[name].Show(name) } func (u *fix5Units) Exe(pid int) (string, error) { for _, unit := range u.units { if unit.props.MainPID == pid { return unit.Exe(pid) } } return "", os.ErrNotExist } func (u *fix5Units) StartTime(pid int) (string, error) { for _, unit := range u.units { if unit.props.MainPID == pid { return unit.StartTime(pid) } } return "", os.ErrNotExist } func (u *fix5Units) JournalTail(string, int) string { return "" } func (u *fix5Units) TryRestart(name string) error { u.events = append(u.events, "try-restart "+name) if u.tryError { u.tryError = false return errors.New("try-restart: bus timeout") } return u.units[name].TryRestart(name) } func (u *fix5Units) Restart(name string) error { u.events = append(u.events, "restart "+name) return u.units[name].Restart(name) } func (u *fix5Units) ResetFailed(name string) error { u.events = append(u.events, "reset-failed "+name) if u.resetError { return errors.New("reset-failed: bus timeout") } return u.units[name].ResetFailed(name) } func (u *fix5Units) Start(name string) error { u.events = append(u.events, "start "+name) return u.units[name].Start(name) } type fix5Result struct { Events []string Live, Previous string Route registry.Route Processes map[string]string } func fix5Run(t *testing.T, kind, history string, main bool) []fix5Result { t.Helper() sourceKind := kind if kind == "record" { sourceKind = "none" } if kind == "different stamp" { sourceKind = "list" } f := r2Fixture(t, sourceKind) if kind == "record" { f.rt.Account = "bab" plan14PutRecord(t, f.e.store, provision.Record{Schema: 1, Host: f.rt.Host, Account: "bab", App: f.rt.App, Kind: f.rt.Kind, Unit: f.rt.Unit, Addr: f.rt.Addr, Channel: f.rt.Channel, Health: &provision.Health{Path: "/ready"}}) } u := &fix5Units{units: map[string]*r2Units{f.rt.Unit: f.u}} target := f.u if kind == "different stamp" { copyUnit := *f.u.plan14RestartUnits copyUnit.props.Name, copyUnit.props.MainPID = "stamped.service", 700 target = &r2Units{plan14RestartUnits: ©Unit} f.rt.Unit = copyUnit.props.Name u.units[f.rt.Unit] = target } if err := f.e.root.reg.Upsert(f.rt); err != nil { t.Fatal(err) } f.source.sys = u if kind != "none" && !(kind == "record" && os.Getenv("CARLOS_HOST_CLAIM") == "1") { if _, err := f.source.forHost(t.Context(), f.rt.Host); err != nil { t.Fatal("fixture must bind", err) } } incident := strings.HasPrefix(history, "incident") || history == "TestR4ForwardResetFailedError" if incident { if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil { t.Fatal(err) } if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil { t.Fatal(err) } if err := f.e.root.reg.SetVersions(f.rt.Host, "v2", "v1"); err != nil { t.Fatal(err) } target.exe = release.VersionPath(f.u.opt, "", "v2", f.rt.App) target.props.ActiveState, target.props.SubState, target.props.MainPID = "failed", "failed", 0 f.promote(t, "v1") } switch history { case "crash", "failed-then-reset-failed before rollback": target.crashNew = true case "auto-restart": target.props.ActiveState, target.props.SubState, target.props.MainPID = "activating", "auto-restart", 0 case "inactive": target.props.ActiveState, target.props.SubState, target.props.MainPID = "inactive", "dead", 0 case "incident reset before": if err := u.ResetFailed(f.rt.Unit); err != nil { t.Fatal(err) } case "TestR4ForwardResetFailedError": u.resetError = true case "TestR4ForwardTryRestartError": u.tryError = true case "interrupted repoint": // A root pass crashed after pointing v2 live and before restarting // v1's process; the next build, v3, then fails to come up. Main // rolls back to what the link names (v2), not to the running v1. account := release.RouteAccount(f.rt) if err := release.StageArtifact(f.u.opt, account, "v2", f.rt.App, []byte("new build")); err != nil { t.Fatal(err) } if err := release.PointLive(f.u.opt, f.rt.Host, account, "v2", f.rt.App); err != nil { t.Fatal(err) } m := release.Manifest{App: f.rt.App, Version: "v3", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} if err := release.Ship(t.Context(), f.e.store, "bab", m, map[string][]byte{release.AdoptPlatform(): []byte("third build")}); err != nil { t.Fatal(err) } f.promote(t, "v3") v3 := release.VersionPath(f.u.opt, account, "v3", f.rt.App) target.afterEffect = func() { if target.exe == v3 { target.props.ActiveState, target.props.SubState, target.props.MainPID = "failed", "failed", 0 } } } getenv := func(string) string { return "5s" } observer := f.e.root.obs observer.resolve, observer.sys = f.c.resolve, u observer.procExe = func(pid int) string { exe, _ := u.Exe(pid); return exe } pr := f.e.root pr.obs = observer restart := rootRestartRouteWith(f.e.root.reg, u, f.u.opt, getenv, observer.recordSettled(f.e.bake), f.c) rollback := rollbackUnitRestart(f.e.root.reg, u, f.u.opt, getenv, f.c) startChecks := f.c startChecks.legacyStart = u.Start start := rollbackUnitRestart(f.e.root.reg, u, f.u.opt, getenv, startChecks) if main { restart = makeRestartRoute(f.e.root.reg, func(host string) { _ = u.TryRestart(activator.UnitName(host)) }, fix5MainCheckedRestart(u, f.u.opt, getenv, nil, f.c), nil) rollback = fix5MainRollbackRestart(f.e.root.reg, u, f.u.opt, getenv, f.c) start = makeStartRoute(f.e.root.reg, u.Start) } else { // Binding may observe health and retain a running build, but cannot enter // the strict effect machinery or touch its durable restart journal. f.c.attempts = func() (rootstate.Store, error) { t.Fatal("no-census pass opened restart journal") return nil, os.ErrPermission } f.c.dispatch = func(context.Context, registry.Route) (bool, error) { t.Fatal("no-census pass dispatched binding") return false, os.ErrPermission } restart = rootRestartRouteWith(f.e.root.reg, u, f.u.opt, getenv, observer.recordSettled(f.e.bake), f.c) rollback = rollbackUnitRestart(f.e.root.reg, u, f.u.opt, getenv, f.c) startChecks = f.c startChecks.legacyStart = u.Start start = rollbackUnitRestart(f.e.root.reg, u, f.u.opt, getenv, startChecks) } originalRollback := rollback rollback = func(host string) error { if history == "failed-then-reset-failed before rollback" { if err := u.ResetFailed(f.rt.Unit); err != nil { return err } } return originalRollback(host) } snapshot := func() fix5Result { t.Helper() live, err := release.LiveVersion(f.u.opt, f.rt.Host) if err != nil { t.Fatal(err) } previous, err := filepath.EvalSymlinks(filepath.Join(f.u.opt, "previous", f.rt.Host)) if err != nil && !os.IsNotExist(err) { t.Fatal(err) } rt, err := f.e.root.reg.Get(f.rt.Host) if err != nil { t.Fatal(err) } rt.CreatedAt = 0 processes := map[string]string{} for name, unit := range u.units { processes[name] = strings.TrimPrefix(unit.exe, f.u.opt) + "|" + unit.props.ActiveState + "|" + unit.props.SubState + "|" + unit.props.InvocationID } return fix5Result{append([]string(nil), u.events...), live, strings.TrimPrefix(previous, f.u.opt), *rt, processes} } pass := func() { t.Helper() run := func() error { opts := []release.PassOption{release.WithStartUnit(start), release.WithRollbackRestart(rollback)} if !main { recovery := productionNamedUnitRecovery(f.e.store, f.pub, f.u.opt, hostwrite.Config{}) opts = append(opts, release.WithRootHistory(nil), release.WithRunningVersion(f.c.runningVersion), release.WithRestartAttemptSettled(f.c.finish), recovery.passOption(f.e.root.reg, f.c, getenv)) } return release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, restart, opts...) } var err error if main { err = run() } else { err = pr.around(t.Context(), run) } if err != nil { t.Fatal("pass status differs from main", err) } } pass() results := []fix5Result{snapshot()} if incident { u.resetError = false if history != "incident reset before" { if err := u.ResetFailed(f.rt.Unit); err != nil { t.Fatal(err) } } if err := u.Start(f.rt.Unit); err != nil { t.Fatal(err) } results = append(results, snapshot()) } // A second pass detects a synthetic refusal that would reject the fix. pass() return append(results, snapshot()) } func TestPlan14Fix5MainEquivalence(t *testing.T) { slots := make(chan struct{}, 4) if !plan14RootFixture(t) { return } for _, flag := range []string{"", "1"} { t.Run(map[string]string{"": "claims-off", "1": "claims-on"}[flag], func(t *testing.T) { t.Setenv("CARLOS_HOST_CLAIM", flag) for _, kind := range []string{"none", "record", "list", "guard", "different stamp"} { for _, history := range []string{"healthy", "crash", "failed-then-reset-failed before rollback", "auto-restart", "inactive", "incident reset before", "incident reset after", "TestR4ForwardResetFailedError", "TestR4ForwardTryRestartError", "interrupted repoint"} { t.Run(kind+"/"+history, func(t *testing.T) { t.Parallel() slots <- struct{}{} defer func() { <-slots }() want := fix5Run(t, kind, history, true) got := fix5Run(t, kind, history, false) if !reflect.DeepEqual(got, want) { t.Fatalf("no-census effects/pointers differ from main:\ngot %+v\nwant %+v", got, want) } }) } } }) } } func TestPlan14Fix5NoCensusSkipsJournalRecovery(t *testing.T) { if !plan14RootFixture(t) { return } for _, flag := range []string{"", "1"} { t.Setenv("CARLOS_HOST_CLAIM", flag) f := r2Fixture(t, "list") f.c.attempts = func() (rootstate.Store, error) { t.Fatal("read retained journal without a census") return nil, os.ErrPermission } if err := recoverRootRestarts(t.Context(), f.u, f.u.opt, nil, f.c, hostwrite.Config{}); err != nil { t.Fatal(err) } if len(f.u.effects) != 0 { t.Fatal(f.u.effects) } } }