package host import ( "context" "encoding/json" "errors" "os" "os/exec" "path/filepath" "sync" "testing" "time" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootstate" ) type rollbackDisk struct { rootstate.Store dir string afterWrite func(string, []byte) beforeWrite func(string, []byte) } func (s *rollbackDisk) Read(k string) ([]byte, error) { return os.ReadFile(filepath.Join(s.dir, k)) } func (s *rollbackDisk) Write(k string, b []byte) error { if s.beforeWrite != nil { s.beforeWrite(k, b) } f, err := os.CreateTemp(s.dir, "pending-") if err != nil { return err } n := f.Name() defer os.Remove(n) if _, err = f.Write(b); err == nil { err = f.Sync() } err = errors.Join(err, f.Close()) if err != nil { return err } if err = os.Rename(n, filepath.Join(s.dir, k)); err != nil { return err } d, err := os.Open(s.dir) if err != nil { return err } err = errors.Join(d.Sync(), d.Close()) if err != nil { return err } if s.afterWrite != nil { s.afterWrite(k, b) } return nil } func (s *rollbackDisk) Remove(k string) error { err := os.Remove(filepath.Join(s.dir, k)) if errors.Is(err, os.ErrNotExist) { return nil } return err } func (s *rollbackDisk) Names() ([]string, error) { es, err := os.ReadDir(s.dir) var out []string for _, e := range es { out = append(out, e.Name()) } return out, err } func (s *rollbackDisk) Lock(context.Context, string) (func(), error) { return func() {}, nil } func (s *rollbackDisk) Close() error { return nil } type rollbackProcessFixture struct { Props unitProps Exe, Opt, Bake, Store string Effects []string } func TestPlan14RollbackProcessCrash(t *testing.T) { t.Parallel() if boundary := os.Getenv("PLAN14_CRASH_BOUNDARY"); boundary != "" { dir := os.Getenv("PLAN14_CRASH_DIR") e, rt, u, c, pub := plan14AdoptionFixture(t) storeDir := filepath.Join(dir, "deployment") store := release.NewDirStore(storeDir) rec, _, err := provision.FindRecordByHost(t.Context(), e.store, rt.Host) if err != nil { t.Fatal(err) } plan14PutRecord(t, store, *rec) disk := &rollbackDisk{dir: filepath.Join(dir, "attempts")} if err := os.Mkdir(disk.dir, 0700); err != nil { t.Fatal(err) } c.attempts = func() (rootstate.Store, error) { return disk, nil } crash := func() { b, err := json.Marshal(rollbackProcessFixture{Props: u.props, Exe: u.exe, Opt: u.opt, Bake: e.bake, Store: storeDir, Effects: u.effects}) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "fixture.json"), b, 0600); err != nil { t.Fatal(err) } os.Exit(73) } disk.afterWrite = func(k string, b []byte) { if k != rollbackJournalKey(rt.Host) { return } var a rollbackAttempt if json.Unmarshal(b, &a) != nil { return } if boundary == "captured" && !a.Failed && !a.Recovering && !a.Settled { crash() } if boundary == "failed journal" && a.Failed && !a.Recovering && !a.Settled { crash() } if boundary == "rollback pending" && a.Recovering && !a.Settled { crash() } if boundary == "settled" && a.Settled { crash() } } u.crashNew = boundary != "forward running" u.afterEffect = func() { if (boundary == "forward restart" || boundary == "forward running") && len(u.effects) == 1 { crash() } if boundary == "rollback running" && len(u.effects) == 3 { crash() } } forward := rootRestartRouteWith(e.root.reg, u, u.opt, func(string) string { return "5s" }, nil, c) _ = release.Pass(t.Context(), e.root.reg, e.store, pub, u.opt, forward, release.WithRestartFailurePropagation(), release.WithRollbackRestart(rollbackUnitRestart(e.root.reg, u, u.opt, func(string) string { return "5s" }, c))) t.Fatal("crash boundary not reached") } for _, boundary := range []string{"captured", "forward running", "forward restart", "failed journal", "rollback pending", "rollback running", "settled"} { t.Run(boundary, func(t *testing.T) { dir := t.TempDir() ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestPlan14RollbackProcessCrash$", "-test.count=1") cmd.Dir = dir cmd.Env = append(os.Environ(), "TMPDIR="+dir, "PLAN14_CRASH_BOUNDARY="+boundary, "PLAN14_CRASH_DIR="+dir, "CARLOS_HOST_CLAIM=") out, err := cmd.CombinedOutput() var exit *exec.ExitError if !errors.As(err, &exit) || exit.ExitCode() != 73 { t.Fatalf("subprocess did not crash at %s: %v\n%s", boundary, err, out) } body, err := os.ReadFile(filepath.Join(dir, "fixture.json")) if err != nil { t.Fatal(err) } var f rollbackProcessFixture if err := json.Unmarshal(body, &f); err != nil { t.Fatal(err) } u := &plan14RestartUnits{props: f.Props, exe: f.Exe, opt: f.Opt, effects: f.Effects} disk := &rollbackDisk{dir: filepath.Join(dir, "attempts")} source := &proofBindingSource{store: release.NewDirStore(f.Store), sys: u, optDir: u.opt, openFile: os.Open, unitDirs: []string{filepath.Dir(u.props.FragmentPath)}, healthy: func(acct, app, ver, digest string) *release.HealthyBuild { for _, b := range release.ReadHealthy(f.Bake, acct, app) { if b.Version == ver && b.SHA256 == digest { return &b } } return nil }} a, err := readRollbackAttempt(disk, observedHost) if err != nil { t.Fatal(err) } now := a.Created.Add(time.Second) c := rootRestartChecks{targetFile: os.Open, started: &sync.Map{}, attempts: func() (rootstate.Store, error) { return disk, nil }, prepareRollback: source.prepareRollback, rollback: source.rollback, host: source.forHost, ready: func(registry.Route) func() error { return func() error { return nil } }, now: func() time.Time { return now }, sleep: func(d time.Duration) { now = now.Add(d) }} before := len(u.effects) err = recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}) if boundary == "captured" { if err == nil || len(u.effects) != before { t.Fatalf("unstarted update restarted: %v %v", err, u.effects) } } else { if err != nil { t.Fatal(err) } want := "v1" if boundary == "forward running" { want = "v2" } if u.exe != release.VersionPath(u.opt, "bab", want, "console") || u.props.MainPID <= 0 { t.Fatalf("no recovered process: %+v", u) } if (boundary == "forward running" || boundary == "rollback running" || boundary == "settled") && len(u.effects) != before { t.Fatalf("restarted twice: %v", u.effects) } } a, err = readRollbackAttempt(disk, observedHost) if err != nil || !a.Settled { t.Fatalf("unsettled recovery: %+v %v", a, err) } if _, err := disk.Read(rollbackCapabilityKey(a.Host, a.ID)); !errors.Is(err, os.ErrNotExist) { t.Fatal("capability survived recovery") } before = len(u.effects) if err := recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}); err != nil || len(u.effects) != before { t.Fatalf("settled replay: %v %v", err, u.effects) } }) } }