package host import ( "encoding/json" "errors" "os" "os/exec" "path/filepath" "sync" "testing" "time" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootstate" ) func TestPlan14HistoryRecoveryProcessCrash(t *testing.T) { t.Parallel() if boundary := os.Getenv("PLAN14_HISTORY_CRASH"); boundary != "" { dir := os.Getenv("PLAN14_HISTORY_DIR") e, rt, u, c, pub := plan14AdoptionFixture(t) storeDir := filepath.Join(dir, "deployment") rec, _, err := provision.FindRecordByHost(t.Context(), e.store, rt.Host) if err != nil { t.Fatal(err) } plan14PutRecord(t, release.NewDirStore(storeDir), *rec) h := seedHostRootHistory(t, rt) disk := &rollbackDisk{dir: filepath.Join(dir, "attempts")} historyDisk := &rollbackDisk{dir: filepath.Join(dir, "history")} for _, d := range []*rollbackDisk{disk, historyDisk} { if err := os.Mkdir(d.dir, 0700); err != nil { t.Fatal(err) } } for k, raw := range h.Store.(*rollbackMemory).files { if err := historyDisk.Write(k, raw); err != nil { t.Fatal(err) } } h.Store = historyDisk deps := boundRootHistory(h, c.resolve, e.root.obs) c.attempts = func() (rootstate.Store, error) { return disk, nil } c.history = func() (*release.RootHistoryPass, error) { return deps, nil } linkRootHistory(deps, c, u.opt, func(string) string { return "5s" }) crash := func() { raw, err := json.Marshal(rollbackProcessFixture{Props: u.props, Exe: u.exe, Opt: u.opt, Bake: e.bake, Store: storeDir, Effects: u.effects}) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "fixture.json"), raw, 0600); err != nil { t.Fatal(err) } os.Exit(74) } disk.afterWrite = func(k string, raw []byte) { if k != rollbackJournalKey(rt.Host) { return } var a rollbackAttempt if json.Unmarshal(raw, &a) != nil || a.History == nil { return } if boundary == "linked" && !a.Failed && !a.Recovering && !a.Settled { crash() } if boundary == "forward settled" && a.Settled { crash() } if boundary == "rollback pending" && a.Recovering && !a.Settled { crash() } } historyDisk.afterWrite = func(k string, raw []byte) { if boundary != "history settled" { return } var r struct { Binding release.HistoryBinding `json:"binding"` Attempt *release.HistoryAttempt `json:"attempt"` } if json.Unmarshal(raw, &r) == nil && r.Binding.Host == rt.Host && r.Attempt == nil && len(u.effects) > 0 { crash() } } u.crashNew = boundary != "forward running" && boundary != "forward unhealthy" && boundary != "forward settled" && boundary != "history settled" u.afterEffect = func() { if (boundary == "forward failed" || boundary == "forward running" || boundary == "forward unhealthy" || boundary == "rollback fails") && len(u.effects) == 1 { crash() } if boundary == "rollback running" && len(u.effects) == 3 { crash() } } forward := rootRestartRouteWith(e.root.reg, u, u.opt, func(string) string { return "5s" }, nil, c) _ = release.Pass(t.Context(), e.root.reg, e.store, pub, u.opt, forward, release.WithRootHistory(deps), release.WithRestartFailurePropagation(), release.WithRollbackRestart(rollbackUnitRestart(e.root.reg, u, u.opt, func(string) string { return "5s" }, c))) t.Fatal("crash boundary not reached") } for _, boundary := range []string{"linked", "forward failed", "forward running", "forward unhealthy", "forward settled", "rollback pending", "rollback running", "history settled", "rollback fails"} { t.Run(boundary, func(t *testing.T) { dir := t.TempDir() cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestPlan14HistoryRecoveryProcessCrash$", "-test.count=1") cmd.Dir = dir cmd.Env = append(os.Environ(), "TMPDIR="+dir, "PLAN14_HISTORY_CRASH="+boundary, "PLAN14_HISTORY_DIR="+dir, "CARLOS_HOST_CLAIM=") out, err := cmd.CombinedOutput() var exit *exec.ExitError if !errors.As(err, &exit) || exit.ExitCode() != 74 { t.Fatalf("child: %v\n%s", err, out) } raw, err := os.ReadFile(filepath.Join(dir, "fixture.json")) if err != nil { t.Fatal(err) } var f rollbackProcessFixture if err := json.Unmarshal(raw, &f); err != nil { t.Fatal(err) } u := &plan14RestartUnits{props: f.Props, exe: f.Exe, opt: f.Opt, effects: f.Effects, badRollback: boundary == "rollback fails"} disk := &rollbackDisk{dir: filepath.Join(dir, "attempts")} source := &proofBindingSource{store: release.NewDirStore(f.Store), sys: u, optDir: u.opt, openFile: os.Open, unitDirs: []string{filepath.Dir(u.props.FragmentPath)}, healthy: func(acct, app, ver, digest string) *release.HealthyBuild { for _, b := range release.ReadHealthy(f.Bake, acct, app) { if b.Version == ver && b.SHA256 == digest { return &b } } return nil }} a, err := readRollbackAttempt(disk, observedHost) if err != nil || a.History == nil { t.Fatalf("missing linked intent: %+v %v", a, err) } now := a.Created.Add(time.Second) ready := func(registry.Route) func() error { return func() error { if boundary == "forward unhealthy" && u.exe == release.VersionPath(u.opt, "bab", "v2", "console") { return os.ErrPermission } return nil } } h := &release.RootHistory{Store: &rollbackDisk{dir: filepath.Join(dir, "history")}} obs := proofObserver{resolve: source.resolve, sys: u, optDir: u.opt, procExe: func(int) string { return u.exe }, now: time.Now, ready: ready} deps := boundRootHistory(h, source.resolve, obs) c := rootRestartChecks{history: func() (*release.RootHistoryPass, error) { return deps, nil }, started: &sync.Map{}, attempts: func() (rootstate.Store, error) { return disk, nil }, prepareRollback: source.prepareRollback, rollback: source.rollback, host: source.forHost, ready: ready, now: func() time.Time { return now }, sleep: func(d time.Duration) { now = now.Add(d) }} before := len(u.effects) err = recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}) refs, refsErr := h.Refusals(a.HistoryBinding) if boundary == "rollback fails" { if err == nil || refsErr == nil { t.Fatalf("failed rollback settled: %v %v", err, refsErr) } before := len(u.effects) if err := recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}); err != nil || len(u.effects) != before { t.Fatalf("terminal rollback repeated: %v %v", err, u.effects) } if _, err := h.Refusals(a.HistoryBinding); err == nil { t.Fatal("terminal failure erased history intent") } return } if err != nil || refsErr != nil { t.Fatalf("resume: %v %v", err, refsErr) } forward := boundary == "forward running" || boundary == "forward settled" || boundary == "history settled" if forward && (len(refs) != 0 || u.exe != release.VersionPath(u.opt, "bab", "v2", "console")) { t.Fatalf("forward not recovered: %+v %s", refs, u.exe) } if !forward && (len(refs) != 1 || !refs[0].Recovered || refs[0].Version != "v2" || refs[0].RestoredVersion != "v1") { t.Fatalf("rollback deny missing: %+v", refs) } if boundary != "forward failed" && boundary != "forward unhealthy" && boundary != "rollback pending" && len(u.effects) != before { t.Fatal("recovery repeated an observed effect") } before = len(u.effects) if err := recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}); err != nil || len(u.effects) != before { t.Fatalf("replay: %v", err) } }) } }