package host import ( "encoding/json" "errors" "os" "os/exec" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootstate" ) func TestPlan14ActiveRecoveryProcessCrash(t *testing.T) { if boundary := os.Getenv("PLAN14_ACTIVE_CRASH"); boundary != "" { dir := os.Getenv("PLAN14_ACTIVE_DIR") e, u, c, source, pub := signedRecoveryFixture(t) storeDir := filepath.Join(dir, "deployment") store := release.NewDirStore(storeDir) keys, err := e.store.List(t.Context(), "") if err != nil { t.Fatal(err) } for _, key := range keys { raw, _, err := e.store.Get(t.Context(), key) if err != nil { t.Fatal(err) } if _, err := store.Put(t.Context(), key, raw, ""); err != nil { t.Fatal(err) } } source.store = store disk := &rollbackDisk{dir: filepath.Join(dir, "attempts")} if err := os.Mkdir(disk.dir, 0700); err != nil { t.Fatal(err) } c.attempts = func() (rootstate.Store, error) { return disk, nil } crash := func() { raw, err := json.Marshal(rollbackProcessFixture{Props: u.props, Exe: u.exe, Opt: u.opt, Store: storeDir, Effects: u.effects}) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "fixture.json"), raw, 0600); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "public-key"), []byte(os.Getenv("CARLOS_RELEASE_PUBKEY")), 0600); err != nil { t.Fatal(err) } os.Exit(75) } disk.afterWrite = func(k string, raw []byte) { if k != rollbackJournalKey(observedHost) { return } var a rollbackAttempt if json.Unmarshal(raw, &a) != nil { t.Fatal("bad attempt") } if boundary == "intent" && !a.Failed && !a.Recovering && !a.Settled { crash() } if boundary == "rollback pending" && a.Recovering && !a.Settled { crash() } if boundary == "verified" && a.Verified && a.Settled { crash() } } u.crashNew = boundary == "forward failed" || boundary == "rollback pending" || boundary == "rollback running" u.afterEffect = func() { if len(u.effects) == 1 && (boundary == "forward running" || boundary == "forward failed") { crash() } if len(u.effects) == 3 && boundary == "rollback running" { crash() } } n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink} _, err = n.recover(t.Context(), observedHost, c, func(string) string { return "5s" }, e.root.reg) t.Fatalf("crash boundary not reached: %v", err) } for _, boundary := range []string{"intent", "forward running", "forward failed", "rollback pending", "rollback running", "verified"} { t.Run(boundary, func(t *testing.T) { dir := t.TempDir() cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestPlan14ActiveRecoveryProcessCrash$", "-test.count=1") cmd.Dir = dir cmd.Env = append(os.Environ(), "TMPDIR="+dir, "PLAN14_ACTIVE_CRASH="+boundary, "PLAN14_ACTIVE_DIR="+dir, "CARLOS_HOST_CLAIM=") out, err := cmd.CombinedOutput() var exit *exec.ExitError if !errors.As(err, &exit) || exit.ExitCode() != 75 { t.Fatalf("child: %v\n%s", err, out) } raw, err := os.ReadFile(filepath.Join(dir, "fixture.json")) if err != nil { t.Fatal(err) } var f rollbackProcessFixture if err := json.Unmarshal(raw, &f); err != nil { t.Fatal(err) } key, err := os.ReadFile(filepath.Join(dir, "public-key")) if err != nil { t.Fatal(err) } t.Setenv("CARLOS_RELEASE_PUBKEY", string(key)) u := &plan14RestartUnits{props: f.Props, exe: f.Exe, opt: f.Opt, effects: f.Effects} source := &proofBindingSource{store: release.NewDirStore(f.Store), sys: u, optDir: u.opt, openFile: os.Open, unitDirs: []string{filepath.Dir(u.props.FragmentPath)}} disk := &rollbackDisk{dir: filepath.Join(dir, "attempts")} a, err := readRollbackAttempt(disk, observedHost) if err != nil { t.Fatal(err) } now := a.Created.Add(time.Second) c := rootRestartChecks{attempts: func() (rootstate.Store, error) { return disk, nil }, host: source.forHost, rollback: source.rollback, prepareRollback: source.prepareRollback, recovered: source.recovered, ready: func(registry.Route) func() error { return func() error { return nil } }, now: func() time.Time { return now }, sleep: func(d time.Duration) { now = now.Add(d) }, history: func() (*release.RootHistoryPass, error) { return nil, os.ErrPermission }} before := len(u.effects) if err := recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}); err != nil { t.Fatal(err) } want := "v1" if boundary == "forward running" || boundary == "verified" { want = "v2" } if u.exe != release.VersionPath(u.opt, "bab", want, "console") || u.props.MainPID <= 0 { t.Fatalf("wrong recovered process: %+v", u) } if boundary != "forward failed" && boundary != "rollback pending" && len(u.effects) != before { t.Fatalf("effect repeated: %v", u.effects) } before = len(u.effects) if err := recoverRootRestarts(t.Context(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}); err != nil || len(u.effects) != before { t.Fatalf("replay: %v %v", err, u.effects) } }) } }