package release import ( "encoding/json" "errors" "os" "os/exec" "path/filepath" "testing" ) type restartCrashStore struct { *healthyRetryStore boundary string } func (s *restartCrashStore) Write(name string, raw []byte) error { var r rootRestartRecord if err := json.Unmarshal(raw, &r); err != nil { return err } stage := "baseline" if r.Pending != nil { stage = "pending" if r.Pending.Targets[0].Before != "" { stage = "intent" } if r.Pending.Targets[0].After != "" { stage = "completed" } } else if r.ScopeID == "request-1" { stage = "consumed" } crash := s.boundary == stage || (s.boundary == "intent-visible" && stage == "intent") if crash && s.boundary == "intent-visible" { s.fail = 1 } err := s.healthyRetryStore.Write(name, raw) if crash { if err != nil && !errors.Is(err, s.fault) { return err } os.Exit(74) } return err } func TestPlan14RootRestartProcessCrash(t *testing.T) { if boundary := os.Getenv("PLAN14_RESTART_CRASH"); boundary != "" { _, _, b, in := rootRestartFixture(t) dir := os.Getenv("PLAN14_RESTART_DIR") disk := &healthyRetryStore{dir: dir, fault: errors.New("injected directory sync failure")} h := &RootRestarts{Store: &restartCrashStore{healthyRetryStore: disk, boundary: boundary}} if boundary == "baseline" { h.Observe(b, in) t.Fatal("baseline crash not reached") } in = nextRootRestart(in) p, err := h.Observe(b, in) if err != nil { t.Fatal(err) } target := p.Targets[0].Target if err := h.Begin(b, p.ID, target, "boot-a/old"); err != nil { t.Fatal(err) } // This is the fake unit's effect. The process exits without running defers. if err := os.WriteFile(filepath.Join(dir, "invocation"), []byte("boot-a/new"), 0600); err != nil { t.Fatal(err) } if boundary == "effect" { os.Exit(74) } if err := h.Complete(b, p.ID, target, "boot-a/new"); err != nil { t.Fatal(err) } if err := h.Finish(b, p.ID); err != nil { t.Fatal(err) } t.Fatal("crash boundary not reached") } for _, boundary := range []string{"baseline", "pending", "intent", "intent-visible", "effect", "completed", "consumed"} { t.Run(boundary, func(t *testing.T) { h, s, b, in := rootRestartFixture(t) if boundary != "baseline" { if _, err := h.Observe(b, in); err != nil { t.Fatal(err) } } cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestPlan14RootRestartProcessCrash$", "-test.count=1") cmd.Dir = s.dir cmd.Env = append(os.Environ(), "TMPDIR="+s.dir, "PLAN14_RESTART_CRASH="+boundary, "PLAN14_RESTART_DIR="+s.dir) out, err := cmd.CombinedOutput() var exit *exec.ExitError if !errors.As(err, &exit) || exit.ExitCode() != 74 { t.Fatalf("child: %v\n%s", err, out) } h = &RootRestarts{Store: &healthyRetryStore{dir: s.dir}} if boundary == "baseline" { if p, err := h.Observe(b, in); err != nil || p != nil { t.Fatalf("baseline replay: %+v %v", p, err) } return } in = nextRootRestart(in) p, err := h.Observe(b, in) if err != nil { t.Fatal(err) } if boundary == "consumed" { if p != nil { t.Fatal("consumed request replayed") } return } if p == nil { t.Fatal("lost pending request") } target := p.Targets[0].Target if boundary == "pending" { if err := h.Begin(b, p.ID, target, "boot-a/old"); err != nil { t.Fatal(err) } return } if err := h.Begin(b, p.ID, target, "boot-a/old"); err == nil { t.Fatal("crashed attempt dispatched twice") } invocation, err := os.ReadFile(filepath.Join(s.dir, "invocation")) if boundary == "intent" || boundary == "intent-visible" { if !errors.Is(err, os.ErrNotExist) { t.Fatalf("pre-effect crash: %v", err) } if err := h.Complete(b, p.ID, target, "boot-a/old"); err == nil { t.Fatal("ambiguous attempt consumed") } if err := h.Finish(b, p.ID); err == nil { t.Fatal("unfinished attempt consumed") } return } if err != nil { t.Fatal(err) } if err := h.Complete(b, p.ID, target, string(invocation)); err != nil { t.Fatal(err) } if err := h.Finish(b, p.ID); err != nil { t.Fatal(err) } if p, err := h.Observe(b, in); err != nil || p != nil { t.Fatalf("recovered request replayed: %+v %v", p, err) } }) } }