package release import ( "encoding/json" "errors" "os" "path/filepath" "reflect" "testing" ) func rootRestartFixture(t *testing.T) (*RootRestarts, *rootRestartSyncStore, RestartBinding, RestartSnapshot) { t.Helper() s := &rootRestartSyncStore{healthyRetryStore: &healthyRetryStore{dir: t.TempDir(), fault: errors.New("injected directory sync failure")}} b := RestartBinding{Box: "box-a", Account: "bab", App: "console"} in := RestartSnapshot{Complete: true, AppID: "initial", AppBody: []byte(`{"schema":1}`), Scopes: ScopedRestartGeneration{Schema: 1}, Targets: []RestartTarget{ {Host: "one.example", Source: "record-one", Installation: "installation-one", Unit: "one.service", Environment: "prod"}, {Host: "two.example", Source: "record-two", Installation: "installation-two", Unit: "two.service", Environment: "test"}, }} return &RootRestarts{Store: s}, s, b, in } func nextRootRestart(in RestartSnapshot) RestartSnapshot { in.Scopes.Requests = []RestartRequest{{ID: "request-1", RequestedAt: "2026-10-13T12:00:00Z", Scope: RestartScope{Kind: RestartScopeEnvironment, Value: "prod"}}} return in } func TestPlan14RootRestartConsumption(t *testing.T) { for _, attack := range []string{"delete", "rename", "forge"} { t.Run(attack, func(t *testing.T) { h, _, b, in := rootRestartFixture(t) if p, err := h.Observe(b, in); err != nil || p != nil { t.Fatalf("baseline: %v %v", p, err) } in = nextRootRestart(in) p, err := h.Observe(b, in) if err != nil || p == nil || len(p.Targets) != 1 || p.Targets[0].Target.Host != "one.example" { t.Fatalf("exact scope: %+v %v", p, err) } legacy := filepath.Join(t.TempDir(), "restart-marks.json") if err := os.WriteFile(legacy, []byte(`{"schema":1,"apps":{},"scopes":{"bab|console":"request-1"}}`), 0600); err != nil { t.Fatal(err) } switch attack { case "delete": err = os.Remove(legacy) case "rename": err = os.Rename(filepath.Dir(legacy), filepath.Dir(legacy)+"-moved") t.Cleanup(func() { os.RemoveAll(filepath.Dir(legacy) + "-moved") }) case "forge": err = os.WriteFile(legacy, []byte(`{"schema":1,"apps":{},"scopes":{}}`), 0600) } if err != nil { t.Fatal(err) } target := p.Targets[0].Target if err := h.Begin(b, p.ID, target, "boot-a/invocation-a"); err != nil { t.Fatal(err) } if err := h.Complete(b, p.ID, target, "boot-a/invocation-b"); err != nil { t.Fatal(err) } if err := h.Finish(b, p.ID); err != nil { t.Fatal(err) } if p, err := h.Observe(b, in); err != nil || p != nil { t.Fatalf("replayed after %s: %+v %v", attack, p, err) } }) } } func TestPlan14RootRestartPendingSurvivesRollover(t *testing.T) { h, _, b, in := rootRestartFixture(t) if _, err := h.Observe(b, in); err != nil { t.Fatal(err) } in = nextRootRestart(in) p, err := h.Observe(b, in) if err != nil { t.Fatal(err) } // A held target is not begun. Neither a new request nor vanished routes may erase it. in.Targets = nil in.AppID = "later" in.Scopes.Requests = []RestartRequest{{ID: "rolled-over", Scope: RestartScope{Kind: RestartScopeHost, Value: "other.example"}}} again, err := h.Observe(b, in) if err != nil || !reflect.DeepEqual(p, again) { t.Fatalf("pending work changed: %+v %v", again, err) } if err := h.Complete(b, p.ID, p.Targets[0].Target, "unrelated"); err == nil { t.Fatal("unperformed request consumed") } } func TestPlan14RootRestartAmbiguousDoesNotReplay(t *testing.T) { h, _, b, in := rootRestartFixture(t) if _, err := h.Observe(b, in); err != nil { t.Fatal(err) } in = nextRootRestart(in) p, err := h.Observe(b, in) if err != nil { t.Fatal(err) } target := p.Targets[0].Target if err := h.Begin(b, p.ID, target, "boot-a/invocation-a"); err != nil { t.Fatal(err) } if err := h.Begin(b, p.ID, target, "boot-a/invocation-a"); err == nil { t.Fatal("ambiguous attempt granted a second dispatch") } for _, change := range []string{"same-invocation", "empty", "source", "installation", "unit", "request"} { t.Run(change, func(t *testing.T) { target := target id := p.ID after := "boot-a/invocation-b" switch change { case "same-invocation": after = "boot-a/invocation-a" case "empty": after = "" case "source": target.Source = "changed" case "installation": target.Installation = "changed" case "unit": target.Unit = "foreign.service" case "request": id = "other" } if err := h.Complete(b, id, target, after); err == nil { t.Fatal("ambiguous or foreign evidence consumed request") } }) } if err := h.Complete(b, p.ID, target, "boot-a/invocation-b"); err != nil { t.Fatal(err) } } func TestPlan14RootRestartVisibleSync(t *testing.T) { for _, boundary := range []string{"baseline", "pending", "begin", "complete"} { t.Run(boundary, func(t *testing.T) { h, s, b, in := rootRestartFixture(t) if boundary == "baseline" { s.failAt = s.writes + 1 if _, err := h.Observe(b, in); !errors.Is(err, s.fault) { t.Fatalf("baseline: %v", err) } } if _, err := h.Observe(b, in); err != nil { t.Fatal(err) } in = nextRootRestart(in) if boundary == "pending" { s.failAt = s.writes + 2 if _, err := h.Observe(b, in); !errors.Is(err, s.fault) { t.Fatalf("pending: %v", err) } } p, err := h.Observe(b, in) if err != nil { t.Fatal(err) } target := p.Targets[0].Target if boundary == "begin" { s.failAt = s.writes + 2 if err := h.Begin(b, p.ID, target, "boot-a/invocation-a"); !errors.Is(err, s.fault) { t.Fatalf("begin: %v", err) } if err := h.Begin(b, p.ID, target, "boot-a/invocation-a"); err == nil { t.Fatal("visible intent dispatched twice") } return } if err := h.Begin(b, p.ID, target, "boot-a/invocation-a"); err != nil { t.Fatal(err) } if boundary == "complete" { s.failAt = s.writes + 2 if err := h.Complete(b, p.ID, target, "boot-a/invocation-b"); !errors.Is(err, s.fault) { t.Fatalf("complete: %v", err) } } if err := h.Complete(b, p.ID, target, "boot-a/invocation-b"); err != nil { t.Fatal(err) } if err := h.Finish(b, p.ID); err != nil { t.Fatal(err) } if p, err := h.Observe(b, in); err != nil || p != nil { t.Fatalf("completion not durable: %+v %v", p, err) } }) } } func TestPlan14RootRestartStrictState(t *testing.T) { for _, change := range []string{"schema", "box", "complete", "revision", "extra", "duplicate", "trailing", "null"} { t.Run(change, func(t *testing.T) { h, s, b, in := rootRestartFixture(t) h.Observe(b, in) raw, err := s.Read(rootRestartKey(b)) if err != nil { t.Fatal(err) } var r rootRestartRecord if err := json.Unmarshal(raw, &r); err != nil { t.Fatal(err) } switch change { case "schema": r.Schema++ case "box": r.Binding.Box = "other" case "complete": r.Complete = 0 case "revision": r.Revision = 0 } raw, _ = json.Marshal(r) switch change { case "extra": raw = append(raw[:len(raw)-1], []byte(`,"extra":true}`)...) case "duplicate": raw = append(raw[:len(raw)-1], []byte(`,"schema":1}`)...) case "trailing": raw = append(raw, []byte(` {}`)...) case "null": raw = []byte(`null`) } if err := s.Write(rootRestartKey(b), raw); err != nil { t.Fatal(err) } if _, err := h.Observe(b, nextRootRestart(in)); err == nil { t.Fatal("invalid state granted consumption") } }) } } type rootRestartSyncStore struct { *healthyRetryStore writes int failAt int } func (s *rootRestartSyncStore) Write(name string, raw []byte) error { s.writes++ if s.writes == s.failAt { s.fail = 1 } return s.healthyRetryStore.Write(name, raw) } func TestPlan14RootRestartSelection(t *testing.T) { for _, kind := range []string{"app", "host", "environment", "unknown", "retry", "hibernate", "rollover", "incomplete", "absent", "duplicate-target", "duplicate-request"} { t.Run(kind, func(t *testing.T) { h, _, b, in := rootRestartFixture(t) if kind == "absent" { in.AppID = RootRestartAbsent in.AppBody = nil } if _, err := h.Observe(b, in); err != nil { t.Fatal(err) } in = nextRootRestart(in) want := 1 switch kind { case "app", "absent": in.AppID = "new-app-request" in.AppBody = []byte(`{"requested_at":"2026-10-13T12:00:00Z"}`) want = 2 case "host": in.Scopes.Requests[0].Scope = RestartScope{Kind: RestartScopeHost, Value: "two.example"} case "unknown": in.Scopes.Requests[0].Scope = RestartScope{Kind: "future-selector", Value: "prod"} want = 0 case "retry": in.Scopes.Requests[0].Scope = RestartScope{Kind: RestartScopeRetry, Value: "one.example"} want = 0 case "hibernate": in.Targets[0].Hibernate = true want = 0 case "rollover": if p, err := h.Observe(b, RestartSnapshot{Complete: true, AppID: in.AppID, AppBody: in.AppBody, Scopes: in.Scopes}); err != nil || p != nil { t.Fatalf("first empty target request: %+v %v", p, err) } in.Scopes.Requests[0].ID = "new-position" want = 0 case "incomplete": in.Complete = false case "duplicate-target": in.Targets = append(in.Targets, in.Targets[0]) case "duplicate-request": in.Scopes.Requests = append(in.Scopes.Requests, in.Scopes.Requests[0]) } p, err := h.Observe(b, in) if kind == "incomplete" || kind == "duplicate-target" || kind == "duplicate-request" { if err == nil { t.Fatal("invalid census/request accepted") } return } if err != nil { t.Fatal(err) } if want == 0 { if p != nil { t.Fatalf("unexpected restart: %+v", p) } return } if p == nil || len(p.Targets) != want { t.Fatalf("targets: %+v", p) } if kind == "host" && p.Targets[0].Target.Host != "two.example" { t.Fatal("host selection widened") } if kind == "app" && string(p.AppBody) != string(in.AppBody) { t.Fatal("lost exact app request") } if err := h.Finish(b, p.ID); err == nil { t.Fatal("unperformed targets consumed") } if err := h.Begin(b, p.ID, p.Targets[0].Target, "old"); err != nil { t.Fatal(err) } if err := h.Complete(b, p.ID, p.Targets[0].Target, "new"); err != nil { t.Fatal(err) } if want == 2 { if err := h.Finish(b, p.ID); err == nil { t.Fatal("partial target set consumed") } } }) } } func TestPlan14RootRestartFailureIsTerminal(t *testing.T) { h, _, b, in := rootRestartFixture(t) if _, err := h.Observe(b, in); err != nil { t.Fatal(err) } in = nextRootRestart(in) p, err := h.Observe(b, in) if err != nil { t.Fatal(err) } target := p.Targets[0].Target if err := h.Fail(b, p.ID, target); err == nil { t.Fatal("undispatched request consumed as failure") } if err := h.Begin(b, p.ID, target, "before"); err != nil { t.Fatal(err) } if err := h.Fail(b, p.ID, target); err != nil { t.Fatal(err) } if err := h.Begin(b, p.ID, target, "before"); err == nil { t.Fatal("terminal failure retried") } if err := h.Complete(b, p.ID, target, "after"); err == nil { t.Fatal("terminal failure changed to success") } if err := h.Finish(b, p.ID); err != nil { t.Fatal(err) } if p, err := h.Observe(b, in); err != nil || p != nil { t.Fatalf("failure replayed: %+v %v", p, err) } in.Scopes.Requests = append(in.Scopes.Requests, RestartRequest{ID: "retry-as-new-request", Scope: RestartScope{Kind: RestartScopeHost, Value: target.Host}}) if p, err := h.Observe(b, in); err != nil || p == nil { t.Fatalf("fresh explicit request refused: %+v %v", p, err) } } func TestPlan14RootRestartReadMustSync(t *testing.T) { for _, step := range []string{"observe", "begin", "complete", "finish"} { t.Run(step, func(t *testing.T) { h, s, b, in := rootRestartFixture(t) if _, err := h.Observe(b, in); err != nil { t.Fatal(err) } in = nextRootRestart(in) p, err := h.Observe(b, in) if err != nil { t.Fatal(err) } target := p.Targets[0].Target if step == "complete" || step == "finish" { if err := h.Begin(b, p.ID, target, "before"); err != nil { t.Fatal(err) } } if step == "finish" { if err := h.Complete(b, p.ID, target, "after"); err != nil { t.Fatal(err) } } s.failAt = s.writes + 1 switch step { case "observe": _, err = h.Observe(b, in) case "begin": err = h.Begin(b, p.ID, target, "before") case "complete": err = h.Complete(b, p.ID, target, "after") case "finish": err = h.Finish(b, p.ID) } if !errors.Is(err, s.fault) { t.Fatalf("%s granted without durable read: %v", step, err) } }) } } func TestPlan14RootRestartPendingStateTampering(t *testing.T) { for _, change := range []string{"remove-target", "append-target", "source", "selector", "body", "id", "completed-without-begin", "failed-without-begin"} { t.Run(change, func(t *testing.T) { h, s, b, in := rootRestartFixture(t) if _, err := h.Observe(b, in); err != nil { t.Fatal(err) } if _, err := h.Observe(b, nextRootRestart(in)); err != nil { t.Fatal(err) } raw, err := s.Read(rootRestartKey(b)) if err != nil { t.Fatal(err) } var r rootRestartRecord if err := json.Unmarshal(raw, &r); err != nil { t.Fatal(err) } switch change { case "remove-target": r.Pending.Targets = nil case "append-target": r.Pending.Targets = append(r.Pending.Targets, RestartProgress{Target: in.Targets[1]}) case "source": r.Pending.Targets[0].Target.Source = "other" case "selector": r.Pending.Requests[0].Scope.Value = "test" case "body": r.Pending.AppBody = []byte("other") case "id": r.Pending.ID = "other" case "completed-without-begin": r.Pending.Targets[0].After = "after" case "failed-without-begin": r.Pending.Targets[0].Failed = true } raw, _ = json.Marshal(r) if err := s.Write(rootRestartKey(b), raw); err != nil { t.Fatal(err) } if _, err := h.Observe(b, nextRootRestart(in)); err == nil { t.Fatal("changed pending authority accepted") } }) } }