package release import ( "bytes" "context" "crypto/ed25519" "crypto/rand" "encoding/json" "errors" "os" "os/exec" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/registry" ) type reconstructionCrashStore struct { *healthyRetryStore stage string fail bool } func TestPlan14RootHistoryReconstructionDeliveryPass(t *testing.T) { for _, attack := range []string{"valid", "missing", "forged", "source", "revision"} { t.Run(attack, func(t *testing.T) { h, _, b := rootHistoryFixture(t) r, err := h.read(b) if err != nil { t.Fatal(err) } r.Refusals.Complete = false if err := h.write(r); err != nil { t.Fatal(err) } ctx := t.Context() st := NewDirStore(t.TempDir()) priv, pub := testKeys(t) reg := testRegistry(t) addRoute(t, reg, b.Host, b.App, "canary/x", false) shipRelease(t, st, b.Account, b.App, "new", adoptPlatform, []byte("new")) if err := Promote(ctx, st, priv, b.Account, b.App, "canary/x", "new"); err != nil { t.Fatal(err) } status, err := h.ReconstructionStatus() if err != nil { t.Fatal(err) } now := time.Now().UTC() g := HistoryReconstruction{Schema: 1, Kind: "carlos-release-history", Identity: status.Identity, Revision: status.Revision, Operation: "delivered", IssuedAt: now, Account: b.Account, App: b.App, Hosts: []HistoryReconstructionHost{{Host: b.Host, Source: "record", Revision: 1}}} if attack == "source" { g.Hosts[0].Source = "forged" } if attack == "revision" { g.Hosts[0].Revision++ } signer := RootSigner(priv) if attack == "forged" { _, other, _ := ed25519.GenerateKey(rand.Reader) signer = RootSigner(other) } if attack != "missing" { if err := PublishHistoryReconstruction(ctx, st, signer, g); err != nil { t.Fatal(err) } } deps := &RootHistoryPass{History: h, Resolve: func(context.Context, registry.Route) (HistoryBinding, string, error) { return b, "record", nil }, Installed: func(context.Context, registry.Route, string) (string, error) { return "", nil }} deps.Reconstruct = func(ctx context.Context, b HistoryBinding) error { return h.ReceiveReconstruction(ctx, st, pub, b, now, func(HistoryBinding) (string, error) { return "record", nil }) } effects := 0 opt := t.TempDir() err = Pass(ctx, reg, st, pub, opt, func(string) error { effects++; return nil }, WithRootHistory(deps)) if attack == "valid" { if err != nil || effects != 1 { t.Fatalf("delivered reconstruction: effects=%d err=%v", effects, err) } if live, _ := LiveVersion(opt, b.Host); live != "new" { t.Fatalf("live=%q", live) } if err := deps.Reconstruct(ctx, b); err != nil { t.Fatalf("consumed delivery replay: %v", err) } } else if err == nil || effects != 0 { t.Fatalf("%s: effects=%d err=%v", attack, effects, err) } }) } } func (s *reconstructionCrashStore) Write(name string, raw []byte) error { old, _ := s.Read(name) if err := s.healthyRetryStore.Write(name, raw); err != nil { return err } if bytes.Equal(old, raw) { return nil } stage := name if name == reconstructionKey { var state reconstructionState if err := json.Unmarshal(raw, &state); err != nil { return err } if state.Pending != nil { stage = "intent" } else if state.Last != "" { stage = "complete" } } if s.stage == stage { if s.fail { return s.fault } os.Exit(83) } return nil } func reconstructionCrashFixture(t *testing.T) (*RootHistory, *healthyRetryStore, HistoryBinding, ed25519.PublicKey, []byte, time.Time) { t.Helper() h, s, b := rootHistoryFixture(t) status, err := h.ReconstructionStatus() if err != nil { t.Fatal(err) } pub, priv, _ := ed25519.GenerateKey(rand.Reader) now := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) g := HistoryReconstruction{Schema: 1, Kind: "carlos-release-history", Identity: status.Identity, Revision: status.Revision, Operation: "crash", IssuedAt: now, Account: b.Account, App: b.App, Hosts: []HistoryReconstructionHost{{Host: b.Host, Source: "canonical", Revision: 1}}, Holds: map[string]HistoryReconstructionHold{"stable": {Window: time.Hour, SetAt: now}}} wire, err := SignRootEnvelope(RootSigner(priv), g) if err != nil { t.Fatal(err) } return h, s, b, pub, wire, now } func TestPlan14RootHistoryReconstructionCrash(t *testing.T) { resolve := func(HistoryBinding) (string, error) { return "canonical", nil } if dir := os.Getenv("PLAN14_RECONSTRUCTION_CHILD"); dir != "" { pub, err := os.ReadFile(filepath.Join(dir, "pub")) if err != nil { t.Fatal(err) } wire, err := os.ReadFile(filepath.Join(dir, "wire")) if err != nil { t.Fatal(err) } s := &reconstructionCrashStore{healthyRetryStore: &healthyRetryStore{dir: dir}, stage: os.Getenv("PLAN14_RECONSTRUCTION_STAGE")} h := &RootHistory{Store: s} err = h.Reconstruct(pub, wire, "bab", "console", time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC), resolve) t.Fatalf("crash boundary not reached: %v", err) } for _, stage := range []string{"intent", "host", "app", rootBakeKey, "complete"} { t.Run(stage, func(t *testing.T) { h, s, b, pub, wire, now := reconstructionCrashFixture(t) name := stage if stage == "host" { name = historyKey(b) } if stage == "app" { name = historyAppKey(b.Account, b.App) } for key, data := range map[string][]byte{"pub": pub, "wire": wire} { if err := os.WriteFile(filepath.Join(s.dir, key), data, 0600); err != nil { t.Fatal(err) } } cmd := exec.Command(os.Args[0], "-test.run=^TestPlan14RootHistoryReconstructionCrash$") cmd.Env = append(os.Environ(), "PLAN14_RECONSTRUCTION_CHILD="+s.dir, "PLAN14_RECONSTRUCTION_STAGE="+name) out, err := cmd.CombinedOutput() var exit *exec.ExitError if !errors.As(err, &exit) || exit.ExitCode() != 83 { t.Fatalf("child: %v %s", err, out) } var accepted HistoryReconstruction if err := VerifyRootEnvelope(pub, wire, &accepted); err != nil { t.Fatal(err) } acceptedRaw, _ := json.Marshal(accepted) view, err := h.MigrationStatus(b, resolve) if err != nil { t.Fatal(err) } want := "" if stage == "complete" { want = reconstructionDigest(acceptedRaw) } if view.LastReconstructionDigest != want { t.Fatal("crash reported wrong completion") } if stage != "complete" { if _, err := h.Refusals(b); err == nil { t.Fatal("partial reconstruction granted refusal completeness") } if _, err := h.Programs(b); err == nil { t.Fatal("partial reconstruction granted program history") } if _, err := h.AppRefusals(b.Account, b.App); err == nil { t.Fatal("partial reconstruction granted mint inventory") } if _, err := h.LoadBake([]HistoryBinding{b}, now); err == nil { t.Fatal("partial reconstruction granted bake completeness") } if err := h.ResumeReconstruction(func(HistoryBinding) (string, error) { return "changed", nil }); err == nil { t.Fatal("changed canonical source resumed") } } if err := h.ResumeReconstruction(resolve); err != nil { t.Fatal(err) } view, err = h.MigrationStatus(b, resolve) if err != nil || view.LastReconstructionDigest != reconstructionDigest(acceptedRaw) || view.Pending { t.Fatalf("resume completion: %+v %v", view, err) } if _, err := h.AppRefusals(b.Account, b.App); err != nil { t.Fatal(err) } before, _ := s.Read(rootBakeKey) if err := h.ResumeReconstruction(resolve); err != nil { t.Fatal(err) } after, _ := s.Read(rootBakeKey) if !bytes.Equal(before, after) { t.Fatal("resume changed committed clock") } }) } } func TestPlan14RootHistoryReconstructionVisibleSync(t *testing.T) { h, s, b, pub, wire, now := reconstructionCrashFixture(t) resolve := func(HistoryBinding) (string, error) { return "canonical", nil } fault := &reconstructionCrashStore{healthyRetryStore: s, stage: historyKey(b), fail: true} h.Store = fault if err := h.Reconstruct(pub, wire, b.Account, b.App, now, resolve); !errors.Is(err, s.fault) { t.Fatalf("write: %v", err) } if _, err := h.Refusals(b); err == nil { t.Fatal("visible unsynced reconstruction granted permission") } fault.stage = "" if err := h.ResumeReconstruction(resolve); err != nil { t.Fatal(err) } if _, err := h.Refusals(b); err != nil { t.Fatal(err) } } func TestPlan14RootHistoryReconstruction(t *testing.T) { for _, attack := range []string{"valid", "signature", "identity", "revision", "source", "expired", "future", "pending", "omitted-deny"} { t.Run(attack, func(t *testing.T) { h, disk, b := rootHistoryFixture(t) now := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) ref := Refusal{Schema: 1, Host: b.Host, App: b.App, Version: "bad", RefusedAt: now, PointerPromotedAt: now} if err := h.AddRefusal(b, ref); err != nil { t.Fatal(err) } status, err := h.ReconstructionStatus() if err != nil { t.Fatal(err) } pub, priv, _ := ed25519.GenerateKey(rand.Reader) g := HistoryReconstruction{Schema: 1, Kind: "carlos-release-history", Identity: status.Identity, Revision: status.Revision, Operation: "op1", IssuedAt: now, Account: b.Account, App: b.App, Hosts: []HistoryReconstructionHost{{Host: b.Host, Source: "canonical", Revision: 2, Refusals: []Refusal{ref}}}, Holds: map[string]HistoryReconstructionHold{"stable": {Window: time.Hour, SetAt: now}}, Production: true, ProductionSetAt: now} switch attack { case "identity": g.Identity = "other-box" case "revision": g.Revision++ case "source": g.Hosts[0].Source = "forged" case "expired": g.IssuedAt = now.Add(-25 * time.Hour) case "future": g.IssuedAt = now.Add(2 * time.Minute) case "pending": if err := h.Begin(b, HistoryAttempt{ID: "attempt", Version: "bad", Source: "canonical", Started: now}); err != nil { t.Fatal(err) } g.Hosts[0].Revision++ case "omitted-deny": g.Hosts[0].Refusals = nil } wire, err := SignRootEnvelope(RootSigner(priv), g) if err != nil { t.Fatal(err) } if attack == "signature" { pub, _, _ = ed25519.GenerateKey(rand.Reader) } resolve := func(HistoryBinding) (string, error) { return "canonical", nil } err = h.Reconstruct(pub, wire, b.Account, b.App, now, resolve) if attack != "valid" && attack != "omitted-deny" { if err == nil { t.Fatal("invalid reconstruction accepted") } return } if err != nil { t.Fatal(err) } refs, err := h.AppRefusals(b.Account, b.App) if err != nil || len(refs) != 1 || refs[0].Version != "bad" { t.Fatalf("retained denials: %v %v", refs, err) } clock, err := h.LoadBake([]HistoryBinding{b}, now) if err != nil || clock.Holds[channelClockKey(b.Account, b.App, "stable")].Window != time.Hour { t.Fatalf("bake: %v %v", clock, err) } before, _ := disk.Read(rootBakeKey) if err := h.Reconstruct(pub, wire, b.Account, b.App, now.Add(time.Hour), resolve); err != nil { t.Fatal(err) } after, _ := disk.Read(rootBakeKey) if string(before) != string(after) { t.Fatal("replay changed bake observation") } if err := os.WriteFile(filepath.Join(t.TempDir(), "refusal.json"), []byte(`{"recovered":true}`), 0600); err != nil { t.Fatal(err) } if blocked, err := h.Blocks(b, &ChannelPointer{Version: "bad", PromotedAt: now}); err != nil || !blocked { t.Fatalf("reconstruction cleared deny: %v %v", blocked, err) } }) } } func TestPlan14RootHistoryReconstructionUnknownAndFresh(t *testing.T) { h, disk, b := rootHistoryFixture(t) if err := os.Remove(filepath.Join(disk.dir, historyKey(b))); err != nil { t.Fatal(err) } status, err := h.ReconstructionStatus() if err != nil { t.Fatal(err) } if _, err := h.Refusals(b); err == nil { t.Fatal("status initialized history") } pub, priv, _ := ed25519.GenerateKey(rand.Reader) now := time.Now().UTC() g := HistoryReconstruction{Schema: 1, Kind: "carlos-release-history", Identity: status.Identity, Revision: status.Revision, Operation: "fresh", IssuedAt: now, Account: b.Account, App: b.App, Hosts: []HistoryReconstructionHost{{Host: b.Host, Source: "canonical"}}} wire, _ := SignRootEnvelope(RootSigner(priv), g) if err := h.Reconstruct(pub, wire, b.Account, b.App, now, func(HistoryBinding) (string, error) { return "canonical", nil }); err != nil { t.Fatal(err) } if _, err := h.Programs(b); err == nil { t.Fatal("reconstruction invented installed programs") } if refs, err := h.Refusals(b); err != nil || len(refs) != 0 { t.Fatalf("explicit empty reconstruction: %v %v", refs, err) } } func TestPlan14RootHistoryObservationKeepsReconstructionRevision(t *testing.T) { h, _, b := rootHistoryFixture(t) before, err := h.read(b) if err != nil { t.Fatal(err) } if err := h.RecordInstalledProgram(b, "old/program"); err != nil { t.Fatal(err) } after, err := h.read(b) if err != nil { t.Fatal(err) } if after.Revision != before.Revision { t.Fatalf("repeat observation invalidated operator reconstruction: %d -> %d", before.Revision, after.Revision) } if err := h.RecordInstalledProgram(b, "new/program"); err != nil { t.Fatal(err) } after, err = h.read(b) if err != nil { t.Fatal(err) } if after.Revision != before.Revision+1 { t.Fatal("new program did not advance revision") } } func TestPlan14RootHistoryReconstructionPreservesOtherRecovery(t *testing.T) { h, _, b, pub, wire, now := reconstructionCrashFixture(t) other := b other.Host = "other.example.com" other.App = "another" if err := h.write(historyRecord{Schema: 1, Binding: other, Revision: 1, Programs: historyPrograms{Revision: 1, Complete: true}, Refusals: historyRefusals{Revision: 1, Complete: true}, Attempt: &HistoryAttempt{ID: "unresolved", Version: "bad", Source: "source", Started: now}}); err != nil { t.Fatal(err) } if err := h.Reconstruct(pub, wire, b.Account, b.App, now, func(HistoryBinding) (string, error) { return "canonical", nil }); err == nil { t.Fatal("reconstruction could block unrelated automatic recovery") } if pending, err := h.Pending(other); err != nil || pending == nil || pending.ID != "unresolved" { t.Fatalf("recovery unavailable: %v %v", pending, err) } } func TestPlan14RootHistoryReconstructionRestartsHistoricalWait(t *testing.T) { h, _, b, pub, wire, now := reconstructionCrashFixture(t) seedRootBake(t, h, b, now.Add(-24*time.Hour)) clock, err := h.LoadBake([]HistoryBinding{b}, now.Add(-24*time.Hour)) if err != nil { t.Fatal(err) } clock.Observe(b.Account, b.App, "stable", "v1", now.Add(-24*time.Hour)) old := now.Add(-24 * time.Hour) clock.Production = map[string]*prodMemory{prodKey(b.Account, b.App): {WasProduction: true, NewestSetAt: old, OffSetAt: old, OffFirstSeen: old}} clock.Holds = map[string]*holdMemory{channelClockKey(b.Account, b.App, "stable"): {Window: 2 * time.Hour, Pending: time.Minute, PendingAt: old, DeletedAt: old}} clock.Channels[channelClockKey(b.Account, "other", "stable")] = map[string]time.Time{"v1": old} if err := clock.Save(); err != nil { t.Fatal(err) } var grant HistoryReconstruction if err := VerifyRootEnvelope(pub, wire, &grant); err != nil { t.Fatal(err) } grant.BakeRevision = 2 priv, pub := testKeys(t) wire, err = SignRootEnvelope(RootSigner(priv), grant) if err != nil { t.Fatal(err) } if err := h.Reconstruct(pub, wire, b.Account, b.App, now, func(HistoryBinding) (string, error) { return "canonical", nil }); err != nil { t.Fatal(err) } clock, err = h.LoadBake([]HistoryBinding{b}, now) if err != nil { t.Fatal(err) } eligible, ok := clock.EligibleAt(b.Account, b.App, "stable", "v1", map[string]time.Duration{"stable": time.Hour}) if !ok || eligible.Before(now.Add(time.Hour)) { t.Fatalf("reconstructed bound inherited elapsed time: %v %v", eligible, ok) } mem := clock.Holds[channelClockKey(b.Account, b.App, "stable")] if mem.Window != 2*time.Hour || !mem.PendingAt.Equal(now) || !mem.DeletedAt.Equal(now) { t.Fatalf("historical hold changed: %+v", mem) } prod := clock.Production[prodKey(b.Account, b.App)] if !prod.WasProduction || !prod.OffFirstSeen.Equal(now) { t.Fatalf("historical production changed: %+v", prod) } if !clock.Channels[channelClockKey(b.Account, "other", "stable")]["v1"].Equal(old) { t.Fatal("reconstruction changed another app's clock") } }