package host import ( "crypto/ed25519" "encoding/json" "os" "strings" "testing" "time" "github.com/carlosframework/platform/internal/historyreadback" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootstate" ) func TestPlan14RecoveryRetirementProductionCallback(t *testing.T) { for _, replacement := range []bool{false, true} { t.Run(map[bool]string{false: "reconstruct", true: "replace"}[replacement], func(t *testing.T) { e, u, c, source, pub := signedRecoveryFixture(t) rt, err := e.root.reg.Get(observedHost) if err != nil { t.Fatal(err) } h := seedHostRootHistory(t, *rt) n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink} if _, err := n.recover(t.Context(), rt.Host, c, func(string) string { return "5s" }, e.root.reg); err != nil { t.Fatal(err) } disk, err := c.attempts() if err != nil { t.Fatal(err) } good, err := readRollbackAttempt(disk, rt.Host) if err != nil { t.Fatal(err) } old := good old.ID = strings.Repeat("1", 32) old.Verified = false attempt := *good.History attempt.ID = old.ID attempt.Version = "v0" attempt.SHA256 = "old-digest" attempt.Started = attempt.Started.Add(-time.Hour) attempt.PromotedAt = attempt.PromotedAt.Add(-time.Hour) old.History = &attempt if err := writeRollbackState(disk, recoveryResultKey(old), old); err != nil { t.Fatal(err) } b := good.HistoryBinding signingPub, key, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } projection := &rollbackMemory{files: map[string][]byte{}} now := time.Now().UTC() maintenance := rootHistoryMaintenance{history: h, store: e.store, source: source, checks: c, pub: func() (ed25519.PublicKey, error) { return signingPub, nil }, projection: func() (rootstate.Store, error) { return projection, nil }, now: func() time.Time { return now }} directory, responder, scope := completionReadbackFixture(t, e.store, signingPub, key, b, now) maintenance.readback = responder.publish challenge := func() historyreadback.Request { t.Helper() q, err := historyreadback.RequestStatus(t.Context(), directory, release.RootSigner(key), scope, now) if err != nil { t.Fatal(err) } return q } q := challenge() deps := boundRootHistory(h, source.resolve, e.root.obs) deps.Reconstruct = maintenance.run if err := deps.Reconstruct(t.Context(), b); err == nil { t.Fatal("ambiguous original did not restrict reconstruction result") } if _, err := c.mintRefusals(h, b); err == nil { t.Fatal("unsigned ambiguity allowed mint") } var view release.HistoryMigrationStatus raw, err := projection.Read(release.HistoryStatusKey(b.Account, b.App)) if err != nil { t.Fatal(err) } if err := json.Unmarshal(raw, &view); err != nil { t.Fatal(err) } if len(view.Hosts) != 1 || len(view.Hosts[0].Recoveries) != 2 { t.Fatalf("independent evidence missing from status: %+v", view) } grant := release.HistoryReconstruction{Schema: 1, Kind: "carlos-release-history", Identity: view.Identity, Revision: view.Revision, Operation: "reconstruct-with-recovery", IssuedAt: now, Account: b.Account, App: b.App, AppRevision: view.AppRevision, BakeRevision: view.BakeRevision, Hosts: []release.HistoryReconstructionHost{view.Hosts[0].HistoryReconstructionHost}} if replacement { prior := grant prior.Operation = "interrupted-history" prior.Hosts = append([]release.HistoryReconstructionHost(nil), grant.Hosts...) prior.Hosts[0].Source = "previous-source" prior.Holds = map[string]release.HistoryReconstructionHold{"stable": {Window: time.Hour, SetAt: now}} wire, err := release.SignRootEnvelope(release.RootSigner(key), prior) if err != nil { t.Fatal(err) } base := h.Store h.Store = &historyIntentFailure{Store: base} if err := h.Reconstruct(signingPub, wire, b.Account, b.App, now, func(release.HistoryBinding) (string, error) { return "previous-source", nil }); err == nil { t.Fatal("missing interrupted intent") } h.Store = base if err := deps.Reconstruct(t.Context(), b); err == nil { t.Fatal("changed source resumed old intent") } raw, err := projection.Read(release.HistoryStatusKey(b.Account, b.App)) if err != nil { t.Fatal(err) } var pending release.HistoryMigrationStatus if err := json.Unmarshal(raw, &pending); err != nil { t.Fatal(err) } if !pending.Pending || pending.PendingDigest == "" { t.Fatal("no pending status") } grant.Replaces = pending.PendingDigest } if err := publishHistoryFixture(t.Context(), e.store, release.RootSigner(key), release.HistoryReconstructionKey(grant.Account, grant.App, grant.Identity, grant.Revision), grant); err != nil { t.Fatal(err) } if err := deps.Reconstruct(t.Context(), b); err == nil { t.Fatal("signed reconstruction erased ambiguity") } receipt, err := historyreadback.ReadReconstruction(t.Context(), directory, q, grant, now) if err != nil { t.Fatal("completed reconstruction lacks exact readback", err) } if len(receipt.Status.RecoverySources.Unresolved) != 1 { t.Fatal("reconstruction cleared original ambiguity") } other := grant other.Operation = "not-accepted" if _, err := historyreadback.ReadReconstruction(t.Context(), directory, q, other, now); err == nil { t.Fatal("different authorization reported complete") } retirement := release.HistoryRecoveryRetirement{Schema: 1, Kind: "carlos-release-recovery-retirement", Identity: view.Identity, Operation: "retire-exact-ambiguity", IssuedAt: now, Binding: b, Source: view.Hosts[0].Source, AttemptID: old.ID, OutcomeDigest: view.Hosts[0].RecoveryDigests[old.ID], VerifiedID: good.ID, VerifiedDigest: view.Hosts[0].RecoveryDigests[good.ID]} if err := publishHistoryFixture(t.Context(), e.store, release.RootSigner(key), release.HistoryRecoveryRetirementKey(retirement.Binding, retirement.Identity, retirement.OutcomeDigest), retirement); err != nil { t.Fatal(err) } q = challenge() if _, err := historyreadback.ReadRetirement(t.Context(), directory, q, retirement, now); err == nil { t.Fatal("delivery counted as completion") } before := len(u.effects) if err := deps.Reconstruct(t.Context(), b); err != nil { t.Fatal(err) } if _, err := historyreadback.ReadRetirement(t.Context(), directory, q, retirement, now); err != nil { t.Fatal("retirement lacks exact readback", err) } if refs, err := c.mintRefusals(h, b); err != nil || len(refs) != 0 { t.Fatalf("exact authorized retirement blocked mint: %+v %v", refs, err) } if len(u.effects) != before { t.Fatal("bookkeeping repeated a process effect") } raw, err = projection.Read(release.HistoryStatusKey(b.Account, b.App)) if err != nil { t.Fatal(err) } view = release.HistoryMigrationStatus{} if err := json.Unmarshal(raw, &view); err != nil { t.Fatal(err) } if len(view.Hosts[0].RetiredRecoveries) != 1 || view.Hosts[0].RetiredRecoveries[0] != old.ID { t.Fatalf("retirement not projected: %+v", view) } // A stale settled copy cannot hide an outstanding source journal. old.Settled = false if err := writeRollbackState(disk, recoveryResultKey(old), old); err != nil { t.Fatal(err) } if _, err := c.mintRefusals(h, b); err == nil { t.Fatal("retirement consumed a pending original") } q = challenge() if err := deps.Reconstruct(t.Context(), b); err == nil { t.Fatal("maintenance consumed a pending original") } if _, err := historyreadback.ReadRetirement(t.Context(), directory, q, retirement, now); err == nil { t.Fatal("readback consumed pending original") } raw, err = projection.Read(release.HistoryStatusKey(b.Account, b.App)) if err != nil { t.Fatal(err) } view = release.HistoryMigrationStatus{} if err := json.Unmarshal(raw, &view); err != nil { t.Fatal(err) } if view.RecoverySources == nil || !view.RecoverySources.Known || len(view.RecoverySources.Unresolved) != 1 || view.RecoverySources.Unresolved[0].Attempt.ID != old.ID || len(view.Hosts[0].RetiredRecoveries) != 0 { t.Fatalf("pending original hidden by projection: %+v", view) } }) } } func TestPlan14IndependentRecoveryPublishesStatus(t *testing.T) { for _, state := range []string{"success", "failed rollback", "history unavailable"} { t.Run(state, func(t *testing.T) { e, u, c, source, pub := signedRecoveryFixture(t) rt, err := e.root.reg.Get(observedHost) if err != nil { t.Fatal(err) } h := seedHostRootHistory(t, *rt) projection := &rollbackMemory{files: map[string][]byte{}} m := rootHistoryMaintenance{history: h, store: e.store, source: source, checks: c, pub: func() (ed25519.PublicKey, error) { return pub, nil }, projection: func() (rootstate.Store, error) { return projection, nil }, now: time.Now} deps := boundRootHistory(h, source.resolve, e.root.obs) deps.Reconstruct = m.run c.history = func() (*release.RootHistoryPass, error) { if state == "history unavailable" { return nil, os.ErrPermission } return deps, nil } u.crashNew, u.badRollback = state == "failed rollback", state == "failed rollback" n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink} handled, err := n.recover(t.Context(), rt.Host, c, func(string) string { return "5s" }, e.root.reg) if !handled || (err != nil) != (state == "failed rollback") { t.Fatalf("history changed independent effect: %v %v", handled, err) } if state == "history unavailable" { if u.exe != release.VersionPath(u.opt, rt.Account, "v2", rt.App) { t.Fatal("unavailable history blocked fix") } return } raw, err := projection.Read(release.HistoryStatusKey(rt.Account, rt.App)) if err != nil { t.Fatal("independent status absent", err) } var view release.HistoryMigrationStatus if err := json.Unmarshal(raw, &view); err != nil { t.Fatal(err) } if len(view.Hosts) != 1 || len(view.Hosts[0].Recoveries) != 1 { t.Fatalf("wrong projected outcomes: %+v", view) } outcome := view.Hosts[0].Recoveries[0] if !outcome.Settled || outcome.Verified != (state == "success") || outcome.Failed != (state == "failed rollback") { t.Fatalf("wrong outcome: %+v", outcome) } }) } } type historyIntentFailure struct{ rootstate.Store } func (s *historyIntentFailure) Write(name string, raw []byte) error { if err := s.Store.Write(name, raw); err != nil { return err } if name == "reconstruction.state" && strings.Contains(string(raw), `"pending":`) { return os.ErrPermission } return nil } func completionReadbackFixture(t *testing.T, store release.Store, pub ed25519.PublicKey, key ed25519.PrivateKey, b release.HistoryBinding, now time.Time) (hostactors.Directory, historyReadbackPublisher, historyreadback.Scope) { t.Helper() _, evidenceKey, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } enrollment := hostfloor.Enrollment{Box: "box", Enrollment: strings.Repeat("e", 32), InstanceID: "i-box"} directory := hostactors.Directory{Store: store, PublicKey: pub, Bucket: "bucket"} roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Boxes: map[string]hostactors.Box{"box": {Enrollment: enrollment.Enrollment, Namespace: "platform", BoxID: "box", Label: "box", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-box"}, EvidenceSHA256: strings.Repeat("b", 64), State: "active", CensusPublicKey: evidenceKey.Public().(ed25519.PublicKey)}}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} wire, err := release.SignRootEnvelope(release.RootSigner(key), roster) if err != nil { t.Fatal(err) } if _, err := store.Put(t.Context(), hostactors.Key, wire, ""); err != nil { t.Fatal(err) } responder := historyReadbackPublisher{directory: directory, enrollment: enrollment, key: func() (ed25519.PrivateKey, error) { return evidenceKey, nil }, now: func() time.Time { return now }} return directory, responder, historyreadback.Scope{Bucket: "bucket", Box: enrollment.Box, Enrollment: enrollment.Enrollment, Binding: b} }