package hostclaim import ( "context" "crypto/ed25519" "encoding/json" "errors" "reflect" "strings" "testing" "time" "github.com/carlosframework/platform/internal/release" ) // Existing fixtures declare directly in the recovery phase. The end-to-end // test below supplies a separate owner for landing -> restoring. type testLossRecoveryOwner struct{} func (testLossRecoveryOwner) ValidateLossRecovery(c Claim, loss LossRecord, in Admission) error { if c.Op.Kind != loss.OpKind || c.Op.Phase != loss.OpPhase || c.Op.Exec != in.Box { return ErrWrongState } return nil } type lossRecoveryOwnerFunc func(Claim, LossRecord, Admission) error func (f lossRecoveryOwnerFunc) ValidateLossRecovery(c Claim, loss LossRecord, in Admission) error { return f(c, loss, in) } func validLossRecord(t *testing.T) LossRecord { t.Helper() now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) inc := strings.Repeat("a", 64) return LossRecord{ V: 1, ID: "loss_1", State: "prepared", Actor: "operator", AcceptedAt: now, Host: testHost, HolderIncarnation: inc, HolderStamp: 3, HolderSession: 2, ExpectedClaimGen: 4, ExpectedClaimETag: `"claim"`, OpID: "op", OpKind: KindMoving, OpPhase: "precommit", Box: "box-a", Enrollment: "enroll-a", QuarantineRevision: 9, CloudObservation: LossCloudObservation{ObservedAt: now, Account: "123", Region: "eu-west-1", InstanceID: "i-1", State: "stopped", RequestID: "request-1"}, OperatorAttestation: LossOperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true}, FencedAt: now, ConfirmedConsequences: LossConsequences{RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: json.RawMessage(`{"kind":"lost-target"}`), TimersPotentiallyLost: true}, Timers: TimerLoss{Knowledge: TimersUnknown}, WriteSources: []WriteSource{{Reason: "target", Box: "box-a", Generation: "g1", Session: 3, Region: "ie", Baseline: "snapshot:v1"}}, } } func TestLossRecordCreateOnlyConditionalAndImmutable(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } sg := release.RootSigner(priv) record := validLossRecord(t) created, err := CreateLossRecord(ctx, s, sg, record) if err != nil { t.Fatal(err) } if created.ETag == "" { t.Fatal("create returned no ETag") } retry, err := CreateLossRecord(ctx, s, sg, record) if err != nil || retry.ETag != created.ETag { t.Fatalf("idempotent create = %+v, %v", retry, err) } read, err := ReadLossRecord(ctx, s, pub, record.ID) if err != nil || !reflect.DeepEqual(read.Record, record) { t.Fatalf("read = %+v, %v", read, err) } committed := read.Record committed.State, committed.ResultClaimGen = "committed", 5 next, err := ReplaceLossRecord(ctx, s, sg, read, committed) if err != nil { t.Fatal(err) } mutated := next.Record mutated.FencedAt = mutated.FencedAt.Add(time.Second) if _, err := ReplaceLossRecord(ctx, s, sg, next, mutated); !errors.Is(err, ErrLossRecord) { t.Fatalf("immutable evidence mutation = %v", err) } if _, err := ReplaceLossRecord(ctx, s, sg, read, committed); !errors.Is(err, ErrConflict) { t.Fatalf("stale conditional replace = %v", err) } } func TestTimerKnowledgeNeverConflatesUnknownWithEmptyComplete(t *testing.T) { r := validLossRecord(t) if err := r.Validate(); err != nil { t.Fatal(err) } r.Timers.Knowledge = TimersComplete if err := r.Validate(); !errors.Is(err, ErrLossRecord) { t.Fatalf("complete without source = %v", err) } r.Timers = TimerLoss{} if err := r.Validate(); !errors.Is(err, ErrLossRecord) { t.Fatalf("empty knowledge = %v", err) } } func TestLossRecordCreateRetryUsesSerializedEvidence(t *testing.T) { for _, name := range []string{"ordinary timestamps", "spaced recovery intent", "both"} { t.Run(name, func(t *testing.T) { ctx := t.Context() s := release.NewDirStore(t.TempDir()) _, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } sg := release.RootSigner(priv) record := validLossRecord(t) if name != "spaced recovery intent" { now := time.Now() record.AcceptedAt, record.FencedAt, record.CloudObservation.ObservedAt = now, now, now } if name != "ordinary timestamps" { record.ConfirmedConsequences.RecoveryIntent = json.RawMessage(`{ "kind": "lost-target", "txid": 9007199254740993 }`) } created, err := CreateLossRecord(ctx, s, sg, record) if err != nil { t.Fatal(err) } retry, err := CreateLossRecord(ctx, s, sg, record) if err != nil || retry.ETag != created.ETag { t.Errorf("identical serialized evidence retry = %+v, %v", retry, err) } changed := record changed.FencedAt = changed.FencedAt.Add(time.Nanosecond) if _, err := CreateLossRecord(ctx, s, sg, changed); !errors.Is(err, ErrLossRecord) { t.Fatalf("changed evidence accepted: %v", err) } changed = record changed.ConfirmedConsequences.RecoveryIntent = json.RawMessage(`{"kind":"lost-target","txid":9007199254740992}`) if _, err := CreateLossRecord(ctx, s, sg, changed); !errors.Is(err, ErrLossRecord) { t.Fatalf("changed recovery intent accepted: %v", err) } fresh, err := ReadLossRecord(ctx, s, sg.RootPublicKey(), record.ID) if err != nil || fresh.ETag != created.ETag { t.Fatalf("retry changed stored evidence: %v", err) } }) } } func TestAdmissionLossRetentionRequiresExactDurableCopy(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } lost := WritesLost{Box: "target", Generation: "g-target", Session: 4, RemoteMaxTXID: 12} source := WritesLost{Box: "source", Generation: "g-source", Session: 2, RemoteMaxTXID: 10} obs := servingClaim(t, ctx, s) obs, err = Update(ctx, s, obs, func(c *Claim) error { c.LossRefs = []string{"loss_1"} c.Holder.Admissions = []AdmissionRecord{{Region: "ie", Session: 8, LossID: "loss_1", WritesLost: &lost, AdditionalWritesLost: []WritesLost{source}}} return nil }) if err != nil { t.Fatal(err) } if err := RequireLossEvidenceArchived(obs.Claim); !errors.Is(err, ErrLossEvidencePending) { t.Fatalf("unarchived evidence = %v", err) } record := validLossRecord(t) record.Host = obs.Host created, err := CreateLossRecord(ctx, s, release.RootSigner(priv), record) if err != nil { t.Fatal(err) } exact := []ArchivedAdmissionLoss{{LossID: "loss_1", AdmissionClaimGen: 8, WritesLost: &lost, AdditionalWritesLost: []WritesLost{source}}} committed := created.Record committed.State, committed.ResultClaimGen, committed.ArchivedAdmissionLosses = "committed", obs.Claim.Gen, exact if _, err := ReplaceLossRecord(ctx, s, release.RootSigner(priv), created, committed); err != nil { t.Fatal(err) } archived, err := ArchiveAdmissionLosses(ctx, s, pub, obs, "loss_1") if err != nil { t.Fatal(err) } if err := RequireLossEvidenceArchived(archived.Claim); err != nil { t.Fatal(err) } } func TestLossBearingAdmissionRequiresEveryFrozenTargetAndSourceCut(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } obs := servingClaim(t, ctx, s) obs, err = Update(ctx, s, obs, func(c *Claim) error { c.Op = Op{Kind: KindReverting, ID: "return", Phase: "recover", Exec: "recovery", Participants: []string{"recovery"}} c.Holder.WriterBox, c.Holder.Generation, c.Holder.Session = "target", "g-target", 4 c.LossRefs = []string{"loss_1"} return nil }) if err != nil { t.Fatal(err) } earlier := WritesLost{Box: "target", Generation: "g-earlier", Session: 3, RemoteMaxTXID: 7} source := WritesLost{Box: "source", Generation: "g-source", Session: 2, RemoteMaxTXID: 10} in := Admission{ObservedOpID: "return", ObservedPhase: "recover", ObservedSession: 4, ObservedGeneration: "g-target", Box: "recovery", Mode: "cold", NewGeneration: "g-recovery", Region: "ie", PredecessorRegion: "ie", RemoteMaxTXID: 12, AdditionalWritesLost: []WritesLost{earlier, source}} if _, err := Update(ctx, s, obs, AdmitWriter(Admission{ObservedOpID: in.ObservedOpID, ObservedPhase: in.ObservedPhase, ObservedSession: in.ObservedSession, ObservedGeneration: in.ObservedGeneration, Box: in.Box, Mode: in.Mode, NewGeneration: in.NewGeneration, Region: in.Region, PredecessorRegion: in.PredecessorRegion, LossID: "loss_1", AdditionalWritesLost: in.AdditionalWritesLost})); !errors.Is(err, ErrWrongState) { t.Fatalf("direct loss admission = %v", err) } record := validLossRecord(t) record.Host = obs.Host record.HolderIncarnation, record.HolderStamp, record.HolderSession = obs.Claim.Holder.Incarnation, obs.Claim.Holder.Stamp, 4 record.OpID, record.OpKind, record.OpPhase, record.Box = "return", KindReverting, "recover", "target" record.WriteSources = []WriteSource{ {Reason: "target", Box: "target", Generation: "g-target", Session: 4, Region: "ie", Baseline: "target-snapshot"}, {Reason: "target", Box: "target", Generation: "g-earlier", Session: 3, Region: "ie", Baseline: "earlier-snapshot"}, {Reason: "source-cut", Box: "source", Generation: "g-source", Session: 2, Region: "ie", Baseline: "source-seal"}, } created, err := CreateLossRecord(ctx, s, release.RootSigner(priv), record) if err != nil { t.Fatal(err) } committed := created.Record committed.State, committed.ResultClaimGen = "committed", obs.Claim.Gen if _, err := ReplaceLossRecord(ctx, s, release.RootSigner(priv), created, committed); err != nil { t.Fatal(err) } validated := []ValidatedWriteSource{ {Reason: "target", Box: "target", Generation: "g-target", Session: 4, Region: "ie", Baseline: "target-snapshot", RemoteMaxTXID: in.RemoteMaxTXID}, {Reason: "target", Box: "target", Generation: "g-earlier", Session: 3, Region: "ie", Baseline: "earlier-snapshot", RemoteMaxTXID: earlier.RemoteMaxTXID}, {Reason: "source-cut", Box: "source", Generation: "g-source", Session: 2, Region: "ie", Baseline: "source-seal", RemoteMaxTXID: source.RemoteMaxTXID}, } for name, mutate := range map[string]func(*Admission, *[]ValidatedWriteSource){ "omitted earlier target": func(a *Admission, _ *[]ValidatedWriteSource) { a.AdditionalWritesLost = a.AdditionalWritesLost[1:] }, "omitted earlier target with same txid": func(a *Admission, v *[]ValidatedWriteSource) { (*v)[1].RemoteMaxTXID = a.RemoteMaxTXID a.AdditionalWritesLost = a.AdditionalWritesLost[1:] }, "omitted source cut": func(a *Admission, _ *[]ValidatedWriteSource) { a.AdditionalWritesLost = a.AdditionalWritesLost[:1] }, "omitted all additional": func(a *Admission, _ *[]ValidatedWriteSource) { a.AdditionalWritesLost = nil }, "earlier target uses predecessor txid": func(a *Admission, _ *[]ValidatedWriteSource) { a.AdditionalWritesLost[0].RemoteMaxTXID = a.RemoteMaxTXID }, "source cut uses predecessor txid": func(a *Admission, _ *[]ValidatedWriteSource) { a.AdditionalWritesLost[1].RemoteMaxTXID = a.RemoteMaxTXID }, "predecessor uses earlier txid": func(a *Admission, _ *[]ValidatedWriteSource) { a.RemoteMaxTXID = earlier.RemoteMaxTXID }, "duplicate box session with another generation": func(a *Admission, _ *[]ValidatedWriteSource) { duplicate := earlier duplicate.Generation = "another-generation" a.AdditionalWritesLost = append(a.AdditionalWritesLost, duplicate) }, "unfrozen tuple": func(a *Admission, _ *[]ValidatedWriteSource) { a.AdditionalWritesLost[0].Session++ }, "omitted baseline": func(_ *Admission, v *[]ValidatedWriteSource) { *v = (*v)[:2] }, "changed baseline": func(_ *Admission, v *[]ValidatedWriteSource) { (*v)[1].Baseline = "another-snapshot" }, } { t.Run(name, func(t *testing.T) { candidate := in candidate.AdditionalWritesLost = append([]WritesLost(nil), in.AdditionalWritesLost...) evidence := append([]ValidatedWriteSource(nil), validated...) mutate(&candidate, &evidence) if _, err := AdmitWriterWithLoss(ctx, s, pub, obs, candidate, "loss_1", evidence, testLossRecoveryOwner{}); !errors.Is(err, ErrLossRecord) { t.Fatalf("invalid declaration admitted: %v", err) } fresh, err := Read(ctx, s, obs.Host) if err != nil || fresh.ETag != obs.ETag { t.Fatalf("refusal changed claim: %v", err) } }) } admitted, err := AdmitWriterWithLoss(ctx, s, pub, obs, in, "loss_1", validated, testLossRecoveryOwner{}) if err != nil { t.Fatal(err) } last := admitted.Claim.Holder.Admissions[len(admitted.Claim.Holder.Admissions)-1] if last.LossID != "loss_1" || !reflect.DeepEqual(last.AdditionalWritesLost, []WritesLost{earlier, source}) || last.WritesLost == nil || last.WritesLost.RemoteMaxTXID != 12 || last.WritesLost.Session != 4 { t.Fatalf("loss admission = %+v", last) } t.Run("retry validates authorization and evidence", func(t *testing.T) { denied := errors.New("recovery no longer authorized") for _, name := range []string{ "identical", "primary txid", "primary txid and baseline", "region", "predecessor region", "observed session", "observed generation", "observed phase", "observed operation", "certified", "coverage", "additional loss", "missing baseline", "changed baseline", "owner denial", "operation", "phase", "executor", "incarnation", "signature", "missing declaration", } { t.Run(name, func(t *testing.T) { candidate := in candidate.AdditionalWritesLost = append([]WritesLost(nil), in.AdditionalWritesLost...) evidence := append([]ValidatedWriteSource(nil), validated...) observation := admitted observation.Claim, err = cloneClaim(admitted.Claim) if err != nil { t.Fatal(err) } key := pub var store release.Store = s switch name { case "primary txid": candidate.RemoteMaxTXID++ case "primary txid and baseline": candidate.RemoteMaxTXID++ evidence[0].RemoteMaxTXID++ case "region": candidate.Region = "uk" case "predecessor region": candidate.PredecessorRegion = "uk" case "observed session": candidate.ObservedSession++ case "observed generation": candidate.ObservedGeneration = "other" case "observed phase": candidate.ObservedPhase = "other" case "observed operation": candidate.ObservedOpID = "other" case "certified": candidate.PredecessorCertified = true case "coverage": candidate.Coverage = "sealed" case "additional loss": candidate.AdditionalWritesLost[0].RemoteMaxTXID++ case "missing baseline": evidence = nil case "changed baseline": evidence[0].Baseline = "other" case "operation": observation.Claim.Op.ID = "later-operation" candidate.ObservedOpID = "later-operation" case "phase": observation.Claim.Op.Phase = "other" candidate.ObservedPhase = "other" case "executor": observation.Claim.Op.Exec = "other" case "incarnation": observation.Claim.Holder.Incarnation = strings.Repeat("b", 64) case "signature": key = make(ed25519.PublicKey, ed25519.PublicKeySize) case "missing declaration": store = release.NewDirStore(t.TempDir()) } calls := 0 owner := lossRecoveryOwnerFunc(func(c Claim, loss LossRecord, a Admission) error { calls++ if name == "owner denial" { return denied } return (testLossRecoveryOwner{}).ValidateLossRecovery(c, loss, a) }) retry, err := AdmitWriterWithLoss(ctx, store, key, observation, candidate, record.ID, evidence, owner) if name == "identical" { if err != nil || retry.ETag != admitted.ETag || !reflect.DeepEqual(retry.Claim, admitted.Claim) || calls != 1 { t.Errorf("identical retry: err %v, owner calls %d, observation %+v", err, calls, retry) } } else if err == nil { t.Errorf("invalid retry accepted") } else if name == "owner denial" && !errors.Is(err, denied) { t.Errorf("owner denial = %v", err) } fresh, err := Read(ctx, s, admitted.Host) if err != nil || fresh.ETag != admitted.ETag { t.Fatalf("retry changed claim: %v", err) } }) } }) } func TestLossAdmissionRetryInheritsPredecessorRegion(t *testing.T) { for _, region := range []string{"ie", "uk"} { for _, fallback := range []string{"", "fr"} { t.Run(region+"/fallback="+fallback, func(t *testing.T) { ctx := t.Context() s := release.NewDirStore(t.TempDir()) pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } obs := servingClaim(t, ctx, s) obs = updateGenerationTest(t, ctx, s, obs, AdmitWriter(Admission{ Region: "ie", Box: obs.Claim.Op.Exec, Mode: "cold", NewGeneration: "g-target", PredecessorCertified: true, })) obs = updateGenerationTest(t, ctx, s, obs, func(c *Claim) error { c.Op = Op{Kind: KindReverting, ID: "return", Phase: "recover", Exec: "recovery", Participants: []string{"recovery"}} c.LossRefs = []string{"loss_1"} return nil }) h := obs.Claim.Holder record := validLossRecord(t) record.Host, record.HolderIncarnation = obs.Host, h.Incarnation record.HolderStamp, record.HolderSession = h.Stamp, h.Session record.OpID, record.OpKind, record.OpPhase, record.Box = "return", KindReverting, "recover", h.WriterBox record.WriteSources = []WriteSource{{Reason: "target", Box: h.WriterBox, Generation: h.Generation, Session: h.Session, Region: "ie", Baseline: "target-snapshot"}} created, err := CreateLossRecord(ctx, s, release.RootSigner(priv), record) if err != nil { t.Fatal(err) } committed := created.Record committed.State, committed.ResultClaimGen = "committed", obs.Claim.Gen if _, err := ReplaceLossRecord(ctx, s, release.RootSigner(priv), created, committed); err != nil { t.Fatal(err) } in := Admission{ObservedOpID: "return", ObservedPhase: "recover", ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: "recovery", Mode: "cold", NewGeneration: "g-recovery", Region: region, PredecessorRegion: fallback, RemoteMaxTXID: 12} validated := []ValidatedWriteSource{{Reason: "target", Box: h.WriterBox, Generation: h.Generation, Session: h.Session, Region: "ie", Baseline: "target-snapshot", RemoteMaxTXID: 12}} admitted, err := AdmitWriterWithLoss(ctx, s, pub, obs, in, record.ID, validated, testLossRecoveryOwner{}) if err != nil { t.Fatal(err) } last := admitted.Claim.Holder.Admissions[len(admitted.Claim.Holder.Admissions)-1] if last.PredecessorRegion != "ie" || last.Region != region { t.Fatalf("admission regions = %q / %q, want %q / %q", last.Region, last.PredecessorRegion, region, "ie") } calls := 0 owner := lossRecoveryOwnerFunc(func(c Claim, loss LossRecord, a Admission) error { calls++ return (testLossRecoveryOwner{}).ValidateLossRecovery(c, loss, a) }) retry, err := AdmitWriterWithLoss(ctx, s, pub, admitted, in, record.ID, validated, owner) if err != nil || retry.ETag != admitted.ETag || !reflect.DeepEqual(retry.Claim, admitted.Claim) || calls != 1 { t.Errorf("identical retry: err %v, owner calls %d, observation %+v", err, calls, retry) } assertAdmissionStored(t, ctx, s, admitted) }) } } } func TestTargetOnlyLossAdmissionRequiresColdUncoveredRecovery(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } obs := servingClaim(t, ctx, s) obs, err = Update(ctx, s, obs, func(c *Claim) error { c.Op = Op{Kind: KindReverting, ID: "return", Phase: "recover", Exec: "recovery", Participants: []string{"recovery"}} c.Holder.WriterBox, c.Holder.Generation, c.Holder.Session = "target", "g-target", 4 c.LossRefs = []string{"loss_1"} return nil }) if err != nil { t.Fatal(err) } record := validLossRecord(t) record.Host, record.HolderIncarnation = obs.Host, obs.Claim.Holder.Incarnation record.HolderStamp, record.HolderSession = obs.Claim.Holder.Stamp, 4 record.OpID, record.OpKind, record.OpPhase, record.Box = "return", KindReverting, "recover", "target" record.WriteSources = []WriteSource{{Reason: "target", Box: "target", Generation: "g-target", Session: 4, Region: "ie", Baseline: "target-snapshot"}} created, err := CreateLossRecord(ctx, s, release.RootSigner(priv), record) if err != nil { t.Fatal(err) } committed := created.Record committed.State, committed.ResultClaimGen = "committed", obs.Claim.Gen if _, err := ReplaceLossRecord(ctx, s, release.RootSigner(priv), created, committed); err != nil { t.Fatal(err) } base := Admission{ObservedOpID: "return", ObservedPhase: "recover", ObservedSession: 4, ObservedGeneration: "g-target", Box: "recovery", Mode: "cold", NewGeneration: "g-recovery", Region: "ie", PredecessorRegion: "ie", RemoteMaxTXID: 12} validated := []ValidatedWriteSource{{Reason: "target", Box: "target", Generation: "g-target", Session: 4, Region: "ie", Baseline: "target-snapshot", RemoteMaxTXID: 12}} for name, mutate := range map[string]func(*Admission){ "warm": func(a *Admission) { a.Mode, a.NewGeneration = "warm", "" }, "certified": func(a *Admission) { a.PredecessorCertified = true }, "sealed": func(a *Admission) { a.Coverage = "sealed" }, } { t.Run(name, func(t *testing.T) { in := base mutate(&in) if _, err := AdmitWriterWithLoss(ctx, s, pub, obs, in, "loss_1", validated, testLossRecoveryOwner{}); !errors.Is(err, ErrLossRecord) { t.Fatalf("unsafe target-loss mode = %v", err) } }) } fresh, err := Read(ctx, s, obs.Host) if err != nil || fresh.Claim.Gen != obs.Claim.Gen { t.Fatalf("refusals mutated claim: gen %d, err %v", fresh.Claim.Gen, err) } } func TestLossDeclarationHandoverAdmission(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } sg := release.RootSigner(priv) obs := servingClaim(t, ctx, s) obs, err = Update(ctx, s, obs, func(c *Claim) error { c.Op = Op{Kind: KindLanding, ID: "move", Phase: "probe", Exec: "target", Participants: []string{"target", "recovery"}} c.Holder.WriterBox, c.Holder.Generation, c.Holder.Session = "target", "g-target", 4 return nil }) if err != nil { t.Fatal(err) } record := validLossRecord(t) record.Host, record.HolderIncarnation = obs.Host, obs.Claim.Holder.Incarnation record.HolderStamp, record.HolderSession = obs.Claim.Holder.Stamp, obs.Claim.Holder.Session record.ExpectedClaimGen, record.ExpectedClaimETag = obs.Claim.Gen, obs.ETag record.OpID, record.OpKind, record.OpPhase, record.Box = "move", KindLanding, "probe", "target" record.WriteSources = []WriteSource{ {Reason: "target", Box: "target", Generation: "g-target", Session: 4, Region: "ie", Baseline: "target-snapshot"}, {Reason: "source-cut", Box: "source", Generation: "g-source", Session: 2, Region: "ie", Baseline: "source-seal"}, } prepared, err := CreateLossRecord(ctx, s, sg, record) if err != nil { t.Fatal(err) } declared, err := DeclareLost(ctx, s, pub, obs, record.ID, testLossOwner{}) if err != nil { t.Fatal(err) } if declared.Claim.Op.Kind != KindLanding || declared.Claim.Op.Phase != "probe" || declared.Claim.Op.Exec != "none" || declared.Claim.Holder.Stamp != record.HolderStamp || declared.Claim.Holder.Session != record.HolderSession { t.Fatalf("declaration changed recovery authority: %+v", declared.Claim) } committed := prepared.Record committed.State, committed.ResultClaimGen = "committed", declared.Claim.Gen if _, err := ReplaceLossRecord(ctx, s, sg, prepared, committed); err != nil { t.Fatal(err) } // The operation owner hands execution to a new installation in the same // operation, preserving the original declaration and writer provenance. recoveryStamp := declared.Claim.Gen + 1 handover, err := Update(ctx, s, declared, func(c *Claim) error { if c.Op.Kind != KindLanding || c.Op.Phase != "probe" || c.Op.Exec != "none" || !lossRefPresent(c, record.ID) { return ErrWrongState } c.Op.Kind, c.Op.Phase, c.Op.Exec = KindRestoring, "restore", "recovery" c.Op.Journal["recovery.loss"] = record.ID c.Holder.Stamp = recoveryStamp return nil }) if err != nil { t.Fatal(err) } source := WritesLost{Box: "source", Generation: "g-source", Session: 2, RemoteMaxTXID: 10} in := Admission{ObservedOpID: "move", ObservedPhase: "restore", ObservedSession: 4, ObservedGeneration: "g-target", Box: "recovery", Mode: "cold", NewGeneration: "g-recovery", Region: "ie", PredecessorRegion: "ie", RemoteMaxTXID: 12, AdditionalWritesLost: []WritesLost{source}} validated := []ValidatedWriteSource{ {Reason: "target", Box: "target", Generation: "g-target", Session: 4, Region: "ie", Baseline: "target-snapshot", RemoteMaxTXID: 12}, {Reason: "source-cut", Box: "source", Generation: "g-source", Session: 2, Region: "ie", Baseline: "source-seal", RemoteMaxTXID: 10}, } owner := lossRecoveryOwnerFunc(func(c Claim, loss LossRecord, a Admission) error { if loss.OpKind != KindLanding || loss.OpPhase != "probe" || c.Op.Kind != KindRestoring || c.Op.Phase != "restore" || c.Op.Exec != "recovery" || a.Box != c.Op.Exec || c.Holder.Stamp != recoveryStamp || c.Op.Journal["recovery.loss"] != loss.ID || c.Op.Journal["loss."+loss.ID] != `{"kind":"lost-target"}` { return ErrWrongState } // A policy callback cannot rewrite any input used by the admission. c.Holder.Incarnation = "changed" c.Op.Journal["recovery.loss"] = "changed" loss.WriteSources[0].Baseline = "changed" a.AdditionalWritesLost[0].RemoteMaxTXID = 999 return nil }) if _, err := AdmitWriterWithLoss(ctx, s, pub, declared, in, record.ID, validated, owner); !errors.Is(err, ErrWrongState) { t.Fatalf("admitted before handover: %v", err) } if _, err := AdmitWriterWithLoss(ctx, s, pub, handover, in, record.ID, validated, nil); !errors.Is(err, ErrLossOwnerRequired) { t.Fatalf("admitted without owner: %v", err) } for name, mutate := range map[string]func(*Claim){ "kind": func(c *Claim) { c.Op.Kind = KindReverting }, "phase": func(c *Claim) { c.Op.Phase = "unauthorized" }, "executor": func(c *Claim) { c.Op.Exec = "other" }, "stamp": func(c *Claim) { c.Holder.Stamp++ }, "handover": func(c *Claim) { delete(c.Op.Journal, "recovery.loss") }, "intent": func(c *Claim) { delete(c.Op.Journal, "loss."+record.ID) }, "operation": func(c *Claim) { c.Op.ID = "other" }, "incarnation": func(c *Claim) { c.Holder.Incarnation = "other" }, "loss ref": func(c *Claim) { c.LossRefs = nil }, } { t.Run(name, func(t *testing.T) { candidate := handover candidate.Claim, err = cloneClaim(handover.Claim) if err != nil { t.Fatal(err) } mutate(candidate.Claim) if _, err := AdmitWriterWithLoss(ctx, s, pub, candidate, in, record.ID, validated, owner); err == nil { t.Fatal("admitted with changed binding") } }) } stale := in stale.ObservedPhase = "probe" if _, err := AdmitWriterWithLoss(ctx, s, pub, handover, stale, record.ID, validated, owner); !errors.Is(err, ErrWrongState) { t.Fatalf("stale admission phase = %v", err) } fresh, err := Read(ctx, s, obs.Host) if err != nil || fresh.ETag != handover.ETag { t.Fatalf("refusal changed claim: %v", err) } admitted, err := AdmitWriterWithLoss(ctx, s, pub, handover, in, record.ID, validated, owner) if err != nil { t.Fatal(err) } last := admitted.Claim.Holder.Admissions[len(admitted.Claim.Holder.Admissions)-1] if last.LossID != record.ID || last.WritesLost == nil || *last.WritesLost != (WritesLost{Box: "target", Generation: "g-target", Session: 4, RemoteMaxTXID: 12}) || !reflect.DeepEqual(last.AdditionalWritesLost, []WritesLost{source}) || admitted.Claim.Holder.WriterBox != "recovery" || admitted.Claim.Op.ID != record.OpID || admitted.Claim.Op.Kind != KindRestoring || admitted.Claim.Op.Phase != "restore" || admitted.Claim.Holder.Incarnation != record.HolderIncarnation || admitted.Claim.Op.Journal["recovery.loss"] != record.ID { t.Fatalf("recovery admission = %+v", admitted.Claim) } frozen, err := ReadLossRecord(ctx, s, pub, record.ID) if err != nil || !reflect.DeepEqual(frozen.Record, committed) { t.Fatalf("recovery rewrote frozen declaration: %+v, %v", frozen.Record, err) } } func TestLegacyLossWithoutDeclarationRetainsEvidence(t *testing.T) { a := AdmissionRecord{Region: "ie", Session: 3, WritesLost: &WritesLost{Box: "legacy", Session: 0}} if !validWriteLosses(a.WritesLost, nil) { t.Fatal("legacy zero-session loss no longer parses") } c := &Claim{Holder: &Holder{Admissions: []AdmissionRecord{a}}} if err := RequireLossEvidenceArchived(c); !errors.Is(err, ErrLossEvidencePending) { t.Fatalf("unarchived legacy loss discarded: %v", err) } }