package hostclaim import ( "bytes" "context" "crypto/ed25519" "encoding/json" "errors" "fmt" "reflect" "testing" "time" "github.com/carlosframework/platform/internal/release" ) type testLossOwner struct { err error mutate bool } func (o testLossOwner) ValidateDeclareLost(c Claim, in LostDeclaration) error { if o.mutate { c.Holder.Stamp++ in.Consequences.RecoveryIntent[2] = 'X' } return o.err } func TestDeclareLostExactScopeAndReadmitRevision(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) obs := servingClaim(t, ctx, s) obs, err := Update(ctx, s, obs, func(c *Claim) error { c.Op = Op{Kind: KindMoving, ID: "op-move", Phase: "precommit", Exec: "box-a", Participants: []string{"box-a", "box-b"}} c.Obligations = []Obligation{ {ID: "local", Op: "discover", Kind: "park-local", Box: "box-a", Retirement: "ret-local", Owner: "box:box-a"}, {ID: "remote", Op: "op-move", Kind: "park-replica", Region: "ie", Retirement: "ret-remote", Owner: "box:box-a"}, } c.Touched.Complete, c.Touched.DiscoveryOp = false, "discover" return nil }) if err != nil { t.Fatal(err) } stamp, session := obs.Claim.Holder.Stamp, obs.Claim.Holder.Session in := LostDeclaration{ LossID: "loss_1", ExpectedClaimGen: obs.Claim.Gen, ExpectedClaimETag: obs.ETag, HolderIncarnation: obs.Claim.Holder.Incarnation, HolderStamp: stamp, HolderSession: session, OpID: "op-move", OpKind: KindMoving, OpPhase: "precommit", Box: "box-a", Enrollment: "enroll-a", QuarantineRevision: 7, FencedAt: time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC), Consequences: LossConsequences{ RevokeExecution: true, RemoveParticipation: true, DischargeLocal: []ObligationRef{{ID: "local", Op: "discover", Retirement: "ret-local"}}, ReassignRegional: []ObligationRef{{ID: "remote", Op: "op-move", Retirement: "ret-remote"}}, DestroyedStorage: []string{"discover/local/ret-local"}, RecoveryIntent: []byte(`{"kind":"lost-target","destination":"owner-decides"}`), }, } pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } record := validLossRecord(t) record.ID, record.Host, record.HolderIncarnation = in.LossID, obs.Host, in.HolderIncarnation record.HolderStamp, record.HolderSession = stamp, session record.ExpectedClaimGen, record.ExpectedClaimETag = in.ExpectedClaimGen, in.ExpectedClaimETag record.OpID, record.OpKind, record.OpPhase = in.OpID, in.OpKind, in.OpPhase record.Box, record.Enrollment, record.QuarantineRevision, record.FencedAt = in.Box, in.Enrollment, in.QuarantineRevision, in.FencedAt record.ConfirmedConsequences = in.Consequences record.OperatorAttestation.DestroyedStorage = append([]string(nil), in.Consequences.DestroyedStorage...) if _, err := CreateLossRecord(ctx, s, release.RootSigner(priv), record); err != nil { t.Fatal(err) } got, err := DeclareLost(ctx, s, pub, obs, in.LossID, testLossOwner{mutate: true}) if err != nil { t.Fatal(err) } if got.Claim.Op.Exec != "none" || len(got.Claim.Op.Participants) != 1 || got.Claim.Op.Participants[0] != "box-b" { t.Fatalf("authority = exec %q participants %v", got.Claim.Op.Exec, got.Claim.Op.Participants) } if got.Claim.Holder.Stamp != stamp || got.Claim.Holder.Session != session { t.Fatalf("installation authority changed: stamp/session %d/%d", got.Claim.Holder.Stamp, got.Claim.Holder.Session) } if len(got.Claim.Obligations) != 1 || got.Claim.Obligations[0].ID != "remote" || got.Claim.Obligations[0].Owner != "region:ie" || got.Claim.Obligations[0].Retirement != "ret-remote" || got.Claim.Obligations[0].Op != "op-move" { t.Fatalf("obligations = %+v", got.Claim.Obligations) } if got.Claim.Touched.DiscoveryOp != "" || !got.Claim.Touched.Complete { t.Fatalf("discovery completion stranded: %+v", got.Claim.Touched) } if got.Claim.LostRevisions["box-a"] != 7 || !stringPresent(got.Claim.Lost, "box-a") || !lossRefPresent(got.Claim, "loss_1") { t.Fatalf("loss state = lost %v revisions %v refs %v", got.Claim.Lost, got.Claim.LostRevisions, got.Claim.LossRefs) } if got.Claim.Op.Journal["loss.loss_1"] != `{"destination":"owner-decides","kind":"lost-target"}` { t.Fatalf("recovery intent = %q", got.Claim.Op.Journal["loss.loss_1"]) } if retry, err := DeclareLost(ctx, s, pub, got, in.LossID, testLossOwner{}); err != nil || retry.ETag != got.ETag { t.Fatalf("lost-response retry = %+v, %v", retry, err) } if _, err := Update(ctx, s, got, Readmit("box-a", 6)); !errors.Is(err, ErrWrongState) { t.Fatalf("old readmit revision = %v", err) } readmitted, err := Update(ctx, s, got, Readmit("box-a", 7)) if err != nil { t.Fatal(err) } if stringPresent(readmitted.Claim.Lost, "box-a") || readmitted.Claim.LostRevisions["box-a"] != 0 || !lossRefPresent(readmitted.Claim, "loss_1") { t.Fatalf("readmission state = lost %v revisions %v refs %v", readmitted.Claim.Lost, readmitted.Claim.LostRevisions, readmitted.Claim.LossRefs) } } func TestDeclareLostFailsClosedWithoutOperationOwnerOrExactDiskProof(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) obs := servingClaim(t, ctx, s) obs, err := Update(ctx, s, obs, func(c *Claim) error { c.Op = Op{Kind: KindDeleting, ID: "delete", Phase: "parking", Exec: "box-a", Participants: []string{"box-a"}} c.Obligations = []Obligation{{ID: "local", Op: "delete", Kind: "park-local", Box: "box-a", Retirement: "ret", Owner: "box:box-a"}} return nil }) if err != nil { t.Fatal(err) } in := LostDeclaration{LossID: "loss", ExpectedClaimGen: obs.Claim.Gen, ExpectedClaimETag: obs.ETag, HolderIncarnation: obs.Claim.Holder.Incarnation, HolderStamp: obs.Claim.Holder.Stamp, HolderSession: obs.Claim.Holder.Session, OpID: "delete", OpKind: KindDeleting, OpPhase: "parking", Box: "box-a", Enrollment: "e", QuarantineRevision: 1, FencedAt: time.Now(), Consequences: LossConsequences{RevokeExecution: true, RemoveParticipation: true, DischargeLocal: []ObligationRef{{ID: "local", Op: "delete", Retirement: "ret"}}, RecoveryIntent: []byte(`{"kind":"local-only"}`)}} pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } record := validLossRecord(t) record.ID, record.Host, record.HolderIncarnation = in.LossID, obs.Host, in.HolderIncarnation record.HolderStamp, record.HolderSession = in.HolderStamp, in.HolderSession record.ExpectedClaimGen, record.ExpectedClaimETag = in.ExpectedClaimGen, in.ExpectedClaimETag record.OpID, record.OpKind, record.OpPhase = in.OpID, in.OpKind, in.OpPhase record.Box, record.Enrollment, record.QuarantineRevision, record.FencedAt = in.Box, in.Enrollment, in.QuarantineRevision, in.FencedAt record.ConfirmedConsequences = in.Consequences record.OperatorAttestation.DestroyedStorage = append([]string(nil), in.Consequences.DestroyedStorage...) if _, err := CreateLossRecord(ctx, s, release.RootSigner(priv), record); err != nil { t.Fatal(err) } if _, err := DeclareLost(ctx, s, pub, obs, in.LossID, nil); !errors.Is(err, ErrLossOwnerRequired) { t.Fatalf("nil owner = %v", err) } if _, err := DeclareLost(ctx, s, pub, obs, in.LossID, testLossOwner{}); !errors.Is(err, ErrWrongState) { t.Fatalf("missing disk assertion = %v", err) } denied := errors.New("move record already committed") if _, err := DeclareLost(ctx, s, pub, obs, in.LossID, testLossOwner{err: denied}); !errors.Is(err, denied) { t.Fatalf("owner denial = %v", err) } fresh, err := Read(ctx, s, testHost) if err != nil { t.Fatal(err) } if fresh.Claim.Gen != obs.Claim.Gen || len(fresh.Claim.Lost) != 0 || len(fresh.Claim.LossRefs) != 0 { t.Fatalf("refusal mutated claim: %+v", fresh.Claim) } } func lossForObservation(t *testing.T, obs Observed, id string, revision uint64, box string, effects LossConsequences) LossRecord { t.Helper() r := validLossRecord(t) r.ID, r.Host = id, obs.Host r.HolderIncarnation, r.HolderStamp, r.HolderSession = obs.Claim.Holder.Incarnation, obs.Claim.Holder.Stamp, obs.Claim.Holder.Session r.ExpectedClaimGen, r.ExpectedClaimETag = obs.Claim.Gen, obs.ETag r.OpID, r.OpKind, r.OpPhase = obs.Claim.Op.ID, obs.Claim.Op.Kind, obs.Claim.Op.Phase r.Box, r.QuarantineRevision = box, revision r.ConfirmedConsequences = effects r.WriteSources = nil r.OperatorAttestation.DestroyedStorage = append([]string(nil), effects.DestroyedStorage...) return r } func TestDeclareLostPreservesLargeRecoveryIntentIntegers(t *testing.T) { ctx := t.Context() s := release.NewDirStore(t.TempDir()) pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } obs := servingClaim(t, ctx, s) obs, err = Update(ctx, s, obs, func(c *Claim) error { c.Op = Op{Kind: KindDeleting, ID: "delete", Phase: "parking", Exec: "box-a", Participants: []string{"box-a"}} return nil }) if err != nil { t.Fatal(err) } intent := `{"kind":"recover","nested":{"txid":9007199254740993},"positions":[18446744073709551615,-9007199254740993]}` record := lossForObservation(t, obs, "large_integers", 1, "box-a", LossConsequences{ RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: json.RawMessage(intent), }) if _, err := CreateLossRecord(ctx, s, release.RootSigner(priv), record); err != nil { t.Fatal(err) } if _, err := DeclareLost(ctx, s, pub, obs, record.ID, testLossOwner{}); err != nil { t.Fatal(err) } fresh, err := Read(ctx, s, obs.Host) if err != nil { t.Fatal(err) } if got := fresh.Claim.Op.Journal["loss."+record.ID]; got != intent { t.Fatalf("persisted recovery intent = %s, want %s", got, intent) } } func TestDeclareLostNewerRevisionAppliesSecondConsequences(t *testing.T) { ctx := t.Context() s := release.NewDirStore(t.TempDir()) pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } obs := servingClaim(t, ctx, s) obs, err = Update(ctx, s, obs, func(c *Claim) error { c.Op = Op{Kind: KindDeleting, ID: "delete", Phase: "parking", Exec: "box-a", Participants: []string{"box-a"}} c.Obligations = []Obligation{{ID: "local", Op: "delete", Kind: "park-local", Box: "box-a", Retirement: "ret", Owner: "box:box-a"}} return nil }) if err != nil { t.Fatal(err) } first := lossForObservation(t, obs, "stopped", 9, "box-a", LossConsequences{ RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: []byte(`{"kind":"wait-for-storage"}`), }) if _, err := CreateLossRecord(ctx, s, release.RootSigner(priv), first); err != nil { t.Fatal(err) } stopped, err := DeclareLost(ctx, s, pub, obs, first.ID, testLossOwner{}) if err != nil { t.Fatal(err) } if len(stopped.Claim.Obligations) != 1 { t.Fatal("compute stop discharged local storage") } stopped, err = Update(ctx, s, stopped, func(c *Claim) error { c.Op.ID = "delete-second"; c.Op.Journal = nil; return nil }) if err != nil { t.Fatal(err) } second := lossForObservation(t, stopped, "destroyed", 10, "box-a", LossConsequences{ DischargeLocal: []ObligationRef{{ID: "local", Op: "delete", Retirement: "ret"}}, DestroyedStorage: []string{"delete/local/ret"}, RecoveryIntent: []byte(`{"kind":"destroyed-storage"}`), }) // A new revision does not excuse stale claim generation or ETag bindings. stale := second stale.ID, stale.ExpectedClaimGen, stale.ExpectedClaimETag = "stale", obs.Claim.Gen, obs.ETag if _, err := CreateLossRecord(ctx, s, release.RootSigner(priv), stale); err != nil { t.Fatal(err) } if _, err := DeclareLost(ctx, s, pub, stopped, stale.ID, testLossOwner{}); !errors.Is(err, ErrConflict) { t.Fatalf("stale second binding = %v", err) } if _, err := CreateLossRecord(ctx, s, release.RootSigner(priv), second); err != nil { t.Fatal(err) } destroyed, err := DeclareLost(ctx, s, pub, stopped, second.ID, testLossOwner{}) if err != nil { t.Fatal(err) } if len(destroyed.Claim.Obligations) != 0 || destroyed.Claim.LostRevisions["box-a"] != 10 || !reflect.DeepEqual(destroyed.Claim.LossRefs, []string{"stopped", "destroyed"}) { t.Fatalf("second declaration not applied: %+v", destroyed.Claim) } for _, rev := range []uint64{8, 9, 10} { r := lossForObservation(t, destroyed, fmt.Sprintf("older_%d", rev), rev, "box-a", LossConsequences{RecoveryIntent: []byte(`{}`)}) if _, err := CreateLossRecord(ctx, s, release.RootSigner(priv), r); err != nil { t.Fatal(err) } if _, err := DeclareLost(ctx, s, pub, destroyed, r.ID, testLossOwner{}); !errors.Is(err, ErrWrongState) { t.Fatalf("revision %d accepted: %v", rev, err) } } if _, err := Update(ctx, s, destroyed, Readmit("box-a", 9)); !errors.Is(err, ErrWrongState) { t.Fatalf("old readmission cleared newer quarantine: %v", err) } for _, id := range []string{first.ID, second.ID} { retry, err := DeclareLost(ctx, s, pub, destroyed, id, testLossOwner{}) if err != nil || retry.ETag != destroyed.ETag { t.Fatalf("replay %s = %v", id, err) } } unchanged, err := ReadLossRecord(ctx, s, pub, first.ID) if err != nil || !reflect.DeepEqual(unchanged.Record, first) { t.Fatal("first declaration was rewritten", err) } fresh, err := Read(ctx, s, obs.Host) if err != nil || fresh.ETag != destroyed.ETag { t.Fatal("refusal or replay mutated claim", err) } readmitted, err := Update(ctx, s, fresh, Readmit("box-a", 10)) if err != nil || len(readmitted.Claim.Lost) != 0 || len(readmitted.Claim.LossRefs) != 2 { t.Fatal("matching readmission", err) } } func TestServingDestroyedLocalOnlyPreservesAuthorityAndOtherChores(t *testing.T) { for _, otherChores := range []bool{false, true} { t.Run(fmt.Sprint(otherChores), func(t *testing.T) { ctx := t.Context() s := release.NewDirStore(t.TempDir()) pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } obs, err := Read(ctx, s, testHost) if err != nil { t.Fatal(err) } var regions []string boxes := []string{"box-a"} if otherChores { regions = []string{"uk"} boxes = append(boxes, "other-box") } obs, err = Update(ctx, s, obs, Adopt(newHolder(), "serving-box", regions, boxes, "discovery", counter())) if err != nil { t.Fatal(err) } if obs.Claim.Op.Kind != KindServing || obs.Claim.Op.ID != "" { t.Fatal("fixture is not an adopted serving claim") } var ref ObligationRef for _, o := range obs.Claim.Obligations { if o.Kind == "park-local" && o.Box == "box-a" { ref = ObligationRef{ID: o.ID, Op: o.Op, Retirement: o.Retirement} } } effects := LossConsequences{DischargeLocal: []ObligationRef{ref}, DestroyedStorage: []string{ref.Op + "/" + ref.ID + "/" + ref.Retirement}, RecoveryIntent: []byte(`{"kind":"destroyed-local-only"}`)} r := lossForObservation(t, obs, "local_only", 11, "box-a", effects) if _, err := CreateLossRecord(ctx, s, release.RootSigner(priv), r); err != nil { t.Fatal(err) } got, err := DeclareLost(ctx, s, pub, obs, r.ID, testLossOwner{mutate: true}) if err != nil { t.Fatal(err) } want, err := cloneClaim(obs.Claim) if err != nil { t.Fatal(err) } remaining := make([]Obligation, 0, len(want.Obligations)-1) for _, o := range want.Obligations { if o.ID != ref.ID { remaining = append(remaining, o) } } want.Obligations = remaining if !otherChores { want.Touched.Complete, want.Touched.DiscoveryOp = true, "" } want.Gen = got.Claim.Gen want.Lost, want.LostRevisions, want.LossRefs = []string{"box-a"}, map[string]uint64{"box-a": 11}, []string{r.ID} // Compare serialized claims so nil/empty omitted arrays are equivalent. wantBytes, _ := json.Marshal(want) gotBytes, _ := json.Marshal(got.Claim) if !bytes.Equal(wantBytes, gotBytes) { t.Fatalf("local-only changed unrelated state:\nwant %s\ngot %s", wantBytes, gotBytes) } if retry, err := DeclareLost(ctx, s, pub, got, r.ID, testLossOwner{}); err != nil || retry.ETag != got.ETag { t.Fatal("local-only replay", err) } }) } } func TestServingDestroyedLocalOnlyRefusesWideningAndStaleBindings(t *testing.T) { ctx := t.Context() s := release.NewDirStore(t.TempDir()) pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } obs, err := Read(ctx, s, testHost) if err != nil { t.Fatal(err) } obs, err = Update(ctx, s, obs, Adopt(newHolder(), "serving-box", nil, []string{"box-a"}, "discovery", counter())) if err != nil { t.Fatal(err) } o := obs.Claim.Obligations[0] ref := ObligationRef{ID: o.ID, Op: o.Op, Retirement: o.Retirement} base := lossForObservation(t, obs, "local", 1, "box-a", LossConsequences{DischargeLocal: []ObligationRef{ref}, DestroyedStorage: []string{ref.Op + "/" + ref.ID + "/" + ref.Retirement}, RecoveryIntent: []byte(`{"kind":"destroyed-local-only"}`)}) for name, mutate := range map[string]func(*LossRecord){ "execution": func(r *LossRecord) { r.ConfirmedConsequences.RevokeExecution = true }, "participation": func(r *LossRecord) { r.ConfirmedConsequences.RemoveParticipation = true }, "regional": func(r *LossRecord) { r.ConfirmedConsequences.ReassignRegional = []ObligationRef{ref} }, "writes": func(r *LossRecord) { r.ConfirmedConsequences.WritesPotentiallyLost = true }, "timers": func(r *LossRecord) { r.ConfirmedConsequences.TimersPotentiallyLost = true }, "recovery": func(r *LossRecord) { r.ConfirmedConsequences.RecoveryIntent = []byte(`{"kind":"destroyed-local-only","destination":"other"}`) }, "empty": func(r *LossRecord) { r.ConfirmedConsequences.DischargeLocal = nil }, "missing-proof": func(r *LossRecord) { r.ConfirmedConsequences.DestroyedStorage = nil r.OperatorAttestation.DestroyedStorage = nil }, "op": func(r *LossRecord) { r.OpID = "discovery" }, "phase": func(r *LossRecord) { r.OpPhase = "recover" }, "non-serving-empty-op": func(r *LossRecord) { r.OpKind = KindDeleting }, } { t.Run(name, func(t *testing.T) { r := base r.ID = name mutate(&r) if _, err := CreateLossRecord(ctx, s, release.RootSigner(priv), r); !errors.Is(err, ErrLossRecord) { t.Fatalf("widened serving declaration = %v", err) } }) } for name, mutate := range map[string]func(*LossRecord){ "claim-gen": func(r *LossRecord) { r.ExpectedClaimGen++ }, "claim-etag": func(r *LossRecord) { r.ExpectedClaimETag = "stale" }, "stamp": func(r *LossRecord) { r.HolderStamp++ }, "session": func(r *LossRecord) { r.HolderSession++ }, "box": func(r *LossRecord) { r.Box = "other-box" }, "original-op": func(r *LossRecord) { r.ConfirmedConsequences.DischargeLocal[0].Op = "other-op" }, "retirement": func(r *LossRecord) { r.ConfirmedConsequences.DischargeLocal[0].Retirement = "other-retirement" }, } { t.Run(name, func(t *testing.T) { r := base r.ID = name r.ConfirmedConsequences.DischargeLocal = append([]ObligationRef(nil), base.ConfirmedConsequences.DischargeLocal...) mutate(&r) ref := r.ConfirmedConsequences.DischargeLocal[0] r.ConfirmedConsequences.DestroyedStorage = []string{ref.Op + "/" + ref.ID + "/" + ref.Retirement} r.OperatorAttestation.DestroyedStorage = r.ConfirmedConsequences.DestroyedStorage if _, err := CreateLossRecord(ctx, s, release.RootSigner(priv), r); err != nil { t.Fatal(err) } if _, err := DeclareLost(ctx, s, pub, obs, r.ID, testLossOwner{}); err == nil { t.Fatal("stale or wrong scope was accepted") } }) } fresh, err := Read(ctx, s, obs.Host) if err != nil || fresh.ETag != obs.ETag { t.Fatal("refusal changed serving claim", err) } }