package hostclaim import ( "encoding/json" "errors" "strconv" "strings" "testing" "time" ) // The console's release of a declared-lost move target (moves spec ยง7.10, // J16): after the declaration's claim CAS took the target out of execution // and participation, one claim write rolls a landing back, steps the // rollback to released, queues the target region's parks and covers its // predecessors, as TargetReleased does for a target that released itself. const ( lostOp = "mv_abcdefghijklmnopqrstuvwxyz" lostRB = "rb_abcdefghijklmnopqrstuvwxyz" lostID = "loss_1" lostHost = "shop.example" ) func lostTargetClaim(t *testing.T, kind, phase string) *Claim { t.Helper() at := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) inc, err := Incarnation("bab", "jam", lostHost, at) if err != nil { t.Fatal(err) } c := &Claim{V: 2, Host: lostHost, Gen: 30, Holder: &Holder{Family: "instance", Key: "apps/bab/jam/instances/" + lostHost + ".json", Account: "bab", App: "jam", CreatedAt: at, Incarnation: inc, Location: "instances/" + lostHost, Generation: "mv", Stamp: 21, Session: 20, WriterBox: "uk-box", Admissions: []AdmissionRecord{ {Session: 15, Box: "ie-box", Generation: "src", Predecessor: "older", Region: "ie", PredecessorRegion: "ie", Mode: "cold"}, {Session: 20, Box: "uk-box", Generation: "mv", Predecessor: "src", Region: "uk", PredecessorRegion: "ie", Mode: "cold", Coverage: "sealed"}, {Session: 25, Box: "uk-box", Generation: "mv2", Predecessor: "mv", Region: "uk", PredecessorRegion: "uk", Mode: "cold"}, }}, Op: Op{Kind: kind, ID: lostOp, Phase: phase, Exec: "none", Participants: []string{"ie-box"}, Intent: map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box"}, Journal: map[string]string{JournalSourceStamp: "15", JournalSrcGen: "src", JournalCutSession: "15", "loss." + lostID: `{"kind":"move-target-lost"}`}}, Lost: []string{"uk-box"}, LostRevisions: map[string]uint64{"uk-box": 2}, LossRefs: []string{lostID}, } if kind == KindRestoring { c.Op.Intent["rollback"] = lostRB c.Op.Journal[JournalTargetStamp] = "21" } if err := c.Validate(); err != nil { t.Fatal(err) } return c } func lostRelease(c *Claim, returnLane bool) LostTargetRelease { at := time.Date(2026, 10, 1, 1, 0, 0, 0, time.UTC) n := 0 r := LostTargetRelease{OpID: lostOp, RollbackID: lostRB, LossID: lostID, Target: "uk-box", TargetRegion: "uk", SourceRegion: "ie", Location: ParkScope{Location: "instances/" + lostHost, Incarnation: c.Holder.Incarnation, EvidenceAt: at, RequestLimit: 2 * time.Minute}, Lane: ParkScope{Location: "moves/" + lostHost + "/" + lostOp, Incarnation: c.Holder.Incarnation, EvidenceAt: at, RequestLimit: 2 * time.Minute}, NewID: func() string { n++; return "id-" + strconv.Itoa(n) }} if returnLane { r.ReturnLane = &ParkScope{Location: "moves/" + lostHost + "/" + lostRB, Incarnation: c.Holder.Incarnation, EvidenceAt: at, RequestLimit: 2 * time.Minute} } return r } func claimJSON(t *testing.T, c *Claim) string { t.Helper() b, err := json.Marshal(c) if err != nil { t.Fatal(err) } return string(b) } func parksOf(c *Claim) map[string]Obligation { out := map[string]Obligation{} for _, o := range c.Obligations { key := o.Kind + "|" + o.Region + "|" + o.Box if o.Park != nil { key += "|" + o.Park.Location } out[key] = o } return out } func TestReleaseLostTargetFromLanding(t *testing.T) { c := lostTargetClaim(t, KindLanding, "probing") if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { t.Fatal(err) } if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseReleased || c.Op.Intent["rollback"] != lostRB || c.Op.Exec != "none" { t.Fatalf("op = %+v", c.Op) } if c.Op.Journal[JournalTargetStamp] != "21" || c.Op.Journal[journalLostTargetReleased] != lostID { t.Fatalf("journal = %v", c.Op.Journal) } parks := parksOf(c) loc, ok := parks["park-replica|uk||instances/"+lostHost] if !ok || loc.Owner != "region:uk" { t.Fatalf("location park = %+v (%v)", loc, parks) } if _, ok := parks["park-replica|uk||moves/"+lostHost+"/"+lostOp]; !ok { t.Fatalf("target lane park missing: %v", parks) } if len(parks) != 2 { t.Fatalf("parks = %v", parks) } for _, o := range c.Obligations { if o.Kind == "park-local" || o.Box != "" { t.Fatalf("a target with no installation in inventory acquired local cleanup: %+v", o) } } // The target's own predecessor is covered by the park; the source's // return admission later is born covered the same way. for _, a := range c.Holder.Admissions { if a.PredecessorRegion == "uk" && !a.Covered { t.Fatalf("uk predecessor not covered: %+v", a) } if a.PredecessorRegion == "ie" && a.Covered && a.Session != 15 { t.Fatalf("ie predecessor covered: %+v", a) } } if err := c.Validate(); err != nil { t.Fatal(err) } // The source's claimed CAS takes it from here: the declaration is the // target's authority (targetGaveUp). if err := ReturnToSource(lostOp, "ie-box", true, ReturnFence{})(c); err != nil { t.Fatalf("return to source after the release: %v", err) } } func TestReleaseLostTargetIsIdempotent(t *testing.T) { c := lostTargetClaim(t, KindLanding, "probing") r := lostRelease(c, false) if err := ReleaseLostTarget(r)(c); err != nil { t.Fatal(err) } once := claimJSON(t, c) if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { t.Fatal(err) } if claimJSON(t, c) != once { t.Fatal("a replay changed the claim") } // A replay after the source has claimed, or after a chore was acked, // is still a no-op. if err := ReturnToSource(lostOp, "ie-box", true, ReturnFence{})(c); err != nil { t.Fatal(err) } c.Obligations = c.Obligations[:1] claimed := claimJSON(t, c) if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { t.Fatal(err) } if claimJSON(t, c) != claimed { t.Fatal("a replay after the source claimed changed the claim") } } func TestReleaseLostTargetFromReleasing(t *testing.T) { c := lostTargetClaim(t, KindRestoring, PhaseReleasing) if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { t.Fatal(err) } if c.Op.Phase != PhaseReleased || len(parksOf(c)) != 2 { t.Fatalf("op %+v parks %v", c.Op, parksOf(c)) } } // A target that released dirty and then died before its return: its own // release queued the location park already; the declaration adds the // partial return lane's park in the source region and the move's lane. func TestReleaseLostTargetAfterDirtyRelease(t *testing.T) { c := lostTargetClaim(t, KindRestoring, PhaseReleased) c.Op.Exec = "" c.Op.Journal[JournalEvidenceTarget] = "moves/x/stopped-target.json" at := time.Date(2026, 10, 1, 0, 30, 0, 0, time.UTC) c.Obligations = []Obligation{{ID: "loc", Op: lostOp, Kind: "park-replica", Region: "uk", Retirement: "r-loc", Owner: "region:uk", Park: &ParkScope{Location: "instances/" + lostHost, Incarnation: c.Holder.Incarnation, EvidenceAt: at, RequestLimit: time.Minute}}} if err := ReleaseLostTarget(lostRelease(c, true))(c); err != nil { t.Fatal(err) } parks := parksOf(c) if len(parks) != 3 || parks["park-replica|uk||instances/"+lostHost].ID != "loc" { t.Fatalf("parks = %v", parks) } lane, ok := parks["park-replica|ie||moves/"+lostHost+"/"+lostRB] if !ok || lane.Owner != "region:ie" { t.Fatalf("return lane park = %+v (%v)", lane, parks) } if c.Op.Journal[journalReturnLane] != "moves/"+lostHost+"/"+lostRB { t.Fatalf("journal = %v", c.Op.Journal) } // The return lane's own probed-time queue is then a no-op. c.Op.Phase = PhaseProbed before := len(c.Obligations) if err := QueueReturnLanePark(lostOp, "ie", *lostRelease(c, true).ReturnLane, func() string { return "x" })(c); err != nil { t.Fatal(err) } if len(c.Obligations) != before { t.Fatal("return lane queued twice") } } func TestReleaseLostTargetRefusals(t *testing.T) { for name, tc := range map[string]struct { mutate func(*Claim, *LostTargetRelease) refusal error }{ "no loss ref": {func(c *Claim, _ *LostTargetRelease) { c.LossRefs = nil }, ErrWrongState}, "no loss journal": {func(c *Claim, _ *LostTargetRelease) { delete(c.Op.Journal, "loss."+lostID) }, ErrWrongState}, "target participates": {func(c *Claim, _ *LostTargetRelease) { c.Op.Participants = []string{"ie-box", "uk-box"} }, ErrWrongState}, "target executes": {func(c *Claim, _ *LostTargetRelease) { c.Op.Exec = "uk-box" }, ErrWrongState}, "target not lost": {func(c *Claim, _ *LostTargetRelease) { c.Lost, c.LostRevisions = nil, nil }, ErrWrongState}, "other target": {func(_ *Claim, r *LostTargetRelease) { r.Target = "ie-box" }, ErrWrongState}, "other region": {func(_ *Claim, r *LostTargetRelease) { r.TargetRegion = "de" }, ErrWrongState}, "source region is to": {func(_ *Claim, r *LostTargetRelease) { r.SourceRegion = "uk" }, ErrMalformed}, "other op": {func(_ *Claim, r *LostTargetRelease) { r.OpID = "mv_bbbbbbbbbbbbbbbbbbbbbbbbbb" }, ErrStaleOp}, "committed": {func(c *Claim, _ *LostTargetRelease) { c.Op.Phase = PhaseCommitted }, ErrWrongState}, "bad rollback": {func(_ *Claim, r *LostTargetRelease) { r.RollbackID = "rb_x" }, ErrMalformed}, "wrong lane": {func(_ *Claim, r *LostTargetRelease) { r.Lane.Location = "moves/" + lostHost }, ErrMalformed}, "wrong return lane": {func(c *Claim, r *LostTargetRelease) { s := r.Lane; r.ReturnLane = &s }, ErrMalformed}, "wrong location": {func(_ *Claim, r *LostTargetRelease) { r.Location.Location = "instances/" + lostHost + "/x" }, ErrMalformed}, "no ids": {func(_ *Claim, r *LostTargetRelease) { r.NewID = nil }, ErrMalformed}, "moving (before flip)": {func(c *Claim, _ *LostTargetRelease) { c.Op.Kind, c.Op.Phase = KindMoving, PhaseSealed }, ErrWrongState}, } { t.Run(name, func(t *testing.T) { c := lostTargetClaim(t, KindLanding, "probing") r := lostRelease(c, false) tc.mutate(c, &r) before := claimJSON(t, c) err := ReleaseLostTarget(r)(c) if !errors.Is(err, tc.refusal) { t.Fatalf("err = %v, want %v", err, tc.refusal) } if claimJSON(t, c) != before { t.Fatal("a refused release changed the claim") } }) } // Another rollback already on the claim. c := lostTargetClaim(t, KindRestoring, PhaseReleasing) r := lostRelease(c, false) r.RollbackID = "rb_bbbbbbbbbbbbbbbbbbbbbbbbbb" if err := ReleaseLostTarget(r)(c); !errors.Is(err, ErrWrongState) { t.Fatalf("other rollback = %v", err) } if !strings.HasPrefix(lostRB, "rb_") { t.Fatal("fixture") } } // D1: the lanes are checked against the holder's incarnation, as every // other writer of a move lane does, and a trailing slash is normalized. func TestReleaseLostTargetLaneIncarnation(t *testing.T) { other, err := Incarnation("bab", "jam", lostHost, time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)) if err != nil { t.Fatal(err) } for name, mutate := range map[string]func(*Claim, *LostTargetRelease){ "lane of another incarnation": func(_ *Claim, r *LostTargetRelease) { r.Lane.Incarnation = other }, "return lane of another incarnation": func(c *Claim, r *LostTargetRelease) { s := lostRelease(c, true).ReturnLane s.Incarnation = other r.ReturnLane = s }, } { t.Run(name, func(t *testing.T) { c := lostTargetClaim(t, KindLanding, "probing") r := lostRelease(c, false) mutate(c, &r) before := claimJSON(t, c) if err := ReleaseLostTarget(r)(c); !errors.Is(err, ErrMalformed) { t.Fatalf("err = %v, want %v", err, ErrMalformed) } if claimJSON(t, c) != before { t.Fatal("a refused release changed the claim") } }) } c := lostTargetClaim(t, KindLanding, "probing") r := lostRelease(c, true) r.Lane.Location += "/" r.ReturnLane.Location += "/" if err := ReleaseLostTarget(r)(c); err != nil { t.Fatal(err) } parks := parksOf(c) if _, ok := parks["park-replica|uk||moves/"+lostHost+"/"+lostOp]; !ok { t.Fatalf("normalized lane park missing: %v", parks) } if _, ok := parks["park-replica|ie||moves/"+lostHost+"/"+lostRB]; !ok || c.Op.Journal[journalReturnLane] != "moves/"+lostHost+"/"+lostRB { t.Fatalf("normalized return lane park missing: %v %v", parks, c.Op.Journal) } } // D2: a target that released dirty and then died owes its return lane's // park; a release without it is refused and leaves the claim as found. func TestReleaseLostTargetReleasedNeedsReturnLane(t *testing.T) { c := lostTargetClaim(t, KindRestoring, PhaseReleased) c.Op.Exec = "" c.Op.Journal[JournalEvidenceTarget] = "moves/x/stopped-target.json" before := claimJSON(t, c) if err := ReleaseLostTarget(lostRelease(c, false))(c); !errors.Is(err, ErrMalformed) { t.Fatalf("err = %v, want %v", err, ErrMalformed) } if claimJSON(t, c) != before { t.Fatal("a refused release changed the claim") } } // D3: Readmit removes the box from lost without looking at the op, so a // retried release whose CAS landed is still a no-op afterwards. func TestReleaseLostTargetReplayAfterReadmit(t *testing.T) { c := lostTargetClaim(t, KindLanding, "probing") if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { t.Fatal(err) } if err := Readmit("uk-box", 2)(c); err != nil { t.Fatal(err) } once := claimJSON(t, c) if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { t.Fatalf("replay after readmit: %v", err) } if claimJSON(t, c) != once { t.Fatal("a replay changed the claim") } } func TestReleaseLostTargetRetainsInventoryBoundLocal(t *testing.T) { for _, kind := range []string{KindLanding, KindRestoring} { t.Run(kind, func(t *testing.T) { phase := PhaseReleasing if kind == KindLanding { phase = PhaseProbing } c := lostTargetClaim(t, kind, phase) c.Inventory = map[string]InventoryEntry{"uk-box": {Paths: map[string]uint64{"root": 21}}} if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { t.Fatal(err) } var local Obligation for _, o := range c.Obligations { if o.Kind == "park-local" && o.Box == "uk-box" { local = o } } if local.ID == "" || local.Op != lostOp || local.Retirement == "" { t.Fatal("target inventory's retirement obligation missing") } before := claimJSON(t, c) if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { t.Fatal(err) } if claimJSON(t, c) != before { t.Fatal("retry lost or replaced local chore") } if err := ReturnToSource(lostOp, "ie-box", true, ReturnFence{})(c); err != nil { t.Fatal("pending target cleanup blocked source recovery", err) } c.Op.Phase, c.Op.Exec = PhaseOpened, "ie-box" c.Obligations = []Obligation{local} if err := ReleaseMove(lostOp, "ie-box")(c); err == nil { t.Fatal("move released with target-local retirement pending") } if err := CompleteObligation(local.Op, local.ID)(c); err != nil { t.Fatal(err) } if err := ReleaseMove(lostOp, "ie-box")(c); err != nil { t.Fatal("retired local still blocks release", err) } }) } }