package hostops import ( "context" "crypto/ed25519" "crypto/rand" "encoding/json" "errors" "reflect" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfence" "github.com/carlosframework/platform/internal/hostresolve" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" "github.com/superfly/ltx" ) // J4's moves owner and the J17 declare-lost grant: a console-initiated // declaration that a move's target box is lost (moves spec ยง7.10), through // the generic hostresolve service with only the move kinds' owner // registered, writing only through the typed ControlWriter's private // scoped store. const ( lossOp = "mv_abcdefghijklmnopqrstuvwxyz" lossHost = "shop.example" lossBucket = "bucket" ) // lossFaultStore fails the first Put of one key, before or after it lands. type lossFaultStore struct { release.Store mu sync.Mutex key string failBefore bool failAfter bool puts []string } func (s *lossFaultStore) Put(ctx context.Context, key string, body []byte, match string) (string, error) { s.mu.Lock() s.puts = append(s.puts, key) before, after := s.key == key && s.failBefore, s.key == key && s.failAfter if s.key == key { s.failBefore, s.failAfter = false, false } s.mu.Unlock() if before { return "", errors.New("injected put failure") } tag, err := s.Store.Put(ctx, key, body, match) if err == nil && after { return "", errors.New("injected lost answer") } return tag, err } type lossWorld struct { raw *lossFaultStore pub ed25519.PublicKey signer release.Signer control *ControlWriter clock time.Time inc string key string claimK string } // newLossWorld is a host mid-move from ie-box (ie) to uk-box (uk): state // "landing" (the target probing, executing), "releasing" (a rollback the // target has not yet released), "released-dirty" (the target released // dirty and has not returned) or "returning". The target admitted two // sessions since the cut (20, 25); the source's cut session is 15 and the // seal did not cover the source's own replica. func newLossWorld(t *testing.T, state string, mutate func(*hostclaim.Claim, *provision.Record, *AppAdmission)) *lossWorld { t.Helper() ctx := context.Background() pub, priv, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } w := &lossWorld{raw: &lossFaultStore{Store: release.NewDirStore(t.TempDir())}, pub: pub, signer: release.RootSigner(priv), clock: time.Now().UTC().Truncate(time.Second)} if w.control, err = NewControlWriter(w.raw, pub, func() time.Time { return w.clock }); err != nil { t.Fatal(err) } d := hostactors.Directory{Store: w.raw, PublicKey: pub, Bucket: lossBucket} if err := d.Initialize(ctx, w.signer); err != nil { t.Fatal(err) } o, err := d.Read(ctx) if err != nil { t.Fatal(err) } r := hostactors.Clone(o.Roster) r.Boxes["ie-box"] = hostactors.Box{Enrollment: strings.Repeat("a", 32), Namespace: "platform", BoxID: "ie-box", Label: "ie-label", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-ie"}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"} r.Boxes["uk-box"] = hostactors.Box{Enrollment: strings.Repeat("b", 32), Namespace: "platform", BoxID: "uk-box", Label: "uk-label", Region: "uk", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-2", InstanceID: "i-uk"}, EvidenceSHA256: strings.Repeat("d", 64), State: "active"} if _, err := d.Replace(ctx, w.signer, o, r); err != nil { t.Fatal(err) } created := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) if w.inc, err = hostclaim.Incarnation("bab", "app", lossHost, created); err != nil { t.Fatal(err) } w.key = provision.RecordKey("bab", "app", lossHost) at := w.clock.Add(-time.Hour) seal := moves.Seal{V: moves.SealVersion, N: 42, FromDB: true, SrcCovered: false, SealedAt: at, LeaseUntil: at.Add(time.Minute), Replica: []moves.SealedObject{{Key: "moves/" + lossHost + "/" + lossOp + "/replica/0009/" + ltx.FormatFilename(1, 42), VersionID: "v1", Size: int64(ltx.HeaderSize)}}, Files: []moves.SealedObject{{Key: "moves/" + lossHost + "/" + lossOp + "/files/archive", VersionID: "v2", Size: 10}}} c := hostclaim.Claim{V: 2, Host: lossHost, Gen: 30, Holder: &hostclaim.Holder{Family: "instance", Key: w.key, Account: "bab", App: "app", Acquisition: "acq", CreatedAt: created, Incarnation: w.inc, Location: "instances/" + lossHost, Generation: "mv2", Stamp: 21, Session: 25, WriterBox: "uk-box", Admissions: []hostclaim.AdmissionRecord{ {Session: 15, Box: "ie-box", Generation: "src", Predecessor: "older", Region: "ie", PredecessorRegion: "ie", Mode: "cold"}, {Session: 20, Box: "uk-box", Generation: "mv", Predecessor: "src", Region: "uk", PredecessorRegion: "ie", Mode: "cold", Coverage: "sealed"}, {Session: 25, Box: "uk-box", Generation: "mv2", Predecessor: "mv", Region: "uk", PredecessorRegion: "uk", Mode: "cold"}, }}, Op: hostclaim.Op{Kind: hostclaim.KindLanding, ID: lossOp, Phase: "probing", Exec: "uk-box", Participants: []string{"ie-box", "uk-box"}, Intent: map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box", "app_id": "canonical", "admission_id": lossOp, "source_enrollment": strings.Repeat("a", 32), "target_enrollment": strings.Repeat("b", 32)}, Journal: map[string]string{hostclaim.JournalSeal: seal.Encode(), hostclaim.JournalCutSession: "15", hostclaim.JournalSourceStamp: "15", hostclaim.JournalSrcGen: "src", hostclaim.JournalEvidenceSource: "moves/bab/app/" + lossHost + "/" + lossOp + "/stopped-source.json"}}} flipped, deadline := at, at.Add(2*time.Hour) rec := provision.Record{Schema: 1, Host: lossHost, Slug: "app", Account: "bab", App: "app", Acquisition: "acq", Channel: "edge", Region: "uk", CreatedAt: created, Move: &provision.MoveState{ID: lossOp, Incarnation: w.inc, From: "ie", To: "uk", Cut: "auto", State: provision.MoveLanding, CutPhase: provision.CutFlipped, FlippedAt: &flipped, LandingDeadline: &deadline, LandingPhase: provision.LandProbing}} rb := "rb_bbbbbbbbbbbbbbbbbbbbbbbbbb" requested, release15 := at.Add(time.Minute), at.Add(16*time.Minute) switch state { case "landing": case "releasing", "released-dirty", "returning": c.Op.Kind, c.Op.Phase = hostclaim.KindRestoring, hostclaim.PhaseReleasing c.Op.Intent["rollback"] = rb c.Op.Journal[hostclaim.JournalTargetStamp] = "21" rec.Region = "ie" rec.Move.State = provision.MoveRollingBack rec.Move.Rollback = &provision.MoveRollback{ID: rb, Reason: provision.RollbackProbeAttempts, TargetPhase: provision.RbTargetReleasing, SourcePhase: provision.RbSourceWaiting, RequestedAt: &requested, ReleaseDeadline: &release15} if state != "releasing" { c.Op.Phase, c.Op.Exec = hostclaim.PhaseReleased, "" c.Op.Journal[hostclaim.JournalEvidenceTarget] = "moves/bab/app/" + lossHost + "/" + rb + "/stopped-target.json" c.Obligations = []hostclaim.Obligation{{ID: "loc", Op: lossOp, Kind: "park-replica", Region: "uk", Retirement: "r-loc", Owner: "region:uk", Park: &hostclaim.ParkScope{Location: "instances/" + lossHost, Incarnation: w.inc, EvidenceAt: at, RequestLimit: time.Minute}}} rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = provision.RbTargetReleased, true if state == "returning" { rec.Move.Rollback.TargetPhase = provision.RbTargetReturning } } default: t.Fatalf("state %q", state) } adm := AppAdmission{V: 1, AppID: "canonical", Epoch: 1, Revision: 1, State: "open", Effects: map[string]Effect{}, Registrations: map[string]Registration{lossOp: {Kind: RegistrationMove, Host: lossHost, Owner: c7Owner, AppID: "canonical", Epoch: 1, Claim: SavedClaim{ETag: "serving", Gen: 1}, Expires: w.clock.Add(RegistrationTTL), CreatedAt: w.clock, Incarnation: w.inc}}} if mutate != nil { mutate(&c, &rec, &adm) } if err := c.Validate(); err != nil { t.Fatal(err) } cb, _ := json.Marshal(c) w.claimK, _ = hostclaim.Key(lossHost) if _, err := w.raw.Store.Put(ctx, w.claimK, cb, ""); err != nil { t.Fatal(err) } rbody, _ := json.Marshal(rec) if _, err := w.raw.Store.Put(ctx, w.key, rbody, ""); err != nil { t.Fatal(err) } ab, _ := json.Marshal(adm) if _, err := w.raw.Store.Put(ctx, "apps/bab/app/admission.json", ab, ""); err != nil { t.Fatal(err) } return w } func (w *lossWorld) claim(t *testing.T) hostclaim.Observed { t.Helper() o, err := hostclaim.Read(context.Background(), w.raw, lossHost) if err != nil { t.Fatal(err) } return o } func (w *lossWorld) record(t *testing.T) (provision.Record, string) { t.Helper() rec, tag, err := provision.GetRecordETag(context.Background(), w.raw, "bab", "app", lossHost) if err != nil { t.Fatal(err) } return *rec, tag } func (w *lossWorld) roster(t *testing.T) hostactors.Observed { t.Helper() o, err := hostactors.Directory{Store: w.raw, PublicKey: w.pub, Bucket: lossBucket}.Read(context.Background()) if err != nil { t.Fatal(err) } return o } func (w *lossWorld) inspect(t *testing.T, box string) hostresolve.Inspection { t.Helper() in, err := w.control.InspectMoveLoss(context.Background(), w.signer, lossBucket, "operator", "bab", "app", lossHost, box) if err != nil { t.Fatal(err) } return in } func (w *lossWorld) submission(in hostresolve.Inspection) hostfence.Submission { w.clock = w.clock.Add(2 * time.Second) return hostfence.Submission{Challenge: in.Challenge, Confirmation: in.Details.ID, Observation: hostfence.CloudObservation{ObservedAt: in.Details.IssuedAt.Add(time.Second), Account: "123", Region: "eu-west-2", InstanceID: "i-uk", State: "stopped", RequestID: "request"}, Attestation: hostfence.OperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true, Disks: in.Details.Consequences.DestroyedStorageAssertion}} } func (w *lossWorld) declare(sub hostfence.Submission) (hostclaim.Observed, error) { return w.control.DeclareMoveLoss(context.Background(), w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box", sub) } func (w *lossWorld) acceptances(t *testing.T) []string { t.Helper() keys, err := w.raw.List(context.Background(), "control/host-loss-acceptances/") if err != nil { t.Fatal(err) } return keys } func lossSnapshot(t *testing.T, w *lossWorld) string { t.Helper() c := w.claim(t) _, tag := w.record(t) return c.ETag + "|" + tag + "|" + w.roster(t).ETag + "|" + strings.Join(w.acceptances(t), ",") } // The landing's target is lost: the declaration rolls the record back clean // with the loss as its evidence, takes the target out of the claim, and // releases it in the claim with the target region's parks, in a crash-safe // sequence whose signed record froze every lost write. func TestJ4DeclareLostLandingTarget(t *testing.T) { ctx := context.Background() w := newLossWorld(t, "landing", nil) in := w.inspect(t, "uk-label") if in.Details.Box != "uk-box" || in.Details.Op.Kind != hostclaim.KindLanding || in.Details.Consequences.Transition != moveLossTransition || !in.Details.Consequences.PossibleWriteLoss || !in.Details.Consequences.PossibleTimerLoss || len(in.Details.Consequences.DestroyedStorageAssertion) != 0 { t.Fatalf("inspection = %+v", in.Details) } _, tag := w.record(t) if in.Details.Record.Key != w.key || in.Details.Record.ETag != tag { t.Fatalf("record observation = %+v", in.Details.Record) } before := w.claim(t) sub := w.submission(in) after, err := w.declare(sub) if err != nil { t.Fatal(err) } id := in.Details.ID lossRef, _ := hostclaim.LossRecordKey(id) loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, id) if err != nil || loss.Record.State != "committed" || loss.Record.OpKind != hostclaim.KindLanding || loss.Record.ExpectedClaimETag != before.ETag { t.Fatalf("loss record = %+v %v", loss.Record, err) } wantSources := []hostclaim.WriteSource{ {Reason: "target", Box: "uk-box", Generation: "mv", Session: 20, Region: "uk", Baseline: "replica:uk/mv"}, {Reason: "target", Box: "uk-box", Generation: "mv2", Session: 25, Region: "uk", Baseline: "replica:uk/mv2"}, {Reason: "source-cut", Box: "ie-box", Generation: "src", Session: 15, Region: "ie", Baseline: "seal:" + lossOp}, } if !reflect.DeepEqual(loss.Record.WriteSources, wantSources) { t.Fatalf("write sources = %+v", loss.Record.WriteSources) } cons := loss.Record.ConfirmedConsequences if !cons.RevokeExecution || !cons.RemoveParticipation || !cons.WritesPotentiallyLost || !cons.TimersPotentiallyLost || len(cons.DischargeLocal) != 0 || len(cons.DestroyedStorage) != 0 { t.Fatalf("consequences = %+v", cons) } if b := w.roster(t).Roster.Boxes["uk-box"]; b.State != "quarantined" || !reflect.DeepEqual(b.LossIDs, []string{id}) { t.Fatalf("roster = %+v", b) } c := w.claim(t) if c.ETag != after.ETag { t.Fatal("the answer is not the claim's last write") } op := c.Claim.Op rb := op.Intent["rollback"] if op.Kind != hostclaim.KindRestoring || op.Phase != hostclaim.PhaseReleased || op.Exec != "none" || !reflect.DeepEqual(op.Participants, []string{"ie-box"}) || !provision.ValidRollbackID(rb) || op.Journal["loss."+id] == "" || !hostclaim.LostTargetReleased(c.Claim, id) { t.Fatalf("claim op = %+v", op) } if !reflect.DeepEqual(c.Claim.Lost, []string{"uk-box"}) || !reflect.DeepEqual(c.Claim.LossRefs, []string{id}) { t.Fatalf("claim lost = %v refs = %v", c.Claim.Lost, c.Claim.LossRefs) } parks := map[string]string{} for _, o := range c.Claim.Obligations { if o.Kind == "park-local" { t.Fatalf("a lost box was given a park-local: %+v", o) } parks[o.Region+"|"+o.Park.Location] = o.Owner } if !reflect.DeepEqual(parks, map[string]string{"uk|instances/" + lossHost: "region:uk", "uk|moves/" + lossHost + "/" + lossOp: "region:uk"}) { t.Fatalf("parks = %v", parks) } rec, _ := w.record(t) r := rec.Move.Rollback if rec.Region != "ie" || rec.Move.State != provision.MoveRollingBack || r == nil || r.ID != rb || r.Reason != provision.RollbackDeclaredLost || r.TargetPhase != provision.RbTargetReleased || r.Dirty || r.LossRef != lossRef || r.Evidence != lossRef || r.SourcePhase != provision.RbSourceWaiting { t.Fatalf("record move = %+v rollback = %+v", rec.Move, r) } // The source's waiting now takes the clean release, and its claimed CAS // takes execution back on the declaration's authority. if err := provision.MoveRollbackSourcePhase(provision.MoveGuard{ID: lossOp, Incarnation: w.inc}, provision.RbSourceWaiting, provision.RbSourceClaimed, nil)(&rec); err != nil { t.Fatalf("source waiting: %v", err) } if err := hostclaim.ReturnToSource(lossOp, "ie-box", true, hostclaim.ReturnFence{})(c.Claim); err != nil { t.Fatalf("source claimed: %v", err) } // A replay writes nothing more. snap := lossSnapshot(t, w) if _, err := w.declare(sub); err != nil { t.Fatal(err) } if lossSnapshot(t, w) != snap { t.Fatal("a replay wrote") } } // The target released dirty and died before its return: the record goes // clean with the loss as evidence (no fake clean-return coverage: the // signed record names the target's sessions and the uncovered source cut), // and the possibly partial return lane is parked in the source region. func TestJ4DeclareLostDirtyReleasedTarget(t *testing.T) { for _, state := range []string{"released-dirty", "returning"} { t.Run(state, func(t *testing.T) { w := newLossWorld(t, state, nil) in := w.inspect(t, "uk-box") if _, err := w.declare(w.submission(in)); err != nil { t.Fatal(err) } loss, err := hostclaim.ReadLossRecord(context.Background(), w.raw, w.pub, in.Details.ID) if err != nil || len(loss.Record.WriteSources) != 3 || loss.Record.ConfirmedConsequences.RevokeExecution { t.Fatalf("loss = %+v %v", loss.Record, err) } c := w.claim(t) parks := map[string]string{} for _, o := range c.Claim.Obligations { parks[o.Region+"|"+o.Park.Location] = o.ID } rb := "rb_bbbbbbbbbbbbbbbbbbbbbbbbbb" if parks["uk|instances/"+lossHost] != "loc" || parks["ie|moves/"+lossHost+"/"+rb] == "" || parks["uk|moves/"+lossHost+"/"+lossOp] == "" || len(parks) != 3 { t.Fatalf("parks = %v", parks) } rec, _ := w.record(t) if r := rec.Move.Rollback; r.TargetPhase != provision.RbTargetReleased || r.Dirty || r.Evidence == "" || r.Evidence != r.LossRef { t.Fatalf("rollback = %+v", r) } }) } } func TestJ4DeclareLostReleasingTarget(t *testing.T) { w := newLossWorld(t, "releasing", nil) in := w.inspect(t, "uk-box") if _, err := w.declare(w.submission(in)); err != nil { t.Fatal(err) } c := w.claim(t) if c.Claim.Op.Phase != hostclaim.PhaseReleased || c.Claim.Op.Intent["rollback"] != "rb_bbbbbbbbbbbbbbbbbbbbbbbbbb" || c.Claim.Op.Exec != "none" { t.Fatalf("op = %+v", c.Claim.Op) } rec, _ := w.record(t) if r := rec.Move.Rollback; r.ID != "rb_bbbbbbbbbbbbbbbbbbbbbbbbbb" || r.Reason != provision.RollbackProbeAttempts || r.TargetPhase != provision.RbTargetReleased || r.Dirty { t.Fatalf("rollback = %+v", r) } } // Every refusal comes before acceptance and quarantine. func TestJ4RefusalsBeforeAcceptance(t *testing.T) { ctx := context.Background() cases := map[string]struct { state string mutate func(*hostclaim.Claim, *provision.Record, *AppAdmission) box string }{ "source box": {"landing", nil, "ie-box"}, "no registration": {"landing", func(_ *hostclaim.Claim, _ *provision.Record, a *AppAdmission) { delete(a.Registrations, lossOp) }, "uk-box"}, "other registration": {"landing", func(_ *hostclaim.Claim, _ *provision.Record, a *AppAdmission) { r := a.Registrations[lossOp] r.Host = "other.example" a.Registrations[lossOp] = r }, "uk-box"}, "committed record": {"landing", func(_ *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { r.Move.State = provision.MoveCommitted at := time.Now() r.Move.CommittedAt = &at }, "uk-box"}, "committed claim": {"landing", func(c *hostclaim.Claim, _ *provision.Record, _ *AppAdmission) { c.Op.Phase = hostclaim.PhaseCommitted }, "uk-box"}, "moving claim (pre-flip)": {"landing", func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindMoving, hostclaim.PhaseSealed, "" r.Move.State, r.Move.CutPhase, r.Move.FlippedAt, r.Region = provision.MoveCutting, provision.CutArchived, nil, "ie" }, "uk-box"}, "stuck rollback": {"releasing", func(_ *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { r.Move.State = provision.MoveStuck }, "uk-box"}, "released clean": {"released-dirty", func(_ *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { r.Move.Rollback.Dirty = false }, "uk-box"}, "no seal": {"landing", func(c *hostclaim.Claim, _ *provision.Record, _ *AppAdmission) { delete(c.Op.Journal, hostclaim.JournalSeal) }, "uk-box"}, "other record move": {"landing", func(_ *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { r.Move.ID = "mv_cccccccccccccccccccccccccc" }, "uk-box"}, // G1: the claim still moving (sealed) while the record already landing. "moving claim, landing record": {"landing", func(c *hostclaim.Claim, _ *provision.Record, _ *AppAdmission) { c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindMoving, hostclaim.PhaseSealed, "" }, "uk-box"}, } for name, tc := range cases { t.Run(name, func(t *testing.T) { mutate := tc.mutate w := newLossWorld(t, tc.state, mutate) snap := lossSnapshot(t, w) _, err := w.control.InspectMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, tc.box) if err == nil { t.Fatal("inspection accepted") } if lossSnapshot(t, w) != snap { t.Fatal("a refused inspection wrote") } }) } // Another app's door cannot inspect this host. w0 := newLossWorld(t, "landing", nil) if _, err := w0.control.InspectMoveLoss(ctx, w0.signer, lossBucket, "operator", "bab", "other", lossHost, "uk-box"); !errors.Is(err, ErrIdentity) { t.Fatalf("other app = %v", err) } // The recorded refusal for a committed move reaches the door as its // sentence. w := newLossWorld(t, "landing", func(_ *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { r.Move.State = provision.MoveCommitted at := time.Now() r.Move.CommittedAt = &at }) _, err := w.control.InspectMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box") if !errors.Is(err, provision.ErrMoveConflict) || !strings.Contains(err.Error(), provision.RefusedCommitted) { t.Fatalf("committed = %v", err) } } // A declaration whose world changed after inspection is refused before // acceptance: a stale record, a stale claim, a removed registration. func TestJ4StaleObservationsRefusedBeforeAcceptance(t *testing.T) { ctx := context.Background() for name, change := range map[string]func(*testing.T, *lossWorld){ "record": func(t *testing.T, w *lossWorld) { if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), w.raw.Store, "bab", "app", lossHost, func(r *provision.Record) error { r.Move.LandingPhase = provision.LandProbed return nil }); err != nil { t.Fatal(err) } }, "claim": func(t *testing.T, w *lossWorld) { if _, err := hostclaim.Update(ctx, w.raw.Store, w.claim(t), func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } }, "registration": func(t *testing.T, w *lossWorld) { a, err := ReadAdmission(ctx, w.raw, "bab", "app", "canonical") if err != nil { t.Fatal(err) } delete(a.Value.Registrations, lossOp) b, _ := json.Marshal(a.Value) if _, err := w.raw.Store.Put(ctx, a.Key, b, a.ETag); err != nil { t.Fatal(err) } }, } { t.Run(name, func(t *testing.T) { w := newLossWorld(t, "landing", nil) sub := w.submission(w.inspect(t, "uk-box")) change(t, w) snap := lossSnapshot(t, w) if _, err := w.declare(sub); err == nil { t.Fatal("stale declaration accepted") } if lossSnapshot(t, w) != snap { t.Fatal("a refused declaration wrote") } }) } } // The submission is the signed challenge: another host, another box, // another account or app, or changed bytes are refused before acceptance. func TestJ4ChangedRequestRefused(t *testing.T) { ctx := context.Background() w := newLossWorld(t, "landing", nil) sub := w.submission(w.inspect(t, "uk-box")) snap := lossSnapshot(t, w) for name, call := range map[string]func() error{ "other host": func() error { _, err := w.control.DeclareMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", "other.example", "uk-box", sub) return err }, "other box": func() error { _, err := w.control.DeclareMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "ie-box", sub) return err }, "other app": func() error { _, err := w.control.DeclareMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "other", lossHost, "uk-box", sub) return err }, "other actor": func() error { _, err := w.control.DeclareMoveLoss(ctx, w.signer, lossBucket, "someone", "bab", "app", lossHost, "uk-box", sub) return err }, "changed body": func() error { bad := sub bad.Challenge = append([]byte(nil), sub.Challenge...) bad.Challenge[len(bad.Challenge)/2] ^= 1 _, err := w.control.DeclareMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box", bad) return err }, "other signer": func() error { _, priv, _ := ed25519.GenerateKey(rand.Reader) _, err := w.control.DeclareMoveLoss(ctx, release.RootSigner(priv), lossBucket, "operator", "bab", "app", lossHost, "uk-box", sub) return err }, "other bucket": func() error { _, err := w.control.DeclareMoveLoss(ctx, w.signer, "other-bucket", "operator", "bab", "app", lossHost, "uk-box", sub) return err }, } { if err := call(); err == nil { t.Fatalf("%s accepted", name) } if lossSnapshot(t, w) != snap { t.Fatalf("%s wrote", name) } } } // Expiry and closed admission do not revoke an accepted move. func TestJ4ExpiredRegistrationAndClosedAdmissionStillDeclare(t *testing.T) { w := newLossWorld(t, "landing", func(_ *hostclaim.Claim, _ *provision.Record, a *AppAdmission) { r := a.Registrations[lossOp] r.Expires = time.Now().Add(-48 * time.Hour) a.Registrations[lossOp] = r a.State = "closed" }) if _, err := w.declare(w.submission(w.inspect(t, "uk-box"))); err != nil { t.Fatal(err) } } // Crash and lost-answer points across the durable sequence: each fault // leaves a state the same submission completes, with nothing written twice. func TestJ4FaultsResumeFromDurableState(t *testing.T) { for _, fault := range []struct { name, key string before, lost bool nth int }{ {"record prepare fails", "record", true, false, 0}, {"record prepare answer lost", "record", false, true, 0}, {"declare CAS fails", "claim", true, false, 0}, {"declare CAS answer lost", "claim", false, true, 0}, {"loss commit fails", "loss", false, false, 2}, {"release CAS fails", "claim", false, false, 2}, {"loss-committed record write fails", "record", false, false, 2}, } { t.Run(fault.name, func(t *testing.T) { w := newLossWorld(t, "landing", nil) sub := w.submission(w.inspect(t, "uk-box")) key := map[string]string{"record": w.key, "claim": w.claimK}[fault.key] if fault.key == "loss" { key, _ = hostclaim.LossRecordKey(sub.Confirmation) } var err error if fault.nth > 0 { hook := &lossNthPut{lossFaultStore: w.raw, key: key, n: fault.nth} cw, cerr := NewControlWriter(hook, w.pub, func() time.Time { return w.clock }) if cerr != nil { t.Fatal(cerr) } _, err = cw.DeclareMoveLoss(context.Background(), w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box", sub) } else { w.raw.key, w.raw.failBefore, w.raw.failAfter = key, fault.before, fault.lost _, err = w.declare(sub) } if err == nil { t.Fatal("fault not reached") } if _, err := w.declare(sub); err != nil { t.Fatalf("resume: %v", err) } c := w.claim(t) rec, _ := w.record(t) if c.Claim.Op.Phase != hostclaim.PhaseReleased || rec.Move.Rollback.TargetPhase != provision.RbTargetReleased || len(c.Claim.Obligations) != 2 { t.Fatalf("after resume: op %+v rollback %+v obligations %d", c.Claim.Op, rec.Move.Rollback, len(c.Claim.Obligations)) } snap := lossSnapshot(t, w) if _, err := w.declare(sub); err != nil || lossSnapshot(t, w) != snap { t.Fatalf("second replay: %v (wrote %v)", err, lossSnapshot(t, w) != snap) } }) } } // lossNthPut fails the nth Put of key (before it lands). type lossNthPut struct { *lossFaultStore key string n int } func (s *lossNthPut) Put(ctx context.Context, key string, body []byte, match string) (string, error) { if key == s.key { s.n-- if s.n == 0 { return "", errors.New("injected put failure") } } return s.lossFaultStore.Store.Put(ctx, key, body, match) } // The owner's hooks are idempotent and never regress: a late PrepareLoss // after FinishLoss writes nothing; a FinishLoss replay writes nothing; a // PrepareLoss for a conflicted declaration writes nothing; a FinishLoss // facing a successor holder with no durable completion refuses. func TestJ4OwnerHooksLateAndConflicted(t *testing.T) { ctx := context.Background() w := newLossWorld(t, "landing", nil) sub := w.submission(w.inspect(t, "uk-box")) if _, err := w.declare(sub); err != nil { t.Fatal(err) } loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, sub.Confirmation) if err != nil { t.Fatal(err) } owner := &moveLossOwner{store: w.raw, pub: w.pub} snap := lossSnapshot(t, w) if err := owner.PrepareLoss(ctx, w.claim(t), loss.Record); err != nil { t.Fatalf("late prepare: %v", err) } if err := owner.FinishLoss(ctx, w.claim(t), loss.Record); err != nil { t.Fatalf("finish replay: %v", err) } if lossSnapshot(t, w) != snap { t.Fatal("a late hook wrote") } // A conflicted declaration: its prepare writes nothing. w2 := newLossWorld(t, "landing", nil) sub2 := w2.submission(w2.inspect(t, "uk-box")) // Accept and prepare by failing the record write once, then mark the // declaration conflicted by moving the claim. w2.raw.key, w2.raw.failBefore = w2.key, true if _, err := w2.declare(sub2); err == nil { t.Fatal("fault not reached") } if _, err := hostclaim.Update(ctx, w2.raw.Store, w2.claim(t), func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } if _, err := w2.declare(sub2); !errors.Is(err, hostclaim.ErrConflict) { t.Fatalf("conflicted retry = %v", err) } conflicted, err := hostclaim.ReadLossRecord(ctx, w2.raw, w2.pub, sub2.Confirmation) if err != nil || conflicted.Record.State != "conflicted" { t.Fatalf("loss = %+v %v", conflicted.Record, err) } snap2 := lossSnapshot(t, w2) prepared := conflicted.Record prepared.State = "prepared" if err := (&moveLossOwner{store: w2.raw, pub: w2.pub}).PrepareLoss(ctx, w2.claim(t), prepared); err == nil { t.Fatal("prepare of a conflicted declaration accepted") } if lossSnapshot(t, w2) != snap2 { t.Fatal("prepare of a conflicted declaration wrote") } if rec, _ := w2.record(t); rec.Move.State != provision.MoveLanding { t.Fatalf("record = %+v", rec.Move) } // A successor holder before the release: FinishLoss refuses and writes // nothing. w3 := newLossWorld(t, "landing", nil) sub3 := w3.submission(w3.inspect(t, "uk-box")) hook := &lossNthPut{lossFaultStore: w3.raw, key: w3.claimK, n: 2} cw, err := NewControlWriter(hook, w3.pub, func() time.Time { return w3.clock }) if err != nil { t.Fatal(err) } if _, err := cw.DeclareMoveLoss(ctx, w3.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box", sub3); err == nil { t.Fatal("release fault not reached") } loss3, err := hostclaim.ReadLossRecord(ctx, w3.raw, w3.pub, sub3.Confirmation) if err != nil || loss3.Record.State != "committed" { t.Fatalf("loss = %+v %v", loss3.Record, err) } if _, err := hostclaim.Update(ctx, w3.raw.Store, w3.claim(t), func(c *hostclaim.Claim) error { c.Holder.CreatedAt = c.Holder.CreatedAt.Add(time.Second) inc, err := hostclaim.Incarnation("bab", "app", lossHost, c.Holder.CreatedAt) c.Holder.Incarnation = inc return err }); err != nil { t.Fatal(err) } snap3 := lossSnapshot(t, w3) if err := (&moveLossOwner{store: w3.raw, pub: w3.pub}).FinishLoss(ctx, w3.claim(t), loss3.Record); err == nil { t.Fatal("finish against a successor holder accepted") } if lossSnapshot(t, w3) != snap3 { t.Fatal("finish against a successor wrote") } } // ValidateDeclareLost is pure and exact: changed signed effects are refused // inside the claim CAS. func TestJ4ValidateDeclareLostRefusesChangedEffects(t *testing.T) { ctx := context.Background() w := newLossWorld(t, "landing", nil) o := w.claim(t) owner := &moveLossOwner{store: w.raw, pub: w.pub} _, tag := w.record(t) _, actual, err := owner.InspectLoss(ctx, o, hostfence.RecordObservation{Key: w.key, ETag: tag, CommitState: "present"}, "uk-box") if err != nil { t.Fatal(err) } d := hostclaim.LostDeclaration{LossID: "loss_1", ExpectedClaimGen: o.Claim.Gen, ExpectedClaimETag: o.ETag, HolderIncarnation: w.inc, HolderStamp: o.Claim.Holder.Stamp, HolderSession: o.Claim.Holder.Session, OpID: lossOp, OpKind: hostclaim.KindLanding, OpPhase: "probing", Box: "uk-box", QuarantineRevision: 1, Enrollment: strings.Repeat("b", 32), FencedAt: w.clock, Consequences: actual} if err := owner.ValidateDeclareLost(*o.Claim, d); err != nil { t.Fatal(err) } for name, change := range map[string]func(*hostclaim.LostDeclaration){ "keep execution": func(d *hostclaim.LostDeclaration) { d.Consequences.RevokeExecution = false }, "no writes lost": func(d *hostclaim.LostDeclaration) { d.Consequences.WritesPotentiallyLost = false }, "destroyed storage": func(d *hostclaim.LostDeclaration) { d.Consequences.DestroyedStorage = []string{"x"} }, "other box": func(d *hostclaim.LostDeclaration) { d.Box = "ie-box" }, "other etag": func(d *hostclaim.LostDeclaration) { d.ExpectedClaimETag = "\"other\"" }, "intent": func(d *hostclaim.LostDeclaration) { d.Consequences.RecoveryIntent = json.RawMessage(`{"kind":"move-target-lost"}`) }, "intent move": func(d *hostclaim.LostDeclaration) { var m map[string]any _ = json.Unmarshal(d.Consequences.RecoveryIntent, &m) m["move"] = "mv_cccccccccccccccccccccccccc" d.Consequences.RecoveryIntent, _ = json.Marshal(m) }, "intent target sessions": func(d *hostclaim.LostDeclaration) { var m map[string]any _ = json.Unmarshal(d.Consequences.RecoveryIntent, &m) m["src_covered"] = true d.Consequences.RecoveryIntent, _ = json.Marshal(m) }, } { changed := d changed.Consequences.RecoveryIntent = append(json.RawMessage(nil), d.Consequences.RecoveryIntent...) change(&changed) if err := owner.ValidateDeclareLost(*o.Claim, changed); err == nil { t.Errorf("%s accepted", name) } } } // The scoped store behind the declaration admits only the bound // declaration's objects and exact transitions; a raw write, a delete, an // unrelated key or a different transition through it is refused. func TestJ4ScopedStoreRefusesEverythingElse(t *testing.T) { ctx := context.Background() w := newLossWorld(t, "landing", nil) o := w.claim(t) s := &lossScopedStore{raw: w.raw, pub: w.pub, now: func() time.Time { return w.clock }, b: lossBinding{lossID: "loss_1", host: lossHost, opID: lossOp, kind: hostclaim.KindLanding, box: "uk-box", claimETag: o.ETag, incarnation: w.inc, account: "bab", app: "app"}} snap := lossSnapshot(t, w) if _, err := s.Put(ctx, "apps/bab/app/other.json", []byte(`{}`), ""); !errors.Is(err, ErrContract) { t.Fatalf("unrelated key = %v", err) } if err := s.Delete(ctx, w.key); !errors.Is(err, ErrContract) { t.Fatalf("delete = %v", err) } // A loss record or acceptance for another declaration id. if _, err := s.Put(ctx, "control/host-loss/loss_2.json", []byte(`{}`), ""); !errors.Is(err, ErrContract) { t.Fatalf("other loss = %v", err) } if _, err := s.Put(ctx, "control/host-loss-acceptances/loss_1.json", []byte(`{}`), ""); err == nil { t.Fatal("unsigned acceptance accepted") } // The roster, unsigned or with another change. if _, err := s.Put(ctx, hostactors.Key, []byte(`{}`), w.roster(t).ETag); err == nil { t.Fatal("unsigned roster accepted") } ro := w.roster(t) r := hostactors.Clone(ro.Roster) b := r.Boxes["ie-box"] b.State, b.QuarantineRevision, b.LossIDs = "quarantined", 1, []string{"loss_1"} r.Boxes["ie-box"] = b if _, err := (hostactors.Directory{Store: s, PublicKey: w.pub, Bucket: lossBucket}).Replace(ctx, w.signer, ro, r); err == nil { t.Fatal("quarantine of another box accepted") } if lossSnapshot(t, w) != snap { t.Fatal("a refused scoped write landed") } } // Only the move kinds are registered, and only on the typed methods: a // serving claim is not resolvable through them. func TestJ4OnlyMoveKindsAreServed(t *testing.T) { w := newLossWorld(t, "landing", func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) { c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "ie-box"} r.Move = nil }) if _, err := w.control.InspectMoveLoss(context.Background(), w.signer, lossBucket, "operator", "bab", "app", lossHost, "ie-box"); !errors.Is(err, hostresolve.ErrResolve) { t.Fatalf("serving = %v", err) } } // lossAfterPut runs after once, right after the first Put of key lands. type lossAfterPut struct { *lossFaultStore key string after func() } func (s *lossAfterPut) Put(ctx context.Context, key string, body []byte, match string) (string, error) { tag, err := s.lossFaultStore.Store.Put(ctx, key, body, match) if err == nil && key == s.key && s.after != nil { f := s.after s.after = nil f() } return tag, err } // The registration is reread at every scoped write, not only at entry: one // removed after the acceptance stops the quarantine and everything after it. func TestJ4RegistrationRecheckedAtEachWrite(t *testing.T) { ctx := context.Background() w := newLossWorld(t, "landing", nil) sub := w.submission(w.inspect(t, "uk-box")) roster := w.roster(t).ETag hook := &lossAfterPut{lossFaultStore: w.raw, key: "control/host-loss-acceptances/" + sub.Confirmation + ".json", after: func() { a, err := ReadAdmission(ctx, w.raw, "bab", "app", "canonical") if err != nil { t.Fatal(err) } delete(a.Value.Registrations, lossOp) b, _ := json.Marshal(a.Value) if _, err := w.raw.Store.Put(ctx, a.Key, b, a.ETag); err != nil { t.Fatal(err) } }} cw, err := NewControlWriter(hook, w.pub, func() time.Time { return w.clock }) if err != nil { t.Fatal(err) } before := w.claim(t) rec, tag := w.record(t) if _, err := cw.DeclareMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box", sub); !errors.Is(err, ErrPending) { t.Fatalf("declaration after the registration went = %v", err) } if w.roster(t).ETag != roster { t.Fatal("quarantined without the registration") } if c := w.claim(t); c.ETag != before.ETag { t.Fatal("claim written without the registration") } if after, tag2 := w.record(t); tag2 != tag || after.Move.State != rec.Move.State { t.Fatal("record written without the registration") } } // A record changed after acceptance (a late box write) is not overwritten // by the declaration's decision: the prepare refuses and the declaration // stays prepared, with the claim untouched. func TestJ4RecordChangedAfterAcceptanceRefused(t *testing.T) { ctx := context.Background() w := newLossWorld(t, "landing", nil) sub := w.submission(w.inspect(t, "uk-box")) lossKey, _ := hostclaim.LossRecordKey(sub.Confirmation) hook := &lossAfterPut{lossFaultStore: w.raw, key: lossKey, after: func() { if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), w.raw.Store, "bab", "app", lossHost, func(r *provision.Record) error { r.Move.LandingPhase = provision.LandProbed return nil }); err != nil { t.Fatal(err) } }} cw, err := NewControlWriter(hook, w.pub, func() time.Time { return w.clock }) if err != nil { t.Fatal(err) } before := w.claim(t) if _, err := cw.DeclareMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box", sub); !errors.Is(err, hostclaim.ErrConflict) { t.Fatalf("declaration over a changed record = %v", err) } rec, _ := w.record(t) if rec.Move.State != provision.MoveLanding || rec.Move.LandingPhase != provision.LandProbed || rec.Move.Rollback != nil { t.Fatalf("record = %+v", rec.Move) } if c := w.claim(t); c.ETag != before.ETag { t.Fatal("claim written over a changed record") } loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, sub.Confirmation) if err != nil || loss.Record.State != "prepared" { t.Fatalf("loss = %+v %v", loss.Record, err) } } // conflictAfterPrepare drives a declaration to B1's stranded state: the // record prepare lands (lost with this loss as its reference), the claim // CAS fails, an unrelated claim write moves the ETag, and the retry marks // the declaration conflicted. func conflictAfterPrepare(t *testing.T, w *lossWorld, sub hostfence.Submission) { t.Helper() ctx := context.Background() w.raw.key, w.raw.failBefore = w.claimK, true if _, err := w.declare(sub); err == nil { t.Fatal("claim fault not reached") } ref, _ := hostclaim.LossRecordKey(sub.Confirmation) if rec, _ := w.record(t); rec.Move.Rollback == nil || rec.Move.Rollback.TargetPhase != provision.RbTargetLost || rec.Move.Rollback.LossRef != ref { t.Fatalf("record after prepare = %+v", rec.Move) } if _, err := hostclaim.Update(ctx, w.raw.Store, w.claim(t), func(*hostclaim.Claim) error { return nil }); err != nil { t.Fatal(err) } if _, err := w.declare(sub); !errors.Is(err, hostclaim.ErrConflict) { t.Fatalf("conflicted retry = %v", err) } if loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, sub.Confirmation); err != nil || loss.Record.State != "conflicted" { t.Fatalf("loss = %+v %v", loss.Record, err) } } // B1: a declaration conflicted after its record prepare leaves the record // lost with a dead reference. A fresh declaration rebinds it and completes, // keeping the rollback the first one decided, and the source takes over. func TestJ4ConflictedPreparedLossRebinds(t *testing.T) { ctx := context.Background() w := newLossWorld(t, "landing", nil) sub1 := w.submission(w.inspect(t, "uk-box")) conflictAfterPrepare(t, w, sub1) rec1, _ := w.record(t) rb1 := rec1.Move.Rollback.ID in2, err := w.control.InspectMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box") if err != nil { t.Fatalf("fresh inspection: %v", err) } sub2 := w.submission(in2) if _, err := w.declare(sub2); err != nil { t.Fatalf("fresh declaration: %v", err) } ref2, _ := hostclaim.LossRecordKey(sub2.Confirmation) c := w.claim(t) if c.Claim.Op.Kind != hostclaim.KindRestoring || c.Claim.Op.Phase != hostclaim.PhaseReleased || c.Claim.Op.Intent["rollback"] != rb1 || !hostclaim.LostTargetReleased(c.Claim, sub2.Confirmation) || !reflect.DeepEqual(c.Claim.LossRefs, []string{sub2.Confirmation}) { t.Fatalf("claim = %+v refs %v", c.Claim.Op, c.Claim.LossRefs) } rec, _ := w.record(t) if r := rec.Move.Rollback; r.ID != rb1 || r.TargetPhase != provision.RbTargetReleased || r.Dirty || r.LossRef != ref2 || r.Evidence != ref2 || r.Reason != provision.RollbackDeclaredLost { t.Fatalf("rollback = %+v", r) } if err := provision.MoveRollbackSourcePhase(provision.MoveGuard{ID: lossOp, Incarnation: w.inc}, provision.RbSourceWaiting, provision.RbSourceClaimed, nil)(&rec); err != nil { t.Fatalf("source waiting: %v", err) } if err := hostclaim.ReturnToSource(lossOp, "ie-box", true, hostclaim.ReturnFence{})(c.Claim); err != nil { t.Fatalf("source claimed: %v", err) } } // lossIntentFor reads a declaration's signed intent. func lossIntentFor(t *testing.T, w *lossWorld, id string) moveLossIntent { t.Helper() loss, err := hostclaim.ReadLossRecord(context.Background(), w.raw, w.pub, id) if err != nil { t.Fatal(err) } in, err := decodeMoveLossIntent(loss.Record.ConfirmedConsequences.RecoveryIntent) if err != nil { t.Fatal(err) } return in } // forgeLoss rewrites a signed loss record with the root signer (standing in // for any signed state the test needs) and returns the bytes it replaced. func forgeLoss(t *testing.T, w *lossWorld, id string, change func(*hostclaim.LossRecord)) []byte { t.Helper() ctx := context.Background() key, _ := hostclaim.LossRecordKey(id) old, tag, err := w.raw.Store.Get(ctx, key) if err != nil { t.Fatal(err) } loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, id) if err != nil { t.Fatal(err) } r := loss.Record change(&r) wire, err := release.SignRootEnvelope(w.signer, r) if err != nil { t.Fatal(err) } if _, err := w.raw.Store.Put(ctx, key, wire, tag); err != nil { t.Fatal(err) } return old } // A dirty target's rebind keeps the signed dirty phase and its possibly // partial return lane, across a chain of conflicted declarations; the // rollback the first declaration found stands throughout, and a late // prepare of a superseded declaration writes nothing. func TestJ4ConflictedLossChainKeepsSignedPhase(t *testing.T) { ctx := context.Background() for _, tc := range []struct { state, phase string dirty, lane bool }{ {"landing", provision.RbTargetReleasing, false, false}, {"released-dirty", provision.RbTargetReleased, true, true}, {"returning", provision.RbTargetReturning, true, true}, } { t.Run(tc.state, func(t *testing.T) { w := newLossWorld(t, tc.state, nil) sub1 := w.submission(w.inspect(t, "uk-box")) conflictAfterPrepare(t, w, sub1) rec1, _ := w.record(t) rb1 := rec1.Move.Rollback.ID ref1, _ := hostclaim.LossRecordKey(sub1.Confirmation) sub2 := w.submission(w.inspect(t, "uk-box")) conflictAfterPrepare(t, w, sub2) ref2, _ := hostclaim.LossRecordKey(sub2.Confirmation) sub3 := w.submission(w.inspect(t, "uk-box")) if _, err := w.declare(sub3); err != nil { t.Fatalf("third declaration: %v", err) } for id, supersedes := range map[string]string{sub2.Confirmation: ref1, sub3.Confirmation: ref2} { in := lossIntentFor(t, w, id) if in.Supersedes != supersedes || in.Rollback != rb1 || in.RecordState != moveLossRecordRollingBack || in.TargetPhase != tc.phase || in.Dirty != tc.dirty || in.ReturnLane != tc.lane { t.Fatalf("intent %s = %+v", id, in) } } ref3, _ := hostclaim.LossRecordKey(sub3.Confirmation) rec, _ := w.record(t) if r := rec.Move.Rollback; r.ID != rb1 || r.TargetPhase != provision.RbTargetReleased || r.Dirty || r.Evidence != ref3 || r.LossRef != ref3 { t.Fatalf("rollback = %+v", r) } c := w.claim(t) if c.Claim.Op.Intent["rollback"] != rb1 || !reflect.DeepEqual(c.Claim.LossRefs, []string{sub3.Confirmation}) { t.Fatalf("claim = %+v %v", c.Claim.Op, c.Claim.LossRefs) } lane := false for _, o := range c.Claim.Obligations { lane = lane || (o.Region == "ie" && o.Park.Location == "moves/"+lossHost+"/"+rb1) } if lane != tc.lane { t.Fatalf("return lane parked = %v, obligations %+v", lane, c.Claim.Obligations) } // Late hooks of the superseded declarations write nothing. snap := lossSnapshot(t, w) owner := &moveLossOwner{store: w.raw, pub: w.pub} for _, id := range []string{sub1.Confirmation, sub2.Confirmation} { loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, id) if err != nil { t.Fatal(err) } prepared := loss.Record prepared.State = "prepared" _ = owner.PrepareLoss(ctx, c, prepared) if _, err := w.declare(map[string]hostfence.Submission{sub1.Confirmation: sub1, sub2.Confirmation: sub2}[id]); !errors.Is(err, hostclaim.ErrConflict) { t.Fatalf("superseded replay = %v", err) } } if lossSnapshot(t, w) != snap { t.Fatal("a superseded declaration wrote") } }) } } // Only a signed, conflicted declaration of this move's target whose rollback // and dirty bit the record still carries may be superseded; everything // else stays refused before acceptance. func TestJ4ConflictedLossRebindRefusals(t *testing.T) { ctx := context.Background() for name, setup := range map[string]func(*testing.T, *lossWorld){ "old prepared": func(t *testing.T, w *lossWorld) { sub := w.submission(w.inspect(t, "uk-box")) w.raw.key, w.raw.failBefore = w.claimK, true if _, err := w.declare(sub); err == nil { t.Fatal("claim fault not reached") } }, "old committed": func(t *testing.T, w *lossWorld) { sub := w.submission(w.inspect(t, "uk-box")) conflictAfterPrepare(t, w, sub) forgeLoss(t, w, sub.Confirmation, func(r *hostclaim.LossRecord) { r.State, r.ResultClaimGen = "committed", r.ExpectedClaimGen+1 }) }, "old other host": func(t *testing.T, w *lossWorld) { sub := w.submission(w.inspect(t, "uk-box")) conflictAfterPrepare(t, w, sub) forgeLoss(t, w, sub.Confirmation, func(r *hostclaim.LossRecord) { r.Host = "other.example" }) }, "old other rollback": func(t *testing.T, w *lossWorld) { sub := w.submission(w.inspect(t, "uk-box")) conflictAfterPrepare(t, w, sub) forgeLoss(t, w, sub.Confirmation, func(r *hostclaim.LossRecord) { var m map[string]any _ = json.Unmarshal(r.ConfirmedConsequences.RecoveryIntent, &m) m["rollback"] = "rb_cccccccccccccccccccccccccc" r.ConfirmedConsequences.RecoveryIntent, _ = json.Marshal(m) }) }, "old other incarnation": func(t *testing.T, w *lossWorld) { sub := w.submission(w.inspect(t, "uk-box")) conflictAfterPrepare(t, w, sub) forgeLoss(t, w, sub.Confirmation, func(r *hostclaim.LossRecord) { var m map[string]any _ = json.Unmarshal(r.ConfirmedConsequences.RecoveryIntent, &m) m["incarnation"] = "other" r.ConfirmedConsequences.RecoveryIntent, _ = json.Marshal(m) r.HolderIncarnation = "other" }) }, "old unsigned": func(t *testing.T, w *lossWorld) { sub := w.submission(w.inspect(t, "uk-box")) conflictAfterPrepare(t, w, sub) key, _ := hostclaim.LossRecordKey(sub.Confirmation) _, tag, _ := w.raw.Store.Get(ctx, key) _, priv, _ := ed25519.GenerateKey(rand.Reader) loss, _ := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, sub.Confirmation) wire, _ := release.SignRootEnvelope(release.RootSigner(priv), loss.Record) if _, err := w.raw.Store.Put(ctx, key, wire, tag); err != nil { t.Fatal(err) } }, "record names a fence": func(t *testing.T, w *lossWorld) { sub := w.submission(w.inspect(t, "uk-box")) conflictAfterPrepare(t, w, sub) if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), w.raw.Store, "bab", "app", lossHost, func(r *provision.Record) error { r.Move.Rollback.LossRef = "control/host-fences/f.json" return nil }); err != nil { t.Fatal(err) } }, "record dirty changed": func(t *testing.T, w *lossWorld) { sub := w.submission(w.inspect(t, "uk-box")) conflictAfterPrepare(t, w, sub) if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), w.raw.Store, "bab", "app", lossHost, func(r *provision.Record) error { r.Move.Rollback.Dirty = true return nil }); err != nil { t.Fatal(err) } }, } { t.Run(name, func(t *testing.T) { w := newLossWorld(t, "landing", nil) setup(t, w) snap := lossSnapshot(t, w) if _, err := w.control.InspectMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box"); err == nil { t.Fatal("inspection accepted") } if lossSnapshot(t, w) != snap { t.Fatal("a refused inspection wrote") } }) } } // The rebind's record write rereads the superseded declaration at the // final write: a superseded declaration no longer signed conflicted stops // both the owner and the scoped store; restored, the same declaration // completes. func TestJ4ConflictedLossRebindRecheckedAtWrite(t *testing.T) { ctx := context.Background() w := newLossWorld(t, "landing", nil) sub1 := w.submission(w.inspect(t, "uk-box")) conflictAfterPrepare(t, w, sub1) sub2 := w.submission(w.inspect(t, "uk-box")) w.raw.key, w.raw.failBefore = w.key, true if _, err := w.declare(sub2); err == nil { t.Fatal("record fault not reached") } loss2, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, sub2.Confirmation) if err != nil || loss2.Record.State != "prepared" { t.Fatalf("loss = %+v %v", loss2.Record, err) } conflicted := forgeLoss(t, w, sub1.Confirmation, func(r *hostclaim.LossRecord) { r.State = "prepared" }) snap := lossSnapshot(t, w) if err := (&moveLossOwner{store: w.raw, pub: w.pub}).PrepareLoss(ctx, w.claim(t), loss2.Record); err == nil { t.Fatal("owner rebind over a prepared declaration accepted") } in2, ref2, err := lossIntentOf(loss2.Record) if err != nil { t.Fatal(err) } s := &lossScopedStore{raw: w.raw, pub: w.pub, now: func() time.Time { return w.clock }, b: lossBinding{lossID: sub2.Confirmation, host: lossHost, opID: lossOp, kind: hostclaim.KindLanding, box: "uk-box", claimETag: loss2.Record.ExpectedClaimETag, incarnation: w.inc, account: "bab", app: "app"}} if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), s, "bab", "app", lossHost, func(r *provision.Record) error { return prepareMoveLoss(r, in2, ref2, loss2.Record.FencedAt) }); err == nil { t.Fatal("scoped rebind over a prepared declaration accepted") } if lossSnapshot(t, w) != snap { t.Fatal("a refused rebind wrote") } key1, _ := hostclaim.LossRecordKey(sub1.Confirmation) _, tag, _ := w.raw.Store.Get(ctx, key1) if _, err := w.raw.Store.Put(ctx, key1, conflicted, tag); err != nil { t.Fatal(err) } if _, err := w.declare(sub2); err != nil { t.Fatalf("resume: %v", err) } if rec, _ := w.record(t); rec.Move.Rollback.TargetPhase != provision.RbTargetReleased || rec.Move.Rollback.Evidence != ref2 { t.Fatalf("rollback = %+v", rec.Move.Rollback) } }