package hostops import ( "bytes" "context" "crypto/ed25519" "crypto/sha256" "encoding/base32" "encoding/hex" "encoding/json" "errors" "fmt" "github.com/carlosframework/platform/internal/park" "io" "reflect" "strconv" "strings" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfence" "github.com/carlosframework/platform/internal/hostresolve" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) // The moves owner of a declared-lost target (moves spec §7.10, claim Plan 10 // J4). hostresolve owns the signed challenge, the acceptance, the roster // quarantine and the loss record; this owner decides which target loss is // legal, what it costs, and completes the move's own record and claim // consequences around the generic claim CAS: // // - InspectLoss (read-only): the exact target, the record observation and // the effects, with every value recovery needs frozen in the signed // recovery intent; // - LossWriteSources (pure): every target session since the cut and an // uncovered source cut, frozen into the signed record; // - ValidateDeclareLost (pure): the signed effects are exactly what this // claim implies; // - PrepareLoss: the record's rollback decision with the target lost, // before the claim CAS, so no commit can follow the declaration; // - FinishLoss: the claim's release of the lost target, then the record's // clean release with the loss as its evidence. const ( moveLossIntentKind = "move-target-lost" // moveLossTransition is the inspection's machine-readable transition. moveLossTransition = "move-target-lost" // moveLossRequestLimit bounds a park's request window for the lost // target's scopes, as the box's own release bounds its (cutRequestLimit). moveLossRequestLimit = 2 * time.Minute moveLossRecordLanding = "landing" moveLossRecordRollingBack = "rolling_back" ) // Move kinds share the exact local-only cleanup owner. Data-authority // target loss remains restricted to landing before commit and restoring // before source takeover; moving and reverting gain no takeover authority. var moveLossKinds = []string{hostclaim.KindMoving, hostclaim.KindLanding, hostclaim.KindReverting, hostclaim.KindRestoring} // moveLossIntent is the signed recovery intent: everything the hooks and a // crash recovery need, bound to the inspected claim and record. type moveLossIntent struct { Kind string `json:"kind"` Move string `json:"move"` Incarnation string `json:"incarnation"` Rollback string `json:"rollback"` Target string `json:"target"` Source string `json:"source"` TargetRegion string `json:"target_region"` SourceRegion string `json:"source_region"` RecordKey string `json:"record_key"` RecordETag string `json:"record_etag"` RecordSeq uint64 `json:"record_seq"` RecordState string `json:"record_state"` TargetPhase string `json:"target_phase,omitempty"` Dirty bool `json:"dirty"` ReturnLane bool `json:"return_lane"` SrcCovered bool `json:"src_covered"` CutSession uint64 `json:"cut_session"` SrcGen string `json:"src_gen,omitempty"` // Supersedes is the loss record key a lost record still names when this // declaration rebinds it: that declaration's claim CAS never landed and // its signed record is conflicted (B1). TargetPhase, Dirty and // ReturnLane are then that declaration's signed values, since lost has // erased the record's own. Supersedes string `json:"supersedes,omitempty"` } func (in moveLossIntent) encode() (json.RawMessage, error) { raw, err := json.Marshal(in) if err != nil { return nil, err } return canonicalJSON(raw) } // canonicalJSON re-encodes one JSON value with sorted keys and exact // numbers, so the signed intent and every recomputation compare as bytes. func canonicalJSON(raw []byte) (json.RawMessage, error) { dec := json.NewDecoder(bytes.NewReader(raw)) dec.UseNumber() var v any if err := dec.Decode(&v); err != nil { return nil, err } if _, err := dec.Token(); err != io.EOF { return nil, ErrIdentity } return json.Marshal(v) } func decodeMoveLossIntent(raw json.RawMessage) (moveLossIntent, error) { dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() var in moveLossIntent if err := dec.Decode(&in); err != nil { return moveLossIntent{}, hostresolve.ErrResolve } if _, err := dec.Token(); err != io.EOF { return moveLossIntent{}, hostresolve.ErrResolve } if in.Kind != moveLossIntentKind || in.Move == "" || in.Incarnation == "" || !provision.ValidRollbackID(in.Rollback) || in.Target == "" || in.Source == "" || in.TargetRegion == "" || in.SourceRegion == "" || in.RecordKey == "" || in.RecordETag == "" || (in.RecordState != moveLossRecordLanding && in.RecordState != moveLossRecordRollingBack) { return moveLossIntent{}, hostresolve.ErrResolve } return in, nil } // moveLossRollbackID is the rollback a landing's declaration starts. It is // derived from the inspected claim observation, so a recomputation inside // the claim CAS (ValidateDeclareLost) and every replay agree on it. func moveLossRollbackID(opID, claimETag, box string) string { sum := sha256.Sum256([]byte(moveLossIntentKind + "\x00" + opID + "\x00" + claimETag + "\x00" + box)) enc := strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(sum[:])) return "rb_" + enc[:26] } // moveLossNewID mints the declaration's chore ids deterministically, so the // scoped store can recompute the release's claim write byte for byte. func moveLossNewID(lossID string) func() string { n := 0 return func() string { n++ sum := sha256.Sum256([]byte(moveLossIntentKind + "\x00" + lossID + "\x00" + strconv.Itoa(n))) return hex.EncodeToString(sum[:16]) } } // moveLossRefusal is a refusal the move door answers with its recorded // sentence (phase1Conflict); anything else is the door's fixed fallback. func moveLossRefusal(sentence string) error { return fmt.Errorf("%w: %s", provision.ErrMoveConflict, sentence) } // claimLossIntent is the claim's half of a legal target loss: the box is // the move's exact target, a participant and not the source; the op is a // landing before its commit or this move's rollback before the source took // over; the seal says whether the source's own replica covered its cut. func claimLossIntent(c *hostclaim.Claim, box string) (moveLossIntent, error) { if c == nil || c.Holder == nil || c.Holder.Family != "instance" || box == "" { return moveLossIntent{}, hostresolve.ErrResolve } op := c.Op source, target := op.Intent["source"], op.Intent["target"] if op.ID == "" || op.Intent["admission_id"] != op.ID || source == "" || target == "" || source == target || op.Intent["to"] == "" { return moveLossIntent{}, hostresolve.ErrResolve } if box == source || box != target { return moveLossIntent{}, moveLossRefusal("the source box cannot be declared lost by a move") } if len(op.Participants) != 2 || op.Participants[0] != source || op.Participants[1] != target { return moveLossIntent{}, hostresolve.ErrResolve } for _, lost := range c.Lost { if lost == box { return moveLossIntent{}, hostresolve.ErrResolve } } in := moveLossIntent{Kind: moveLossIntentKind, Move: op.ID, Incarnation: c.Holder.Incarnation, Target: target, Source: source, TargetRegion: op.Intent["to"], RecordKey: c.Holder.Key} switch { case op.Kind == hostclaim.KindLanding && op.Phase == hostclaim.PhaseCommitted: return moveLossIntent{}, moveLossRefusal(provision.RefusedCommitted) case op.Kind == hostclaim.KindLanding: if op.Exec != target { return moveLossIntent{}, hostresolve.ErrResolve } case op.Kind == hostclaim.KindRestoring && op.Phase == hostclaim.PhaseReleasing: if op.Exec != target || !provision.ValidRollbackID(op.Intent["rollback"]) { return moveLossIntent{}, hostresolve.ErrResolve } in.Rollback = op.Intent["rollback"] case op.Kind == hostclaim.KindRestoring && op.Phase == hostclaim.PhaseReleased: if op.Exec != "" || op.Journal[hostclaim.JournalEvidenceTarget] == "" || !provision.ValidRollbackID(op.Intent["rollback"]) { return moveLossIntent{}, hostresolve.ErrResolve } in.Rollback = op.Intent["rollback"] case op.Kind == hostclaim.KindRestoring: return moveLossIntent{}, moveLossRefusal(provision.RefusedNothingToDeclare) default: return moveLossIntent{}, hostresolve.ErrResolve } seal, err := moves.DecodeSeal(op.Journal[hostclaim.JournalSeal]) if err != nil { return moveLossIntent{}, hostresolve.ErrResolve } // The cut is a paired identity: a real session with its generation, or // the legacy {session 0, generation ""} of a source that never had one // (claim spec §6.4a). A missing key or a mismatched pair is refused. rawCut, ok := op.Journal[hostclaim.JournalCutSession] if !ok { return moveLossIntent{}, hostresolve.ErrResolve } cut, err := strconv.ParseUint(rawCut, 10, 64) if err != nil { return moveLossIntent{}, hostresolve.ErrResolve } in.SrcCovered, in.CutSession = seal.SrcCovered, cut if !seal.SrcCovered { gen, ok := op.Journal[hostclaim.JournalSrcGen] if !ok || (cut == 0) != (gen == "") { return moveLossIntent{}, hostresolve.ErrResolve } in.SrcGen = gen } return in, nil } // supersededLoss is the conflicted declaration a lost record still names, // with the target phase, Dirty and ReturnLane it signed (a landing's // declaration signed none: its rollback started the target releasing). type supersededLoss struct { in moveLossIntent ref string } // readSuperseded reads the declaration a lost record names. Only a signed, // conflicted declaration for this host whose id the claim never took may be // superseded; the caller holds its intent to this move and target. A record // that names no loss has nothing to supersede. Read-only. func readSuperseded(ctx context.Context, store release.Store, pub ed25519.PublicKey, o hostclaim.Observed, rec provision.Record) (*supersededLoss, error) { if rec.Move == nil || rec.Move.Rollback == nil || rec.Move.Rollback.LossRef == "" { return nil, nil } rb := rec.Move.Rollback id := strings.TrimSuffix(strings.TrimPrefix(rb.LossRef, "control/host-loss/"), ".json") if key, err := hostclaim.LossRecordKey(id); err != nil || key != rb.LossRef || rb.TargetPhase != provision.RbTargetLost { return nil, hostresolve.ErrResolve } loss, err := hostclaim.ReadLossRecord(ctx, store, pub, id) if errors.Is(err, hostclaim.ErrLossRecordAbsent) || errors.Is(err, hostclaim.ErrLossRecord) { return nil, hostresolve.ErrResolve } if err != nil { return nil, err } if loss.Record.State != "conflicted" || loss.Record.Host != o.Host || o.Claim == nil || slicesContain(o.Claim.LossRefs, id) { return nil, hostresolve.ErrResolve } old, ref, err := lossIntentOf(loss.Record) if err != nil || ref != rb.LossRef { return nil, hostresolve.ErrResolve } if old.RecordState == moveLossRecordLanding { old.TargetPhase, old.Dirty, old.ReturnLane = provision.RbTargetReleasing, false, false } return &supersededLoss{in: old, ref: ref}, nil } // sameMoveTarget reports two intents for the same move, incarnation, // target, regions, record and cut. func sameMoveTarget(a, b moveLossIntent) bool { return a.Kind == b.Kind && a.Move == b.Move && a.Incarnation == b.Incarnation && a.Target == b.Target && a.Source == b.Source && a.TargetRegion == b.TargetRegion && a.SourceRegion == b.SourceRegion && a.RecordKey == b.RecordKey && a.SrcCovered == b.SrcCovered && a.CutSession == b.CutSession && a.SrcGen == b.SrcGen } // lossIntentAt is the intent the claim o and the record r imply for box, // with the declaration a lost record still names. Read-only. func lossIntentAt(ctx context.Context, store release.Store, pub ed25519.PublicKey, o hostclaim.Observed, r Record, box string) (moveLossIntent, error) { in, err := claimLossIntent(o.Claim, box) if err != nil { return moveLossIntent{}, err } if !matches(r, o.Claim.Holder) { return moveLossIntent{}, ErrIdentity } _, seq, err := recordSequence(r.Body) if err != nil { return moveLossIntent{}, err } rec, err := provision.DecodeRecord(r.Body) if err != nil { return moveLossIntent{}, ErrIdentity } prior, err := readSuperseded(ctx, store, pub, o, rec) if err != nil { return moveLossIntent{}, err } if err := recordLossIntent(&in, rec, r.ETag, seq, o.Claim.Op, o.ETag, prior); err != nil { return moveLossIntent{}, err } return in, nil } // recordLossIntent is the record's half: the record journals this move for // this incarnation, has not committed, and its target still holds what the // move needs (§7.10: it owes its stop evidence, or released dirty and has // not returned). A landing's rollback id is derived from the claim ETag. A // record lost for a conflicted declaration (prior) keeps that declaration's // rollback, phase, Dirty and return lane. func recordLossIntent(in *moveLossIntent, rec provision.Record, etag string, seq uint64, op hostclaim.Op, claimETag string, prior *supersededLoss) error { m := rec.Move if m == nil || m.ID != in.Move || m.Incarnation != in.Incarnation || !park.PlacementMatches(op.Intent["home_region"], m.To, in.TargetRegion) || park.PlacementRegion(m.From) == "" || m.From == m.To { return hostresolve.ErrResolve } if m.State == provision.MoveCommitted || m.CommittedAt != nil { return moveLossRefusal(provision.RefusedCommitted) } in.SourceRegion, in.RecordETag, in.RecordSeq = park.PlacementRegion(m.From), etag, seq switch m.State { case provision.MoveLanding: if op.Kind != hostclaim.KindLanding || m.Rollback != nil { return hostresolve.ErrResolve } in.RecordState, in.Rollback = moveLossRecordLanding, moveLossRollbackID(op.ID, claimETag, in.Target) return nil case provision.MoveRollingBack: default: return moveLossRefusal("the move's state does not allow a declared loss") } rb := m.Rollback if rb == nil || !provision.ValidRollbackID(rb.ID) { return hostresolve.ErrResolve } phase, dirty := rb.TargetPhase, rb.Dirty if rb.LossRef != "" || prior != nil { if prior == nil || prior.ref != rb.LossRef || rb.TargetPhase != provision.RbTargetLost || prior.in.Rollback != rb.ID || prior.in.Dirty != rb.Dirty || !sameMoveTarget(prior.in, *in) { return hostresolve.ErrResolve } phase, dirty, in.Supersedes = prior.in.TargetPhase, prior.in.Dirty, prior.ref } switch op.Kind { case hostclaim.KindRestoring: if rb.ID != in.Rollback { return hostresolve.ErrResolve } case hostclaim.KindLanding: // The record decided the rollback; the claim has not followed yet. in.Rollback = rb.ID } in.RecordState, in.TargetPhase, in.Dirty = moveLossRecordRollingBack, phase, dirty switch { case phase == provision.RbTargetReleasing: if op.Phase != hostclaim.PhaseReleasing && op.Kind != hostclaim.KindLanding { return hostresolve.ErrResolve } case phase == provision.RbTargetReturning, phase == provision.RbTargetReleased && dirty: if op.Kind != hostclaim.KindRestoring || op.Phase != hostclaim.PhaseReleased { return hostresolve.ErrResolve } in.ReturnLane = true default: return moveLossRefusal(provision.RefusedNothingToDeclare) } if prior != nil && prior.in.ReturnLane != in.ReturnLane { return hostresolve.ErrResolve } return nil } // moveLossConsequences are the shown and actual effects for an intent: the // target loses execution (when it holds it) and participation; the regional // work it owned goes to its region; nothing about its disks is asserted; the // writes and timers since the cut are possibly lost. func moveLossConsequences(c *hostclaim.Claim, box string, in moveLossIntent) (hostfence.Consequences, hostclaim.LossConsequences, error) { intent, err := in.encode() if err != nil { return hostfence.Consequences{}, hostclaim.LossConsequences{}, err } shown := hostfence.Consequences{Transition: moveLossTransition, PossibleWriteLoss: true, PossibleTimerLoss: true} actual := hostclaim.LossConsequences{RevokeExecution: c.Op.Exec == box, RemoveParticipation: true, WritesPotentiallyLost: true, TimersPotentiallyLost: true, RecoveryIntent: intent} for _, o := range c.Obligations { if (o.Kind == "park-replica" || o.Kind == "park-generation") && o.Owner == "box:"+box { actual.ReassignRegional = append(actual.ReassignRegional, hostclaim.ObligationRef{ID: o.ID, Op: o.Op, Retirement: o.Retirement}) shown.Chores = append(shown.Chores, o.Op+"/"+o.ID+"/"+o.Retirement) } } return shown, actual, nil } // moveLossOwner is the moves owner. store is the scoped store of one // declaration (or a read-only store for inspection); it is never exposed. type moveLossOwner struct { store release.Store pub ed25519.PublicKey now func() time.Time } var _ hostresolve.OperationOwner = (*moveLossOwner)(nil) func (m *moveLossOwner) clock() time.Time { if m.now != nil { return m.now().UTC() } return time.Now().UTC() } // InspectLoss is read-only. The record must be exactly the observation the // service bound into the challenge. func (m *moveLossOwner) InspectLoss(ctx context.Context, o hostclaim.Observed, record hostfence.RecordObservation, box string) (hostfence.Consequences, hostclaim.LossConsequences, error) { if o.Claim == nil || o.Claim.Holder == nil { return hostfence.Consequences{}, hostclaim.LossConsequences{}, hostresolve.ErrResolve } if o.Absent || o.ETag == "" || record.Key != o.Claim.Holder.Key || record.CommitState != "present" || record.ETag == "" { return hostfence.Consequences{}, hostclaim.LossConsequences{}, hostresolve.ErrResolve } r, err := ReadRecord(ctx, m.store, record.Key) if err != nil { return hostfence.Consequences{}, hostclaim.LossConsequences{}, err } if r.ETag != record.ETag { return hostfence.Consequences{}, hostclaim.LossConsequences{}, hostclaim.ErrConflict } if moveLocalLossEligible(o.Claim, box) { if !matches(r, o.Claim.Holder) { return hostfence.Consequences{}, hostclaim.LossConsequences{}, ErrIdentity } rec, e := provision.DecodeRecord(r.Body) if e != nil || rec.Move == nil || rec.Move.ID != o.Claim.Op.ID || rec.Move.Incarnation != o.Claim.Holder.Incarnation { return hostfence.Consequences{}, hostclaim.LossConsequences{}, ErrIdentity } return moveLocalLossConsequences(o.Claim, box) } in, err := lossIntentAt(ctx, m.store, m.pub, o, r, box) if err != nil { return hostfence.Consequences{}, hostclaim.LossConsequences{}, err } return moveLossConsequences(o.Claim, box, in) } // ValidateDeclareLost is pure: inside the claim CAS, the signed effects // must be exactly what this claim implies for the signed intent, whose // claim-derived values must be this claim's. func (m *moveLossOwner) ValidateDeclareLost(c hostclaim.Claim, d hostclaim.LostDeclaration) error { if isMoveLocalLoss(d.Consequences) { _, actual, err := moveLocalLossConsequences(&c, d.Box) if err != nil || !reflect.DeepEqual(actual, d.Consequences) { return hostresolve.ErrResolve } return nil } in, err := decodeMoveLossIntent(d.Consequences.RecoveryIntent) if err != nil { return err } want, err := claimLossIntent(&c, d.Box) if err != nil { return err } if in.Move != want.Move || in.Move != d.OpID || in.Incarnation != want.Incarnation || in.Incarnation != d.HolderIncarnation || in.Target != want.Target || in.Source != want.Source || in.TargetRegion != want.TargetRegion || in.RecordKey != want.RecordKey || in.SrcCovered != want.SrcCovered || in.CutSession != want.CutSession || in.SrcGen != want.SrcGen || in.SourceRegion == in.TargetRegion { return hostresolve.ErrResolve } if in.Supersedes != "" { // A rebind keeps the rollback the conflicted declaration decided; it // is never recomputed from this claim's ETag. own, err := hostclaim.LossRecordKey(d.LossID) if err != nil || in.Supersedes == own || in.RecordState != moveLossRecordRollingBack { return hostresolve.ErrResolve } for _, id := range c.LossRefs { if key, _ := hostclaim.LossRecordKey(id); key == in.Supersedes { return hostresolve.ErrResolve } } } switch { case c.Op.Kind == hostclaim.KindLanding && in.RecordState == moveLossRecordLanding: if in.Rollback != moveLossRollbackID(c.Op.ID, d.ExpectedClaimETag, d.Box) || in.TargetPhase != "" || in.Dirty || in.ReturnLane { return hostresolve.ErrResolve } case c.Op.Kind == hostclaim.KindLanding: if in.TargetPhase != provision.RbTargetReleasing || in.ReturnLane { return hostresolve.ErrResolve } case c.Op.Kind == hostclaim.KindRestoring && in.RecordState == moveLossRecordRollingBack && in.Rollback == want.Rollback: releasing := c.Op.Phase == hostclaim.PhaseReleasing && in.TargetPhase == provision.RbTargetReleasing && !in.ReturnLane owed := c.Op.Phase == hostclaim.PhaseReleased && in.ReturnLane && in.Dirty && (in.TargetPhase == provision.RbTargetReturning || in.TargetPhase == provision.RbTargetReleased) if !releasing && !owed { return hostresolve.ErrResolve } default: return hostresolve.ErrResolve } _, actual, err := moveLossConsequences(&c, d.Box, in) if err != nil { return err } got := d.Consequences if got.RecoveryIntent, err = canonicalJSON(got.RecoveryIntent); err != nil { return hostresolve.ErrResolve } if !reflect.DeepEqual(actual, got) { return hostresolve.ErrResolve } return nil } // LossWriteSources freezes every write the declaration may lose: each // target session admitted since the cut (its generation in the target // region), and the source's cut session when the seal did not cover the // source's own replica (the seal in the lost target's region is the only // holder of those writes, §7.10). Pure. func (m *moveLossOwner) LossWriteSources(_ context.Context, o hostclaim.Observed, actual hostclaim.LossConsequences) ([]hostclaim.WriteSource, error) { if isMoveLocalLoss(actual) { return nil, nil } in, err := decodeMoveLossIntent(actual.RecoveryIntent) if err != nil { return nil, err } c := o.Claim if c == nil || c.Holder == nil || c.Op.ID != in.Move || c.Holder.Incarnation != in.Incarnation { return nil, hostresolve.ErrResolve } h := c.Holder var out []hostclaim.WriteSource seen := map[string]bool{} add := func(s hostclaim.WriteSource) error { if s.Box == "" || (s.Session == 0) != (s.Generation == "") || s.Region == "" { return hostresolve.ErrResolve } key := s.Box + "\x00" + strconv.FormatUint(s.Session, 10) if !seen[key] { seen[key] = true out = append(out, s) } return nil } for _, a := range h.Admissions { if a.Box != in.Target || a.Session <= in.CutSession { continue } if err := add(hostclaim.WriteSource{Reason: "target", Box: a.Box, Generation: a.Generation, Session: a.Session, Region: a.Region, Baseline: MoveLossReplicaBaseline + a.Region + "/" + a.Generation}); err != nil { return nil, err } } if h.WriterBox == in.Target && h.Session > in.CutSession { if err := add(hostclaim.WriteSource{Reason: "target", Box: h.WriterBox, Generation: h.Generation, Session: h.Session, Region: in.TargetRegion, Baseline: MoveLossReplicaBaseline + in.TargetRegion + "/" + h.Generation}); err != nil { return nil, err } } if !in.SrcCovered { if err := add(hostclaim.WriteSource{Reason: "source-cut", Box: in.Source, Generation: in.SrcGen, Session: in.CutSession, Region: in.SourceRegion, Baseline: MoveLossSealBaseline + in.Move}); err != nil { return nil, err } } return out, nil } // lossIntentOf decodes a loss record's intent and holds the record's own // identities to it. func lossIntentOf(loss hostclaim.LossRecord) (moveLossIntent, string, error) { in, err := decodeMoveLossIntent(loss.ConfirmedConsequences.RecoveryIntent) if err != nil { return moveLossIntent{}, "", err } if loss.OpID != in.Move || loss.Box != in.Target || loss.HolderIncarnation != in.Incarnation { return moveLossIntent{}, "", hostresolve.ErrResolve } ref, err := hostclaim.LossRecordKey(loss.ID) return in, ref, err } // durableLoss rereads the signed loss record and holds it to the hook's // copy: the same declaration, the same signed intent. func (m *moveLossOwner) durableLoss(ctx context.Context, loss hostclaim.LossRecord) (hostclaim.LossRecord, error) { durable, err := hostclaim.ReadLossRecord(ctx, m.store, m.pub, loss.ID) if err != nil { return hostclaim.LossRecord{}, err } a, err := canonicalJSON(durable.Record.ConfirmedConsequences.RecoveryIntent) if err != nil { return hostclaim.LossRecord{}, err } b, err := canonicalJSON(loss.ConfirmedConsequences.RecoveryIntent) if err != nil || !bytes.Equal(a, b) || durable.Record.Host != loss.Host || durable.Record.ExpectedClaimETag != loss.ExpectedClaimETag { return hostclaim.LossRecord{}, hostresolve.ErrResolve } return durable.Record, nil } // moveLossRecordDone reports a record that already carries this // declaration (lost, or released or cleaned with it as the evidence), or a // move that has closed. Anything for another move or incarnation refuses. func moveLossRecordDone(rec provision.Record, in moveLossIntent, ref string) (lost, released bool, err error) { m := rec.Move if m == nil { if rec.LastMove != nil && rec.LastMove.ID == in.Move { return true, true, nil } return false, false, ErrIdentity } if m.ID != in.Move || m.Incarnation != in.Incarnation { return false, false, ErrIdentity } rb := m.Rollback if rb == nil || rb.LossRef == "" { return false, false, nil } if in.Supersedes != "" && rb.LossRef == in.Supersedes && rb.ID == in.Rollback && rb.TargetPhase == provision.RbTargetLost { return false, false, nil } if rb.ID != in.Rollback || rb.LossRef != ref { return false, false, hostclaim.ErrConflict } if rb.TargetPhase == provision.RbTargetLost { return true, false, nil } if rb.Evidence == ref && !rb.Dirty && (rb.TargetPhase == provision.RbTargetReleased || rb.TargetPhase == provision.RbTargetCleaned) { return true, true, nil } return false, false, hostclaim.ErrWrongState } // prepareMoveLoss is the record's half of the declaration: a landing rolls // back for the declared loss, and the rollback's target goes to lost with // the loss record as its reference. A rebind moves a lost target's // reference from the superseded declaration to this one. func prepareMoveLoss(rec *provision.Record, in moveLossIntent, ref string, at time.Time) error { g := provision.MoveGuard{ID: in.Move, Incarnation: in.Incarnation} if in.Supersedes != "" { if in.RecordState != moveLossRecordRollingBack || rec.Move == nil || rec.Move.Rollback == nil || rec.Move.Rollback.ID != in.Rollback { return ErrIdentity } return provision.MoveRollbackLossRebind(g, in.Supersedes, ref, at)(rec) } if in.RecordState == moveLossRecordLanding { if err := provision.MoveRollBack(g, in.Rollback, provision.RollbackDeclaredLost, at)(rec); err != nil { return err } } else if rec.Move == nil || rec.Move.Rollback == nil || rec.Move.Rollback.ID != in.Rollback { return ErrIdentity } return provision.MoveRollbackTargetLost(g, ref, at)(rec) } // PrepareLoss writes the record's decision before the claim CAS, so the // move can no longer commit. It never writes the claim, never regresses a // record that already carries this declaration, and writes only while the // signed loss is prepared, the claim is the declaration's exact observation // and the record is the inspected one (ETag and record_seq). func (m *moveLossOwner) PrepareLoss(ctx context.Context, o hostclaim.Observed, loss hostclaim.LossRecord) error { if isMoveLocalLoss(loss.ConfirmedConsequences) { return m.localLossHook(ctx, o, loss, false) } in, ref, err := lossIntentOf(loss) if err != nil { return err } if loss.Host != o.Host { return ErrIdentity } r, err := ReadRecord(ctx, m.store, in.RecordKey) if err != nil { return err } rec, err := provision.DecodeRecord(r.Body) if err != nil { return ErrIdentity } if done, _, err := moveLossRecordDone(rec, in, ref); err != nil || done { return err } durable, err := m.durableLoss(ctx, loss) if err != nil { return err } switch durable.State { case "prepared": case "committed": // A concurrent declaration committed; its FinishLoss writes the record. return nil default: return hostclaim.ErrConflict } claim, err := hostclaim.Read(ctx, m.store, loss.Host) if err != nil { return err } if claim.Absent || claim.ETag != durable.ExpectedClaimETag { return hostclaim.ErrConflict } _, seq, err := recordSequence(r.Body) if err != nil { return err } if r.ETag != in.RecordETag || seq != in.RecordSeq || !matches(r, claim.Claim.Holder) { return hostclaim.ErrConflict } if in.Supersedes != "" { // A rebind rereads the superseded declaration: still conflicted, // and the exact intent this claim and record imply. if want, err := lossIntentAt(ctx, m.store, m.pub, claim, r, in.Target); err != nil || !reflect.DeepEqual(want, in) { return hostclaim.ErrConflict } } if err := prepareMoveLoss(&rec, in, ref, durable.FencedAt); err != nil { return err } rec.Move.UpdatedAt = m.clock() body, err := json.Marshal(rec) if err != nil { return err } _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), m.store, in.RecordKey, r.Body, body, r.ETag) return sequenceError(err) } // moveLossRelease is the claim release the signed declaration implies for // the holder it names. Every scope and id derives from the signed record. func moveLossRelease(in moveLossIntent, loss hostclaim.LossRecord, h *hostclaim.Holder, host string) hostclaim.LostTargetRelease { scope := func(location string) hostclaim.ParkScope { return hostclaim.ParkScope{Location: location, Incarnation: h.Incarnation, EvidenceAt: loss.FencedAt.UTC(), RequestLimit: moveLossRequestLimit} } r := hostclaim.LostTargetRelease{OpID: in.Move, RollbackID: in.Rollback, LossID: loss.ID, Target: in.Target, TargetRegion: in.TargetRegion, SourceRegion: in.SourceRegion, Location: scope(strings.TrimSuffix(h.Location, "/")), Lane: scope(moves.ScopeLocation(host, in.Move)), NewID: moveLossNewID(loss.ID)} if in.ReturnLane { lane := scope(moves.ScopeLocation(host, in.Rollback)) r.ReturnLane = &lane } return r } // FinishLoss completes the declaration after its claim CAS committed: the // claim releases the lost target (ReleaseLostTarget), then the record's // lost target becomes a clean release with the loss as its evidence, which // the source's recovery takes. Each step is skipped once durable. A // successor holder or operation is never touched: only a record that // already shows this declaration complete is success then. func (m *moveLossOwner) FinishLoss(ctx context.Context, o hostclaim.Observed, loss hostclaim.LossRecord) error { if isMoveLocalLoss(loss.ConfirmedConsequences) { return m.localLossHook(ctx, o, loss, true) } in, ref, err := lossIntentOf(loss) if err != nil { return err } if loss.Host != o.Host { return ErrIdentity } durable, err := m.durableLoss(ctx, loss) if err != nil { return err } if durable.State != "committed" { return hostclaim.ErrConflict } claim, err := hostclaim.Read(ctx, m.store, loss.Host) if err != nil { return err } ours := !claim.Absent && claim.Claim != nil && claim.Claim.Holder != nil && claim.Claim.Holder.Incarnation == in.Incarnation && claim.Claim.Op.ID == in.Move released := false if ours { c := claim.Claim if !slicesContain(c.LossRefs, loss.ID) { return hostclaim.ErrConflict } if !hostclaim.LostTargetReleased(c, loss.ID) { if _, err := hostclaim.Update(ctx, m.store, claim, hostclaim.ReleaseLostTarget(moveLossRelease(in, durable, c.Holder, loss.Host))); err != nil { return err } } released = true } r, err := ReadRecord(ctx, m.store, in.RecordKey) if err != nil { return err } rec, err := provision.DecodeRecord(r.Body) if err != nil { return ErrIdentity } lost, done, err := moveLossRecordDone(rec, in, ref) if err != nil { return err } if done { return nil } if !lost || !released { return ErrIdentity } if err := provision.MoveRollbackLossCommitted(provision.MoveGuard{ID: in.Move, Incarnation: in.Incarnation})(&rec); err != nil { return err } rec.Move.UpdatedAt = m.clock() body, err := json.Marshal(rec) if err != nil { return err } _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), m.store, in.RecordKey, r.Body, body, r.ETag) return sequenceError(err) } func slicesContain(values []string, wanted string) bool { for _, v := range values { if v == wanted { return true } } return false } // errMoveLossScope refuses a write outside one declaration's scope. It wraps // ErrContract: the caller asked for something no declaration may do. var errMoveLossScope = fmt.Errorf("%w: write outside the declaration's scope", ErrContract)