package hostops import ( "context" "crypto/rand" "encoding/json" "errors" "strconv" "sync" "sync/atomic" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) // MoveRecoveryGrant is one callback's authority over an exact move and record. // It exposes no store or arbitrary transition. Mutations consume the observation; // another action requires another pass. The grant expires when the hook returns. // Record-first owners close before releasing the claim; LastMove repairs a // crash between those writes on the next pass. type MoveRecoveryGrant struct { state *moveRecoveryGrantState w *ControlWriter o ConsoleMoveObservation authority hostclaim.FenceAuthority scope *moveRecoveryPassScope // wrote is what the grant's record write did (Transition), once it has // made one. wrote *moves.Transition } type moveRecoveryGrantState struct { mu sync.Mutex active bool } func moveRecoveryKind(kind string) bool { switch kind { case hostclaim.KindMoving, hostclaim.KindLanding, hostclaim.KindRestoring, hostclaim.KindReverting, hostclaim.KindRelocating: return true } return false } type moveRecoveryPassKey struct{} type moveRecoveryPassScope struct{ live atomic.Bool } func recoveryWriter(ctx context.Context) *ControlWriter { scope, _ := ctx.Value(moveRecoveryPassKey{}).(*moveRecoveryPassScope) if scope == nil || !scope.live.Load() { return nil } grants, _ := ctx.Value(lifecycleGrantKey{}).([]lifecycleGrant) for _, g := range grants { if g.kind == "recovery" && g.w != nil { return g.w } } return nil } func runMoveRecovery(ctx context.Context, obs hostclaim.Observed, hook func(context.Context, *MoveRecoveryGrant) error, authorities ...hostclaim.FenceAuthority) error { w := recoveryWriter(ctx) if w == nil || hook == nil { return ErrContract } if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || !moveRecoveryKind(obs.Claim.Op.Kind) { return ErrIdentity } h, c := obs.Claim.Holder, obs.Claim if !operationHomeMatches(w.homeRegion, c) { return ErrIdentity } if err := moveAdmission(ctx, w.raw, obs.Host, c); err != nil { return err } r, err := ReadRecord(ctx, w.raw, h.Key) if err != nil { return err } if !matches(r, h) { return ErrIdentity } _, seq, err := recordSequence(r.Body) if err != nil { return err } scope, _ := ctx.Value(moveRecoveryPassKey{}).(*moveRecoveryPassScope) var authority hostclaim.FenceAuthority if len(authorities) == 1 { authority = authorities[0] } g := &MoveRecoveryGrant{state: &moveRecoveryGrantState{active: true}, w: w, scope: scope, authority: authority, o: ConsoleMoveObservation{Account: h.Account, App: h.App, AppID: c.Op.Intent["app_id"], Host: obs.Host, Incarnation: h.Incarnation, OpID: c.Op.ID, ClaimETag: obs.ETag, RecordETag: r.ETag, RecordSeq: seq}} state := g.state defer func() { state.mu.Lock(); state.active = false; state.mu.Unlock() }() // The owner receives only this capability, never the minting context. ctx = context.WithValue(ctx, lifecycleGrantKey{}, []lifecycleGrant(nil)) ctx = context.WithValue(ctx, moveRecoveryPassKey{}, (*moveRecoveryPassScope)(nil)) return hook(ctx, g) } func (g *MoveRecoveryGrant) current(ctx context.Context) (hostclaim.Observed, Record, provision.Record, error) { if g == nil || g.state == nil || !g.state.active || g.w == nil || g.scope == nil || !g.scope.live.Load() { return hostclaim.Observed{}, Record{}, provision.Record{}, ErrPending } o, err := g.w.consoleMoveClaim(ctx, g.o) if err != nil { return o, Record{}, provision.Record{}, err } r, err := ReadRecord(ctx, g.w.raw, o.Claim.Holder.Key) if err != nil { return o, r, provision.Record{}, err } _, seq, err := recordSequence(r.Body) if err != nil { return o, r, provision.Record{}, err } if r.ETag != g.o.RecordETag || seq != g.o.RecordSeq { return o, r, provision.Record{}, release.ErrCASConflict } if !matches(r, o.Claim.Holder) { return o, r, provision.Record{}, ErrIdentity } rec, err := provision.DecodeRecord(r.Body) return o, r, rec, err } // Transition is what the grant's record write did, named as the move log // names it (moves.TransitionOf over the record before and after the write), // once Record has written. It is a value, never authority: the caller's log // entry is built from the write's own evidence, not from a later read. func (g *MoveRecoveryGrant) Transition() (moves.Transition, bool) { if g == nil || g.wrote == nil { return moves.Transition{}, false } return *g.wrote, true } // Observation contains values, never authority. func (g *MoveRecoveryGrant) Observation() ConsoleMoveObservation { return g.o } type MoveRecoveryRecordAction string const ( RecoveryMoveClose MoveRecoveryRecordAction = "close" RecoveryMoveRolledBackClose MoveRecoveryRecordAction = "rolled-back-close" RecoveryMoveRevertClose MoveRecoveryRecordAction = "revert-close" RecoveryMoveDeadlineRollback MoveRecoveryRecordAction = "deadline-rollback" ) func (g *MoveRecoveryGrant) Record(ctx context.Context, action MoveRecoveryRecordAction) error { if g == nil || g.state == nil { return ErrPending } g.state.mu.Lock() defer g.state.mu.Unlock() o, r, rec, err := g.current(ctx) if err != nil { return err } guard := provision.MoveGuard{ID: g.o.OpID, Incarnation: g.o.Incarnation} if rec.Move == nil || rec.Move.ID != guard.ID || rec.Move.Incarnation != guard.Incarnation { return ErrIdentity } switch action { case RecoveryMoveClose, RecoveryMoveRolledBackClose, RecoveryMoveRevertClose: for _, chore := range o.Claim.Obligations { if chore.Op == g.o.OpID { return ErrPending } } } now := g.w.now().UTC() var mutate func(*provision.Record) error switch action { case RecoveryMoveClose: if o.Claim.Op.Kind != hostclaim.KindLanding || o.Claim.Op.Phase != hostclaim.PhaseCommitted { return ErrPending } mutate = provision.MoveClose(guard, now) case RecoveryMoveRolledBackClose: if o.Claim.Op.Kind != hostclaim.KindRestoring || o.Claim.Op.Phase != hostclaim.PhaseOpened { return ErrPending } mutate = provision.MoveRolledBackClose(guard, now) case RecoveryMoveRevertClose: if o.Claim.Op.Kind != hostclaim.KindReverting || o.Claim.Op.Phase != hostclaim.PhaseOpened { return ErrPending } mutate = provision.MoveRevertClose(guard, now) case RecoveryMoveDeadlineRollback: if o.Claim.Op.Kind != hostclaim.KindLanding && !(o.Claim.Op.Kind == hostclaim.KindMoving && o.Claim.Op.Phase == hostclaim.PhaseSealed) { return ErrPending } deadline := rec.Move.CutDeadline if d := rec.Move.LandingDeadline; d != nil && (deadline == nil || d.Before(*deadline)) { deadline = d } if deadline == nil || !now.After(*deadline) { return ErrPending } id, err := moves.NewRollbackID(rand.Reader) if err != nil { return err } mutate = provision.MoveRollBack(guard, id, provision.RollbackLandingDeadline, now) default: return ErrIdentity } if err := mutate(&rec); err != nil { return err } body, err := g.prepareRecord(ctx, rec) if err != nil { return err } if _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), g.w.raw, r.Holder.Key, r.Body, body, r.ETag); err != nil { return err } if before, err := provision.DecodeRecord(r.Body); err == nil { if t, ok := moves.TransitionOf(before, rec); ok { g.wrote = &t } } return nil } func (g *MoveRecoveryGrant) prepareRecord(ctx context.Context, rec provision.Record) ([]byte, error) { if rec.Move != nil { rec.Move.UpdatedAt = g.w.now().UTC() } body, err := json.Marshal(rec) if err != nil { return nil, err } if err := CheckCreateFreeze(ctx, g.w.raw, g.w.pub); err != nil { return nil, err } if _, _, _, err := g.current(ctx); err != nil { return nil, err } g.state.active = false return body, nil } // captureMoveStartBase records the highest attempted CAS base before forwarding // that Put. A lost journal answer never permits the Put. Only the issuing effect // can extend this proof; an uncertain effect can never acquire another base. func (h *StartHandle) captureMoveStartBase(ctx context.Context, seq uint64) error { _, err := h.w.retryAdmission(ctx, h.account, h.app, h.appID, AdmissionObserved{}, func(a *AppAdmission) error { e, ok := a.Effects[h.id] if !ok || e.State != "issued" || e.Action != moveStartAction || e.Owner != h.owner || e.OperationID != h.id || e.AppID != h.appID { return ErrPending } if old, ok := e.Inputs["record_base_seq"]; ok { n, err := strconv.ParseUint(old, 10, 64) if err != nil { return ErrIdentity } if n >= seq { return nil } } if e.Inputs == nil { e.Inputs = map[string]string{} } e.Inputs["record_base_seq"] = strconv.FormatUint(seq, 10) a.Effects[h.id] = e return nil }) return err } // ResolveStart settles only an uncertain start. Unlike the live-handle path it // never manufactures a fence: absence proves failure only above the durable base. func (g *MoveRecoveryGrant) ResolveStart(ctx context.Context) (Outcome, error) { if g == nil || g.state == nil { return OutcomeUncertain, ErrPending } g.state.mu.Lock() defer g.state.mu.Unlock() o, _, rec, err := g.current(ctx) if err != nil { return OutcomeUncertain, err } if o.Claim.Op.Kind != hostclaim.KindMoving { return OutcomeUncertain, ErrIdentity } a, err := g.w.read(ctx, g.o.Account, g.o.App, g.o.AppID) if err != nil { return OutcomeUncertain, err } e, ok := a.Value.Effects[g.o.OpID] if !ok || e.ID != g.o.OpID || e.Action != moveStartAction || e.OperationID != g.o.OpID || e.AppID != g.o.AppID || e.Resource != o.Claim.Holder.Key || e.Inputs["move_id"] != g.o.OpID || e.Inputs["incarnation"] != g.o.Incarnation { return OutcomeUncertain, ErrIdentity } if e.State == "complete" { return settledMoveStart(e, g.o.OpID) } if e.State != "uncertain" { return OutcomeUncertain, ErrPending } result := "" if rec.Move != nil && rec.Move.ID == g.o.OpID && rec.Move.Incarnation == g.o.Incarnation || rec.LastMove != nil && rec.LastMove.ID == g.o.OpID || e.ResultReference == g.o.OpID { result = g.o.OpID } if result == "" { base, err := strconv.ParseUint(e.Inputs["record_base_seq"], 10, 64) if err != nil || g.o.RecordSeq <= base { return OutcomeUncertain, ErrMoveStartFenceUnproven } if e.ResultReference != "" && e.ResultReference != "not-applied" { return OutcomeUncertain, ErrIdentity } result = "not-applied" } if _, _, _, err := g.current(ctx); err != nil { return OutcomeUncertain, err } runner := EffectRunner{w: g.w, account: g.o.Account, app: g.o.App, appID: g.o.AppID, operation: g.o.OpID, owner: e.Owner, now: g.w.now} g.state.active = false if err := runner.resolveUncertain(ctx, e, result); err != nil { if errors.Is(err, errSettledElsewhere) { a, rerr := g.w.read(ctx, g.o.Account, g.o.App, g.o.AppID) if rerr == nil { return settledMoveStart(a.Value.Effects[g.o.OpID], g.o.OpID) } } return OutcomeUncertain, err } if result == g.o.OpID { return OutcomeComplete, nil } return OutcomeNotApplied, nil } type MoveRecoveryClaimAction string const ( RecoveryMoveFlip MoveRecoveryClaimAction = "flip" RecoveryMoveCommit MoveRecoveryClaimAction = "commit" RecoveryMoveRollback MoveRecoveryClaimAction = "rollback" RecoveryMoveRelease MoveRecoveryClaimAction = "release" RecoveryMoveReleaseNotApplied MoveRecoveryClaimAction = "release-not-applied" ) // Claim catches up a claim half from its authoritative record. No caller // supplies executor identity, scopes, generation, or an arbitrary transition. func (g *MoveRecoveryGrant) Claim(ctx context.Context, action MoveRecoveryClaimAction) error { if g == nil || g.state == nil { return ErrPending } g.state.mu.Lock() defer g.state.mu.Unlock() o, _, rec, err := g.current(ctx) if err != nil { return err } c, h, m := o.Claim, o.Claim.Holder, rec.Move var transition func(*hostclaim.Claim) error if action == RecoveryMoveReleaseNotApplied { if m != nil || rec.LastMove != nil && rec.LastMove.ID == g.o.OpID || c.Op.Kind != hostclaim.KindMoving || c.Op.Phase != hostclaim.PhaseMoving { return ErrPending } a, err := g.w.read(ctx, g.o.Account, g.o.App, g.o.AppID) if err != nil { return err } e, ok := a.Value.Effects[g.o.OpID] if !ok || e.Action != moveStartAction || e.OperationID != g.o.OpID || e.AppID != g.o.AppID || e.Resource != h.Key || e.State != "complete" || e.ResultReference != "not-applied" { return ErrPending } transition = hostclaim.ReleaseMove(g.o.OpID, c.Op.Intent["source"]) } else if action == RecoveryMoveRelease { // Close first, then release. A delayed close cannot strand a serving // claim with a live move; a crash here replays from LastMove. if m != nil || rec.LastMove == nil || rec.LastMove.ID != g.o.OpID { return ErrPending } switch { case rec.LastMove.Outcome == "done" && c.Op.Kind == hostclaim.KindLanding && c.Op.Phase == hostclaim.PhaseCommitted: transition = hostclaim.ReleaseMove(g.o.OpID, c.Op.Intent["target"]) case rec.LastMove.Outcome == "rolled_back" && (c.Op.Kind == hostclaim.KindRestoring || c.Op.Kind == hostclaim.KindReverting) && c.Op.Phase == hostclaim.PhaseOpened: transition = hostclaim.ReleaseMove(g.o.OpID, c.Op.Intent["source"]) case rec.LastMove.Outcome == "aborted" && c.Op.Kind == hostclaim.KindMoving && c.Op.Phase == hostclaim.PhaseAborting: transition = hostclaim.ReleaseAbort(g.o.OpID, c.Op.Intent["source"]) default: return ErrPending } } else { if m == nil || m.ID != g.o.OpID || m.Incarnation != g.o.Incarnation { return ErrIdentity } switch action { case RecoveryMoveFlip: if m.State != provision.MoveLanding || m.FlippedAt == nil || rec.Region != m.To { return ErrPending } transition = hostclaim.Flip(g.o.OpID, c.Op.Intent["target"], h.Stamp, h.Generation) case RecoveryMoveRollback: if m.State != provision.MoveRollingBack || m.Rollback == nil || rec.Region != m.From { return ErrPending } if c.Op.Kind == hostclaim.KindMoving { transition = hostclaim.RollBackFlipped(g.o.OpID, m.Rollback.ID, c.Op.Intent["target"], h.Stamp, h.Generation) } else { transition = hostclaim.RollBack(g.o.OpID, m.Rollback.ID) } case RecoveryMoveCommit: if m.State != provision.MoveCommitted || m.CommittedAt == nil || m.FlippedAt == nil || rec.Region != m.To { return ErrPending } source, staging, err := moves.CommitScopes(c, m.FlippedAt) if err != nil { return ErrIdentity } id, err := operationToken() if err != nil { return err } n := 0 // The record's codes become engine regions as the box's commit // makes them (park.PlacementRegion; K9): a move out of home queues // its location park in park.HomeRegion, never "". transition = hostclaim.Commit(g.o.OpID, park.PlacementRegion(m.From), c.Op.Intent["source"], park.PlacementRegion(m.To), source, staging, func() string { n++; return id + strconv.Itoa(n) }) default: return ErrIdentity } } if err := CheckCreateFreeze(ctx, g.w.raw, g.w.pub); err != nil { return err } g.state.active = false _, err = hostclaim.Update(ctx, g.w.raw, o, func(c *hostclaim.Claim) error { // Re-read the exact record immediately before the claim CAS. r, err := ReadRecord(ctx, g.w.raw, h.Key) if err != nil { return err } if r.ETag != g.o.RecordETag { return release.ErrCASConflict } if err := moveAdmission(ctx, g.w.raw, o.Host, c); err != nil { return err } return transition(c) }) return err } // Convert binds C7's existing conversion to the grant and a verified fence. // The first pass writes the terminal record decision; a replay verifies it // and performs the one claim CAS. It never authorizes relocation execution. func (g *MoveRecoveryGrant) Convert(ctx context.Context, ref hostclaim.FenceRef) error { if g == nil || g.state == nil { return ErrPending } authority := g.authority if authority == nil { return ErrContract } g.state.mu.Lock() defer g.state.mu.Unlock() o, r, rec, err := g.current(ctx) if err != nil { return err } fence, err := hostclaim.VerifyFence(ctx, g.w.raw, authority, ref) if err != nil { return err } placement, ok := authority.(oneBoxRegionAuthority) if !ok { return ErrContract } guard := recoveryMovingDecision{movingRecordDecision: movingRecordDecision{store: g.w.raw, placement: placement, homeRegion: g.w.homeRegion}, observation: g.o} c, h := o.Claim, o.Claim.Holder if c.Op.Kind == hostclaim.KindRelocating { if _, err := guard.CheckMovingBeforeFlip(ctx, o, c.Op.ID, ref); err != nil { return err } return hostclaim.VerifyOperationFence(ctx, g.w.raw, authority, c.Op, c.Op.Intent["source_box"]) } if c.Op.Kind != hostclaim.KindMoving || c.Op.Intent["source_enrollment"] != ref.Enrollment || fence.Box() != c.Op.Intent["source"] || !park.PlacementMatches(c.Op.Intent["home_region"], rec.Region, fence.Region()) { return hostclaim.ErrFence } if err := placement.CheckOneBoxRegion(ctx, fence.Box(), fence.Region()); err != nil { return err } if rec.Move == nil || rec.Move.ID != g.o.OpID || rec.Move.Incarnation != h.Incarnation { return ErrIdentity } if rec.Move.State != provision.MoveRelocating { raw, err := json.Marshal(ref) if err != nil { return err } moving, holder, err := movingIdentities(c) if err != nil { return err } if err := provision.MoveRelocatingDecision(provision.MoveGuard{ID: g.o.OpID, Incarnation: h.Incarnation}, fence.Box(), ref.Enrollment, rec.Region, string(raw), c.Gen, o.ETag, moving, holder, g.w.now())(&rec); err != nil { return err } body, err := g.prepareRecord(ctx, rec) if err != nil { return err } _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), g.w.raw, r.Holder.Key, r.Body, body, r.ETag) return err } if err := CheckCreateFreeze(ctx, g.w.raw, g.w.pub); err != nil { return err } g.state.active = false _, err = hostclaim.ConvertMovingToRelocating(ctx, g.w.raw, o, g.o.OpID, ref, authority, guard) return err } // recoveryMovingDecision retains the recovery grant's exact record binding // through the final pre-CAS guard, including after fence authority reads. type recoveryMovingDecision struct { movingRecordDecision observation ConsoleMoveObservation } func (g recoveryMovingDecision) CheckMovingBeforeFlip(ctx context.Context, o hostclaim.Observed, id string, ref hostclaim.FenceRef) (string, error) { region, err := g.movingRecordDecision.CheckMovingBeforeFlip(ctx, o, id, ref) if err != nil { return "", err } r, err := ReadRecord(ctx, g.store, o.Claim.Holder.Key) if err != nil { return "", err } _, seq, err := recordSequence(r.Body) if err != nil { return "", err } if r.ETag != g.observation.RecordETag || seq != g.observation.RecordSeq { return "", release.ErrCASConflict } return region, nil }