package hostops import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "github.com/carlosframework/platform/internal/park" "strconv" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) // movingIdentities digests the claim's operation and holder. The record // decision binds both, so a changed operation can never reuse it, while a // chore acknowledgement (which changes neither) can. func movingIdentities(c *hostclaim.Claim) (string, string, error) { if c == nil || c.Holder == nil { return "", "", ErrIdentity } h, err := json.Marshal(c.Holder) if err != nil { return "", "", err } op, err := json.Marshal(c.Op) if err != nil { return "", "", err } hs, ops := sha256.Sum256(h), sha256.Sum256(op) return hex.EncodeToString(ops[:]), hex.EncodeToString(hs[:]), nil } // moveAdmission rereads the app admission the move started under (Task 7). // Accepted work drains under closed admission and expiry does not revoke it, // so only identity is compared: the exact "move" registration, named by the // move's own op id, for this host and incarnation, at the admission's epoch, // under the claim's app id. func moveAdmission(ctx context.Context, s release.Store, host string, c *hostclaim.Claim) error { if c == nil || c.Holder == nil { return ErrIdentity } appID, id := c.Op.Intent["app_id"], c.Op.Intent["admission_id"] if appID == "" || id == "" || id != c.Op.ID { return ErrIdentity } a, err := ReadAdmission(ctx, s, c.Holder.Account, c.Holder.App, appID) if err != nil { return err } r, ok := a.Value.Registrations[id] if !ok || r.Kind != RegistrationMove || r.AppID != appID || r.Epoch != a.Value.Epoch || r.Host != host || r.Incarnation != c.Holder.Incarnation { return ErrPending } return nil } // movingRecordDecision is the PreFlipGuard over the authoritative record's // durable decision (provision.MoveRelocatingDecision). type oneBoxRegionAuthority interface { CheckOneBoxRegion(context.Context, string, string) error } type movingRecordDecision struct { homeRegion string placement oneBoxRegionAuthority store release.Store } var _ hostclaim.PreFlipGuard = movingRecordDecision{} // CheckMovingBeforeFlip reads the move's own intent while the claim is // moving ("source", "to") and the conversion's once relocating // ("source_box", "region", "move_to_region"). The region it returns is the // verified roster identity for the record placement; the home code stays // empty in the decision. ConvertMovingToRelocating compares the returned // identity with the unmodified fence binding. // ConvertMovingToRelocating calls it again inside its transition, so the // admission reread here is the last read before the one claim CAS. func (g movingRecordDecision) CheckMovingBeforeFlip(ctx context.Context, o hostclaim.Observed, opID string, ref hostclaim.FenceRef) (string, error) { if o.Claim == nil || o.Claim.Holder == nil || o.Absent || o.ETag == "" || opID == "" || o.Claim.Op.ID != opID { return "", ErrIdentity } c, h := o.Claim, o.Claim.Holder if !operationHomeMatches(g.homeRegion, c) { return "", ErrIdentity } if err := moveAdmission(ctx, g.store, o.Host, c); err != nil { return "", err } rec, etag, err := provision.GetRecordETag(ctx, g.store, h.Account, h.App, o.Host) if err != nil { return "", err } inc, err := hostclaim.Incarnation(rec.Account, rec.App, rec.Host, rec.CreatedAt) if err != nil || etag == "" || inc != h.Incarnation || rec.Move == nil || rec.Move.ID != c.Op.ID || rec.Move.Incarnation != h.Incarnation || rec.Move.State != provision.MoveRelocating || rec.Move.Relocation == nil { return "", ErrIdentity } d := rec.Move.Relocation moving, holder, err := movingIdentities(c) if err != nil { return "", err } if holder != d.HolderIdentity { return "", ErrIdentity } var source, region, to string switch c.Op.Kind { case hostclaim.KindMoving: if d.MovingIdentity != moving || c.Gen < d.ClaimGen { return "", hostclaim.ErrConflict } region = park.PlacementRegion(rec.Region) if rec.Region == "" && c.Op.Intent["home_region"] != "" { region = c.Op.Intent["home_region"] } source, to = c.Op.Intent["source"], c.Op.Intent["to"] case hostclaim.KindRelocating: convertedGen, err := strconv.ParseUint(c.Op.Journal["conversion_claim_gen"], 10, 64) raw, e := json.Marshal(d) if e != nil { return "", e } sum := sha256.Sum256(raw) if err != nil || convertedGen < d.ClaimGen || c.Op.Journal["conversion_claim_etag"] == "" || c.Op.Journal["conversion_decision_sha256"] != hex.EncodeToString(sum[:]) { return "", hostclaim.ErrConflict } source, region, to = c.Op.Intent["source_box"], c.Op.Intent["region"], c.Op.Intent["move_to_region"] default: return "", hostclaim.ErrWrongState } raw, err := json.Marshal(ref) if err != nil { return "", err } if source == "" || region == "" || d.DecidedAt.IsZero() || d.Fence != string(raw) || d.Enrollment != ref.Enrollment || d.Enrollment != c.Op.Intent["source_enrollment"] || d.SourceBox != source || !park.PlacementMatches(c.Op.Intent["home_region"], d.Region, region) || rec.Region != d.Region || rec.Move.From != d.Region || !park.PlacementMatches(c.Op.Intent["home_region"], rec.Move.To, to) || rec.Move.FlippedAt != nil || rec.Move.CommittedAt != nil || rec.Move.Rollback != nil { return "", ErrIdentity } switch d.PreviousState { case provision.MovePreparing, provision.MoveReady, provision.MoveCutting: default: return "", ErrIdentity } if g.placement == nil { return "", ErrContract } if err := g.placement.CheckOneBoxRegion(ctx, source, region); err != nil { return "", err } return region, nil } // ConversionDecisionDigest binds the converted claim to the exact record // decision it was converted under. func (g movingRecordDecision) ConversionDecisionDigest(ctx context.Context, o hostclaim.Observed, id string, ref hostclaim.FenceRef) (string, error) { if _, err := g.CheckMovingBeforeFlip(ctx, o, id, ref); err != nil { return "", err } h := o.Claim.Holder rec, err := provision.GetRecord(ctx, g.store, h.Account, h.App, o.Host) if err != nil { return "", err } if rec.Move == nil || rec.Move.Relocation == nil { return "", ErrIdentity } raw, err := json.Marshal(rec.Move.Relocation) if err != nil { return "", err } digest := sha256.Sum256(raw) return hex.EncodeToString(digest[:]), nil } // heldMoveRegistration: mh is the writer-minted handle of the move's own // registration (its id is the op id and admission_id, its app id the // claim's), still owned by mh at mh's epoch and live by the writer clock. // The data boundary checks the same again at each write (the issued effect // at the record Put, the lifecycle grant at the claim CAS); this check makes // an unbound call refuse before any of them. Admission may be closed: // accepted work drains under closed admission. func heldMoveRegistration(ctx context.Context, s release.Store, mh *StartHandle, host string, c *hostclaim.Claim) error { if mh == nil || mh.w == nil || c == nil || c.Holder == nil || mh.account != c.Holder.Account || mh.app != c.Holder.App || mh.id != c.Op.ID || mh.appID != c.Op.Intent["app_id"] { return ErrIdentity } if err := moveAdmission(ctx, s, host, c); err != nil { return err } a, err := ReadAdmission(ctx, s, mh.account, mh.app, mh.appID) if err != nil { return err } r := a.Value.Registrations[mh.id] if r.Owner != mh.owner || r.Epoch != mh.epoch || a.Value.Epoch != mh.epoch || !RegistrationLive(r, mh.w.now()) { return ErrPending } return nil } // RecoverTerminatedMovingHost is called only with claim mode enabled, by the // Moves coordinator holding mh, the handle RegisterMove minted for this move // and kept live (Renew) since (no caller is wired yet). It persists the // record decision as one issued effect of that registration, then converts // the claim in one CAS under its lifecycle grant. No relocation // installation or execution is activated, including on replay. // // While moving, the source is the claim's "source"; once relocating, its // "source_box". The region is never claim input: it is the authoritative // record's, and must equal the fence's roster binding. func RecoverTerminatedMovingHost(ctx context.Context, s release.Store, mh *StartHandle, authority hostclaim.FenceAuthority, host string, ref hostclaim.FenceRef, now time.Time) error { o, err := hostclaim.Read(ctx, s, host) if err != nil { return err } if o.Absent || o.Claim.Holder == nil { return ErrIdentity } c, h := o.Claim, o.Claim.Holder if err := heldMoveRegistration(ctx, s, mh, host, c); err != nil { return err } fence, err := hostclaim.VerifyFence(ctx, s, authority, ref) if err != nil { return err } placement, ok := authority.(oneBoxRegionAuthority) if !ok { return ErrContract } if !operationHomeMatches(mh.w.homeRegion, c) { return ErrIdentity } guard := movingRecordDecision{store: s, placement: placement, homeRegion: mh.w.homeRegion} if c.Op.Kind == hostclaim.KindRelocating { source := c.Op.Intent["source_box"] if fence.Box() != source || fence.Region() != c.Op.Intent["region"] { return hostclaim.ErrFence } if _, err := guard.CheckMovingBeforeFlip(ctx, o, c.Op.ID, ref); err != nil { return err } return hostclaim.VerifyOperationFence(ctx, s, authority, c.Op, source) } if c.Op.Kind != hostclaim.KindMoving || c.Op.Intent["source_enrollment"] != ref.Enrollment { return hostclaim.ErrWrongState } source := c.Op.Intent["source"] if fence.Box() != source { return hostclaim.ErrFence } rec, err := provision.GetRecord(ctx, s, h.Account, h.App, host) if err != nil { return err } region := rec.Region if !park.PlacementMatches(c.Op.Intent["home_region"], region, fence.Region()) || rec.Move == nil || rec.Move.From != region { return hostclaim.ErrFence } if err := placement.CheckOneBoxRegion(ctx, source, fence.Region()); err != nil { return err } raw, err := json.Marshal(ref) if err != nil { return err } movingIdentity, holderIdentity, err := movingIdentities(c) if err != nil { return err } // A replay whose decision is already durable for this exact operation // and holder writes nothing more (a crash after the record Put leaves // that effect uncertain, and it would block a second one). if _, undecided := guard.CheckMovingBeforeFlip(ctx, o, c.Op.ID, ref); undecided != nil { moveGuard := provision.MoveGuard{ID: c.Op.ID, Incarnation: h.Incarnation} effect, err := operationToken() if err != nil { return err } if _, err = runRecordEffect(ctx, mh, Effect{ID: effect, Resource: h.Key, Action: "move-relocating-decision", Inputs: map[string]string{"move_id": c.Op.ID, "claim_etag": o.ETag, "fence_sha256": ref.SHA256}}, func(ctx context.Context) (string, error) { _, err := setLifecycleRecordMove(ctx, s, o, func(rec *provision.Record) error { // Compare the claim before deciding the record. MoveRelocatingDecision // rechecks the fresh record's region against the bound region. fresh, err := hostclaim.Read(ctx, s, host) if err != nil { return err } if fresh.Absent || fresh.ETag != o.ETag { return hostclaim.ErrConflict } return provision.MoveRelocatingDecision(moveGuard, source, ref.Enrollment, region, string(raw), o.Claim.Gen, o.ETag, movingIdentity, holderIdentity, now)(rec) }) if err != nil { return "", err } return c.Op.ID, nil }); err != nil { return err } } fresh, err := hostclaim.Read(ctx, s, host) if err != nil { return err } if fresh.Absent || fresh.Claim.Op.Kind != hostclaim.KindMoving { return hostclaim.ErrWrongState } freshMoving, freshHolder, err := movingIdentities(fresh.Claim) if err != nil { return err } if freshMoving != movingIdentity || freshHolder != holderIdentity { return hostclaim.ErrConflict } _, err = hostclaim.ConvertMovingToRelocating(mh.Lifecycle(ctx), s, fresh, c.Op.ID, ref, authority, guard) return err }