package hostops import ( "context" "errors" "time" ) // ErrNotApplied marks a call that returned before issuing any backend or // store effect (validation, refusal). Its effect settles complete with // result "not-applied" instead of uncertain. var ErrNotApplied = errors.New("hostops: effect not applied") // EffectRunner issues durable effects under one registration or exclusive // operation. Its fields are unexported: runners come only from a // StartHandle or OperationHandle, so a caller cannot impersonate an owner // it merely read from the bucket. type EffectRunner struct { w *ControlWriter account, app, appID string operation string owner RegistrationOwner now func() time.Time exclusive bool } func (r EffectRunner) AppID() string { return r.appID } func (r EffectRunner) Operation() string { return r.operation } func (r EffectRunner) Owner() RegistrationOwner { return r.owner } func (r EffectRunner) authorized(a *AppAdmission) bool { if r.exclusive { return a.State == "closed" && a.Op != nil && a.Op.ID == r.operation && a.Op.AppID == r.appID } reg, ok := a.Registrations[r.operation] return ok && reg.Owner == r.owner && reg.AppID == r.appID && reg.Epoch == a.Epoch && RegistrationLive(reg, r.now()) } // Issue prepares and issues e, then returns a context carrying a live permit // and the settle function. settle MUST be called exactly once after every // call made with the context has joined; it revokes the permit first, then // records complete or uncertain. A deadline is never settlement. func (r EffectRunner) Issue(ctx context.Context, e Effect) (context.Context, func(result string, callErr error) error, error) { if r.w == nil || r.now == nil || r.operation == "" { return nil, nil, ErrContract } if e.AppID == "" { e.AppID = r.appID } if e.OperationID == "" { e.OperationID = r.operation } if e.Owner == (RegistrationOwner{}) { e.Owner = r.owner } if e.Owner != r.owner || e.AppID != r.appID || e.OperationID != r.operation { return nil, nil, ErrContract } if err := CheckCreateFreeze(ctx, r.w.raw, r.w.pub); err != nil { return nil, nil, err } a, err := r.w.read(ctx, r.account, r.app, r.appID) if err != nil { return nil, nil, err } if old, ok := a.Value.Effects[e.ID]; ok { // A recorded ID is a result or pending work, never a fresh permit. if old.AppID != e.AppID || old.OperationID != e.OperationID || old.Resource != e.Resource || old.Action != e.Action { return nil, nil, ErrIdentity } if old.State == "complete" { return nil, nil, errEffectComplete{result: old.ResultReference} } return nil, nil, ErrPending } e.State = "prepared" // Authorization is rechecked inside each CAS, so an operation reopened by // recovery between read and write can never gain this effect. Both CASes // below run BEFORE the call, so replaying them after a lost race is safe: // prepareEffect is idempotent for the identical effect, and // advanceEffect(prepared->issued) refuses an effect someone superseded. if a, err = r.w.retryAdmission(ctx, r.account, r.app, r.appID, a, func(v *AppAdmission) error { if !r.authorized(v) { return ErrPending } if r.exclusive { // A same-operation effect that was only prepared never issued its // call (a crashed or competing worker). Supersede it in this CAS; // that worker's prepared->issued CAS then fails. Issued and // uncertain effects still block. for id, old := range v.Effects { if old.OperationID == r.operation && old.Resource == e.Resource && old.State == "prepared" && id != e.ID { old.State, old.ResultReference = "complete", "superseded" v.Effects[id] = old } } } return prepareEffect(v, e) }); err != nil { return nil, nil, err } if a, err = r.w.retryAdmission(ctx, r.account, r.app, r.appID, a, func(v *AppAdmission) error { if !r.authorized(v) { return ErrPending } if cur, ok := v.Effects[e.ID]; !ok || cur.State != "prepared" && cur.State != "issued" { // Superseded or reconciled while we raced: never call. return ErrPending } return advanceEffect(v, e.ID, r.owner, "issued", "") }); err != nil { return nil, nil, err } issuedCtx, revoke := withAppMutationEffect(ctx, r, e, a.Value.Epoch) issuedCtx = context.WithValue(issuedCtx, effectPermitKey{}, effectPermit{Resource: e.Resource}) settled := false settle := func(result string, callErr error) error { if settled { return ErrContract } settled = true revoke() state := "complete" switch { case callErr != nil && errors.Is(callErr, ErrNotApplied): result = "not-applied" case callErr != nil || result == "": // A transport error does not prove the server did not act. state = "uncertain" } decided, saveErr := r.saveSettlement(ctx, e.ID, "", state, result) if saveErr != nil && !decided { return errors.Join(callErr, saveErr) } if decided { return callErr } if callErr != nil { return callErr } if state != "complete" { return ErrPending } return nil } return issuedCtx, settle, nil } // saveSettlement records a settle step's outcome for effect id. The outcome // is known now and only now: a lost CAS against a routine renewal or a // sibling effect re-reads and re-applies (bounded), so the effect does not // stay "issued" for want of one retry. Settle needs no live registration: // advanceEffect checks the owner only. A non-empty from is the only state // the effect may leave. decided reports that recovery already recorded a // different complete decision, which stands. func (r EffectRunner) saveSettlement(ctx context.Context, id, from, state, result string) (decided bool, err error) { saveCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 20*time.Second) defer cancel() _, err = r.w.retryAdmission(saveCtx, r.account, r.app, r.appID, AdmissionObserved{}, func(v *AppAdmission) error { cur, ok := v.Effects[id] if ok && cur.State == "complete" && (cur.State != state || cur.ResultReference != result) { // Recovery already decided this effect (restore receipt, gate // reconciliation). Its decision stands; never overwrite it. decided = true return errSettledElsewhere } if from != "" && ok && cur.State != from && cur.State != state { return ErrPending } if cur.State == "prepared" && cur.Owner == r.owner && cur.OperationID == r.operation && cur.Action == sqlIngestAction && state == "complete" && result == "sql-fenced" { // SQL's durable fence blocks a delayed issued CAS from ever reaching a // write. Superseding prepared also makes that CAS refuse immediately. cur.State, cur.ResultReference = state, result v.Effects[id] = cur return nil } return advanceEffect(v, id, r.owner, state, result) }) return decided, err } // resolveUncertain is the settle step for a move-start effect that Issue // left uncertain, once ResolveMoveStart has had the bucket decide its record // Put: complete with the move id, or "not-applied". It is scoped to this // runner's own registration and to that one action. An exclusive runner, // another operation, owner or action refuses, and only an uncertain effect // (or one already holding this same decision) is written. func (r EffectRunner) resolveUncertain(ctx context.Context, e Effect, result string) error { if r.w == nil || r.exclusive || r.operation == "" || e.Action != moveStartAction || e.OperationID != r.operation || e.AppID != r.appID || e.Owner != r.owner || result == "" { return ErrContract } decided, err := r.saveSettlement(ctx, e.ID, "uncertain", "complete", result) if decided { return errSettledElsewhere } return err } // Run issues e, calls issue exactly once with the permitted context, joins // it, and settles. issue must not start a goroutine that outlives it. func (r EffectRunner) Run(ctx context.Context, e Effect, issue func(context.Context) (string, error)) (string, error) { if issue == nil { return "", ErrContract } issuedCtx, settle, err := r.Issue(ctx, e) var done errEffectComplete if errors.As(err, &done) { return done.result, nil } if err != nil { return "", err } var result string var callErr error func() { defer func() { if p := recover(); p != nil { _ = settle("", errors.New("hostops: effect call panicked")) panic(p) } }() result, callErr = issue(issuedCtx) }() if err = settle(result, callErr); err != nil { return "", err } return result, nil } // EffectResult reports Issue's "already complete" refusal and its saved // result. That refusal grants no permit; the caller must not call again. func EffectResult(err error) (string, bool) { var done errEffectComplete if errors.As(err, &done) { return done.result, true } return "", false } type errEffectComplete struct{ result string } // errSettledElsewhere aborts a settle CAS whose effect recovery already // recorded complete. var errSettledElsewhere = errors.New("hostops: effect settled by recovery") func (e errEffectComplete) Error() string { return "hostops: effect already complete" }