package hostops // restore_recover.go extends the control writer's restore transitions // (control_writer.go: PrepareRestore, CompleteRestore, RecoverMutation) with // the one case RecoverMutation leaves pending: a restore operation that never // saved its receipt. // // A restore's only external effect is the exact-ETag delete of one trash // marker, and the issuer saves the receipt (PrepareRestore) right after // BeginExclusive, before it issues any effect and before any SQL. So an // operation with no receipt has issued nothing, and reopening it is a fence, // not a guess: after the reopen CAS (which advances the epoch) the stale // issuer can neither save a receipt (OperationHandle.current refuses), nor // issue an effect (EffectRunner.authorized refuses), nor pass the data // boundary (its permit needs the closed operation, its epoch and an issued // effect). An operation that does have an issued or uncertain effect is // never abandoned, whatever its age or receipt. The age bound only avoids // interrupting a live issuer between BeginExclusive and PrepareRestore; it // is not the safety argument. import ( "context" "math" "strconv" "time" ) // RestoreActionStarted is the intent key the restore issuer sets at // BeginExclusive (unix seconds); AbandonRestore measures age from it. const RestoreActionStarted = "started" // RestoreOutcome is what RecoverRestore did. type RestoreOutcome string const ( RestoreNone RestoreOutcome = "" // no restore operation owns the app RestoreRestored RestoreOutcome = "restored" // a saved receipt was finished and admission reopened RestoreAbandoned RestoreOutcome = "abandoned" // no receipt was saved; admission reopened ) // IsRestoreOp reports whether op is a restore app-mutation. func IsRestoreOp(op *AppOperation) bool { return op != nil && op.Kind == OpAppMutation && op.Intent["action"] == "restore" } // AbandonRestore reopens admission held by a receipt-less restore operation // at least minAge old. It reopens only when nothing of the operation was // issued: each of its effects is absent, already complete, or merely // prepared (settled "superseded", as EffectRunner.Issue supersedes one). An // issued or uncertain effect, a saved receipt, a registration or any other // operation's unsettled effect refuses. The reopen advances the epoch, as // every reopen does. func (w *ControlWriter) AbandonRestore(ctx context.Context, account, app, appID, opID string, now time.Time, minAge time.Duration) error { a, err := w.read(ctx, account, app, appID) if err != nil { return err } _, err = w.updateAdmission(ctx, a, func(v *AppAdmission) error { if v.State != "closed" || v.Op == nil || v.Op.ID != opID || v.Op.AppID != appID || !IsRestoreOp(v.Op) || len(v.Registrations) != 0 { return ErrPending } if r := v.LastRestore; r != nil && r.ID == opID { return ErrPending // a saved receipt is finished by RecoverMutation, never abandoned } sec, perr := strconv.ParseInt(v.Op.Intent[RestoreActionStarted], 10, 64) if perr != nil || sec <= 0 || now.Before(time.Unix(sec, 0).Add(minAge)) { return ErrPending } for id, e := range v.Effects { switch { case e.State == "complete": case e.OperationID == opID && e.AppID == appID && e.State == "prepared": e.State, e.ResultReference = "complete", "superseded" v.Effects[id] = e default: return ErrPending // issued, uncertain, or another operation's } } if v.Epoch == math.MaxUint64 { return ErrPending } v.Op, v.State = nil, "open" v.Epoch++ return nil }) return err } // RecoverRestore finishes whatever restore operation owns the app: a saved // receipt through RecoverMutation (the exact-ETag delete replay; a prepared // receipt may have no effect yet, because the receipt precedes Issue), a // receipt-less operation through AbandonRestore. Any other owner, or no // owner, is RestoreNone. It never starts an operation and never reopens an // admission a non-restore operation holds. func (w *ControlWriter) RecoverRestore(ctx context.Context, account, app, appID string, now time.Time, minAge time.Duration) (RestoreOutcome, error) { a, err := w.read(ctx, account, app, appID) if err != nil { return RestoreNone, err } op := a.Value.Op if a.Value.State != "closed" || !IsRestoreOp(op) { return RestoreNone, nil } if r := a.Value.LastRestore; r != nil && r.ID == op.ID { if err := w.RecoverMutation(ctx, account, app, appID, op.ID); err != nil { return RestoreNone, err } return RestoreRestored, nil } if err := w.AbandonRestore(ctx, account, app, appID, op.ID, now, minAge); err != nil { return RestoreNone, err } return RestoreAbandoned, nil }