package hostclaim import ( "fmt" "time" ) const WakeRetryLimit = 5 var wakeRetryDelays = [...]time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second} type WakeRetry struct { EpisodeID string `json:"episode_id"` AttemptsReserved int `json:"attempts_reserved"` State string `json:"state"` NotBefore time.Time `json:"not_before,omitempty"` Attempt *WakeRetryAttempt `json:"attempt,omitempty"` ResetRequest string `json:"reset_request,omitempty"` } type WakeRetryAttempt struct { ID string `json:"id"` Box string `json:"box"` Enrollment string `json:"enrollment"` BootID string `json:"boot_id"` ControllerID string `json:"controller_id"` Session uint64 `json:"session,omitempty"` InvocationID string `json:"invocation_id"` LaunchInputs string `json:"launch_inputs"` } func (r *WakeRetry) validate() error { if r == nil { return nil } if r.EpisodeID == "" || r.AttemptsReserved < 0 || r.AttemptsReserved > WakeRetryLimit { return fmt.Errorf("%w: wake retry episode", ErrMalformed) } switch r.State { case "open": if r.Attempt != nil || r.AttemptsReserved >= WakeRetryLimit { return fmt.Errorf("%w: open wake retry", ErrMalformed) } case "attempting": if r.AttemptsReserved == 0 || r.Attempt == nil || !r.NotBefore.IsZero() || !r.Attempt.valid() { return fmt.Errorf("%w: attempting wake retry", ErrMalformed) } case "failed": if r.AttemptsReserved != WakeRetryLimit || r.Attempt != nil || r.NotBefore.IsZero() { return fmt.Errorf("%w: failed wake retry", ErrMalformed) } default: return fmt.Errorf("%w: wake retry state %q", ErrMalformed, r.State) } return nil } func (a *WakeRetryAttempt) valid() bool { return a.ID != "" && a.Box != "" && a.Enrollment != "" && a.BootID != "" && a.ControllerID != "" && a.InvocationID != "" && a.LaunchInputs != "" } type WakeRetryReservation struct { HolderIncarnation string EpisodeID string Attempt WakeRetryAttempt Now time.Time } // ReserveWakeRetryAttempt spends one serving retry after the caller has // established all read-only prerequisites. Session zero denotes the first // admission and must be bound with BindWakeRetrySession before app code runs. func ReserveWakeRetryAttempt(in WakeRetryReservation) func(*Claim) error { return func(c *Claim) error { if c.Holder == nil || c.Op.Kind != KindServing || c.Holder.Incarnation != in.HolderIncarnation || in.Now.IsZero() || !in.Attempt.valid() || (in.Attempt.Session != 0 && in.Attempt.Session != c.Holder.Session) { return ErrWrongState } r := c.Holder.WakeRetry if r == nil { if in.EpisodeID == "" { return ErrWrongState } r = &WakeRetry{EpisodeID: in.EpisodeID, State: "open"} c.Holder.WakeRetry = r } else if in.EpisodeID != "" && in.EpisodeID != r.EpisodeID { return ErrWrongState } if r.State != "open" || r.AttemptsReserved >= WakeRetryLimit || in.Now.Before(r.NotBefore) { return ErrWrongState } r.AttemptsReserved++ r.State, r.NotBefore = "attempting", time.Time{} a := in.Attempt r.Attempt = &a return nil } } // BindWakeRetrySession attaches the cold admission resulting from a first // reservation. It is required before that invocation may execute app code. func BindWakeRetrySession(incarnation, episode, attempt string, session uint64) func(*Claim) error { return func(c *Claim) error { if c.Holder == nil || c.Holder.Incarnation != incarnation || session == 0 || c.Holder.Session != session { return ErrWrongState } r := c.Holder.WakeRetry if r == nil || r.EpisodeID != episode || r.State != "attempting" || r.Attempt == nil || r.Attempt.ID != attempt || r.Attempt.Session != 0 { return ErrWrongState } r.Attempt.Session = session return nil } } type WakeRetryResult struct { HolderIncarnation string EpisodeID string Attempt WakeRetryAttempt Now time.Time } func sameWakeAttempt(a *WakeRetryAttempt, b WakeRetryAttempt) bool { return a != nil && *a == b } // FailWakeRetryAttempt settles the exact reserved attempt. A crash or // ambiguous result uses this transition too; the reservation is never refunded. func FailWakeRetryAttempt(in WakeRetryResult) func(*Claim) error { return func(c *Claim) error { if c.Holder == nil || c.Holder.Incarnation != in.HolderIncarnation || in.Now.IsZero() { return ErrWrongState } r := c.Holder.WakeRetry if r == nil || r.EpisodeID != in.EpisodeID || r.State != "attempting" || !sameWakeAttempt(r.Attempt, in.Attempt) { return ErrWrongState } r.Attempt = nil r.NotBefore = in.Now.Add(wakeRetryDelays[r.AttemptsReserved-1]) if r.AttemptsReserved == WakeRetryLimit { r.State = "failed" } else { r.State = "open" } return nil } } type WakeRetryHealth struct { WakeRetryResult ReadinessPassed bool } // ClearWakeRetryHealthy accepts only a readiness result for the exact live // reservation. The caller is responsible for performing the backend checks. func ClearWakeRetryHealthy(in WakeRetryHealth) func(*Claim) error { return func(c *Claim) error { if !in.ReadinessPassed || c.Holder == nil || c.Holder.Incarnation != in.HolderIncarnation || in.Attempt.Session == 0 || c.Holder.Session != in.Attempt.Session { return ErrWrongState } r := c.Holder.WakeRetry if r == nil || r.EpisodeID != in.EpisodeID || r.State != "attempting" || !sameWakeAttempt(r.Attempt, in.Attempt) { return ErrWrongState } c.Holder.WakeRetry = nil return nil } } type WakeRetryReset struct { HolderIncarnation string ObservedEpisode string NewEpisodeID string OperatorRequest string AttemptSettled bool } // ResetWakeRetry is the operator-authorized primitive. Authorization and any // required fencing happen outside this package and are asserted by the input. func ResetWakeRetry(in WakeRetryReset) func(*Claim) error { return func(c *Claim) error { if c.Holder == nil || c.Holder.Incarnation != in.HolderIncarnation || in.ObservedEpisode == "" || in.NewEpisodeID == "" || in.NewEpisodeID == in.ObservedEpisode || in.OperatorRequest == "" { return ErrWrongState } r := c.Holder.WakeRetry if r == nil || r.EpisodeID != in.ObservedEpisode || (r.State == "attempting" && !in.AttemptSettled) { return ErrWrongState } c.Holder.WakeRetry = &WakeRetry{EpisodeID: in.NewEpisodeID, State: "open", ResetRequest: in.OperatorRequest} return nil } }