package host import ( "context" "encoding/json" "fmt" "os/exec" "strconv" "sync" "syscall" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) // movesProbeAttempts is the moves OperationAttempts implementation for a // landing or recovery probe (moves Phase 1 Plan 3 Task 5b; claim Plan 9 Group D, D2; // rulings H2, H10). The claim binds its methods into the admitted probe's // activator.ClaimStart; this type journals, counts and settles the probe's // starts in the box-local op journal (moves.Journal, /.moves// // /), and writes no record and no claim. // // One record attempt is one admission. The landing (Task 6) counts it on the // record (provision.MoveProbeAttempt, Landing.Attempts = k) before it asks the // activator for the probe; every method here is bound to that k, refused // unless 1 <= k <= moves.AttemptsFor("landing", admitted), where admitted is // what T11 admitted for this move (Admitted, H10). Within an attempt the // activator makes up to five starts (startAdmitted). // // Steps it journals, per record attempt k: // // - probe_opened_: the first app start's time, RFC 3339. OpenProbe // writes it once and returns it to every later start, a restarted // controller included, so the 30 s window is the attempt's and never // restarts. // - probe_start_: the current start's reservation (probeStart): the // admitted session, the start's number n and the role launcher's // WakeRetryAttempt, and its state, reserved, failed or healthy. // - probe_session_: the record attempt k the admitted session is // bound to, written by its first Reserve(0) or OpenProbe (fix round 1, // finding 5). A session serves one record attempt: once the record's // index moves past k every method refuses and CurrentAttempt answers // the zero attempt, so a new record attempt needs a new cold admission. // // Every method refuses (hostbox.ErrChanged) once the claim no longer names // the admitted writer (op id, kind, session, stamp, generation; phase // probing) or the record no longer names this landing at probing, unmarked, // at attempt k. A landing wake after the commit (Plan 3 Task 7) is served // too: its starts are journalled per admitted session (probe_start_committed- // ), bound to the claim at landing/committed and a committed record, // with no record attempt, no window and no settle. type movesProbeAttempts struct { Claims release.Store Records MovesRecordReader DataDir string // Admitted is the number of probe attempts T11 admitted for the move // (H10: the target's report, or the record once MoveAdmit carries it). Admitted func(context.Context, *provision.Record) (int, error) NewID func() string Now func() time.Time // Settle is the wait before the second readiness request; zero is // moves.SettleSeconds. Ready bounds that request; zero is // moves.ProbeSeconds. Settle, Ready time.Duration // RecoveryAttempt, when set, is told of each recovery probe start this // journal counts: the record, its recovery retry epoch and the start's // number (the move log's probe_attempt--, Plan 6 Task 3; // movesWorker.recoveryAttemptLogger, set by movesWorker.probeAttempts). RecoveryAttempt func(ctx context.Context, rec provision.Record, epoch uint64, n int) mu sync.Mutex current map[string]boundAttempt } // boundAttempt is a session's last reservation and the record attempt the // session is bound to. type boundAttempt struct { k int key string attempt hostclaim.WakeRetryAttempt } var _ OperationAttempts = (*movesProbeAttempts)(nil) // Journal step names and start states. const ( probeStepOpened = "probe_opened_" probeStepStart = "probe_start_" probeStepSession = "probe_session_" probeReserved = "reserved" probeFailed = "failed" probeHealthy = "healthy" ) // probeStart is a probe_start_ step's value. type probeStart struct { Session uint64 `json:"session"` N int `json:"n"` Attempt hostclaim.WakeRetryAttempt `json:"attempt"` State string `json:"state"` } func (p *movesProbeAttempts) valid() bool { return p != nil && p.Claims != nil && p.Records != nil && p.DataDir != "" && p.Admitted != nil && p.NewID != nil && p.Now != nil } // bound returns the landing record attempt, zero for a committed wake, // or -1 for a recovery probe. Every case validates the admitted writer. // Recovery also returns the budget key from the same record read. func (p *movesProbeAttempts) bound(ctx context.Context, s OperationStart) (int, string, error) { k, key, _, err := p.boundRecord(ctx, s) return k, key, err } // boundRecord is bound, with the record a recovery probe was bound against // (nil otherwise): Reserve logs a recovery start from it. func (p *movesProbeAttempts) boundRecord(ctx context.Context, s OperationStart) (int, string, *provision.Record, error) { if !p.valid() { return 0, "", nil, hostwrite.ErrUnavailable } a := s.Admitted if a.Claim != nil && claimRecoveryKind(a.Claim.Op.Kind) { key, rec, err := p.boundRecovery(ctx, s) return -1, key, rec, err } k, key, err := p.boundLanding(ctx, s) return k, key, nil, err } // boundLanding is bound for a landing's (or a committed wake's) probe. func (p *movesProbeAttempts) boundLanding(ctx context.Context, s OperationStart) (int, string, error) { a := s.Admitted if a.Claim == nil || a.Claim.Holder == nil || a.Claim.Op.Kind != hostclaim.KindLanding { return 0, "", hostbox.ErrUnsupported } if !s.Policy.Probe { return 0, "", p.boundCommitted(ctx, s) } h := a.Claim.Holder if s.Owner.Host != a.Host || s.Owner.Session != h.Session || s.Owner.Stamp != h.Stamp || s.Owner.Incarnation != h.Incarnation { return 0, "", hostbox.ErrChanged } fresh, err := hostclaim.Read(ctx, p.Claims, a.Host) if err != nil { return 0, "", err } if !claimSameWriter(a, fresh, hostclaim.SessionContinues) || fresh.Claim.Op.Phase != hostclaim.PhaseLandingProbing { return 0, "", hostbox.ErrChanged } rec, err := p.Records(ctx, h.Account, h.App, a.Host) if err != nil { return 0, "", err } if rec == nil || rec.Host != a.Host || rec.Move == nil { return 0, "", hostbox.ErrChanged } m := rec.Move if m.ID != a.Claim.Op.ID || m.Incarnation != h.Incarnation || m.State != provision.MoveLanding || m.LandingPhase != provision.LandProbing || m.Reason != "" || m.Landing == nil { return 0, "", hostbox.ErrChanged } admitted, err := p.Admitted(ctx, rec) if err != nil { return 0, "", err } k := m.Landing.Attempts if k < 1 || k > moves.AttemptsFor(moves.ProbeLanding, admitted) { return 0, "", hostbox.ErrChanged } return k, "", nil } // boundCommitted is bound for a committed wake: the claim still names the // admitted writer at landing/committed, and the record is this move, // committed. Its starts are journalled per session (startKey). func (p *movesProbeAttempts) boundCommitted(ctx context.Context, s OperationStart) error { a := s.Admitted h := a.Claim.Holder if a.Claim.Op.Phase != hostclaim.PhaseLandingCommitted || s.Owner.Host != a.Host || s.Owner.Session != h.Session || s.Owner.Stamp != h.Stamp || s.Owner.Incarnation != h.Incarnation { return hostbox.ErrChanged } fresh, err := hostclaim.Read(ctx, p.Claims, a.Host) if err != nil { return err } if !claimSameWriter(a, fresh, hostclaim.SessionContinues) || fresh.Claim.Op.Phase != hostclaim.PhaseLandingCommitted { return hostbox.ErrChanged } rec, err := p.Records(ctx, h.Account, h.App, a.Host) if err != nil { return err } if rec == nil || rec.Host != a.Host || rec.Move == nil || rec.Move.ID != a.Claim.Op.ID || rec.Move.Incarnation != h.Incarnation || rec.Move.State != provision.MoveCommitted { return hostbox.ErrChanged } return nil } // startKey names a start's journal step: the record attempt k for a probe, // the admitted session for a committed wake (k 0), or the recovery and // admitted session for a recovery probe (k -1). func startKey(s OperationStart, k int, recoveryKey string) string { if k < 0 { return recoveryKey + "-" + strconv.FormatUint(s.Owner.Session, 10) } if k == 0 { return "committed-" + strconv.FormatUint(s.Owner.Session, 10) } return strconv.Itoa(k) } func (p *movesProbeAttempts) journal(s OperationStart) (*moves.Journal, error) { return moves.ReadJournal(p.DataDir, s.Admitted.Host, s.Admitted.Claim.Op.ID) } func probeKey(s OperationStart) string { return s.Admitted.Host + "\x00" + strconv.FormatUint(s.Owner.Session, 10) } // bindSession checks the session's journalled record attempt against k. // Absent, it is written into j when write is set (the caller writes j) and // refused otherwise. func bindSession(j *moves.Journal, s OperationStart, k int, write bool) error { if k <= 0 { return nil // committed wakes and recoveries have no landing record attempt } step := probeStepSession + strconv.FormatUint(s.Owner.Session, 10) raw, ok := j.Steps[step] switch { case ok && raw == strconv.Itoa(k): return nil case ok: return fmt.Errorf("%w: session %d is bound to record attempt %s, not %d", hostbox.ErrChanged, s.Owner.Session, raw, k) case !write: return fmt.Errorf("%w: session %d is bound to no record attempt", hostbox.ErrChanged, s.Owner.Session) } j.Mark(step, strconv.Itoa(k)) return nil } // OpenProbe journals the attempt's first start time, once. func (p *movesProbeAttempts) OpenProbe(ctx context.Context, s OperationStart, proposed time.Time) (time.Time, error) { k, recoveryKey, err := p.bound(ctx, s) if err != nil { return time.Time{}, err } if k == 0 { return time.Time{}, hostbox.ErrUnsupported } if proposed.IsZero() { return time.Time{}, hostwrite.ErrUnavailable } p.mu.Lock() defer p.mu.Unlock() j, err := p.journal(s) if err != nil { return time.Time{}, err } if err := bindSession(j, s, k, true); err != nil { return time.Time{}, err } step := probeStepOpened + startKey(s, k, recoveryKey) if raw, ok := j.Steps[step]; ok { if err := j.Commit(p.DataDir); err != nil { return time.Time{}, err } opened, err := time.Parse(time.RFC3339Nano, raw) if err != nil { return time.Time{}, fmt.Errorf("moves journal %s/%s: %s: %w", s.Admitted.Host, s.Admitted.Claim.Op.ID, step, err) } return opened, nil } opened := proposed.UTC() j.Mark(step, opened.Format(time.RFC3339Nano)) if err := j.Commit(p.DataDir); err != nil { return time.Time{}, err } return opened, nil } // readStart returns attempt k's journalled start, or false. func readStart(j *moves.Journal, key string) (probeStart, bool, error) { raw, ok := j.Steps[probeStepStart+key] if !ok { return probeStart{}, false, nil } var st probeStart if err := json.Unmarshal([]byte(raw), &st); err != nil { return probeStart{}, false, fmt.Errorf("moves journal %s/%s: %s%s: %w", j.Host, j.Op, probeStepStart, key, err) } return st, true, nil } func writeStart(dataDir string, j *moves.Journal, key string, st probeStart) error { raw, err := json.Marshal(st) if err != nil { return err } j.Mark(probeStepStart+key, string(raw)) return j.Commit(dataDir) } // sameStartOwner reports whether a journalled reservation is the one this // admission's owner made for these launch inputs. func sameStartOwner(a hostclaim.WakeRetryAttempt, s OperationStart, digest string) bool { o := s.Owner return a.ID != "" && a.InvocationID != "" && a.Session == o.Session && a.Box == o.Box && a.Enrollment == o.Enrollment && a.BootID == o.BootID && a.ControllerID == o.ControllerID && a.LaunchInputs == digest } // Reserve journals start n of this admission before it runs. Start 0 is // reserved by the first Reserve(0) of the admission and validated by every // later one; start n > 0 only after start n-1 was settled failed. func (p *movesProbeAttempts) Reserve(ctx context.Context, s OperationStart, n int) (retErr error) { k, recoveryKey, bindRec, err := p.boundRecord(ctx, s) if err != nil { return err } digest, err := ClaimLaunchDigest(s.Inputs) if err != nil { return err } p.mu.Lock() defer p.mu.Unlock() j, err := p.journal(s) if err != nil { return err } if err := bindSession(j, s, k, n == 0); err != nil { return err } st, ok, err := readStart(j, startKey(s, k, recoveryKey)) if err != nil { return err } same := ok && st.Session == s.Owner.Session switch { case n < 0: return hostbox.ErrChanged case n == 0 && same: if st.N != 0 || st.State != probeReserved || !sameStartOwner(st.Attempt, s, digest) { return hostbox.ErrChanged } case n == 0: // The admission's first start; an earlier admission's journalled // start, under another session, is over. st = probeStart{Session: s.Owner.Session, N: 0, State: probeReserved} default: if !same || st.N != n-1 || st.State != probeFailed || st.Attempt.LaunchInputs != digest { return hostbox.ErrChanged } st = probeStart{Session: s.Owner.Session, N: n, State: probeReserved} } if _, bound := j.Steps[probeStepSession+strconv.FormatUint(s.Owner.Session, 10)]; bound && st.Attempt.ID != "" { // Reserve(0) validated an existing reservation; the binding it may // have just written is made durable. if err := j.Commit(p.DataDir); err != nil { return err } } if st.Attempt.ID == "" { counted := 0 if k < 0 { count, err := recoveryAttemptCount(j, recoveryKey) if err != nil { return err } if count >= recoveryProbeAttempts { return ErrRecoveryProbeExhausted } counted = count + 1 j.Mark(recoveryKey, strconv.Itoa(counted)) } defer func() { if counted > 0 && retErr == nil { p.recoveryAttempted(ctx, bindRec, counted) } }() o := s.Owner st.Attempt = hostclaim.WakeRetryAttempt{ID: p.NewID(), Box: o.Box, Enrollment: o.Enrollment, BootID: o.BootID, ControllerID: o.ControllerID, Session: o.Session, InvocationID: p.NewID(), LaunchInputs: digest} if st.Attempt.ID == "" || st.Attempt.InvocationID == "" || st.Attempt.ID == st.Attempt.InvocationID { return hostwrite.ErrUnavailable } if err := writeStart(p.DataDir, j, startKey(s, k, recoveryKey), st); err != nil { return err } } if p.current == nil { p.current = map[string]boundAttempt{} } p.current[probeKey(s)] = boundAttempt{k: k, key: startKey(s, k, recoveryKey), attempt: st.Attempt} return nil } // CurrentAttempt is the reservation the last Reserve journalled for this // admission; the zero attempt, which the role launcher refuses, before any. // It reads the record again (the interface gives it no context, so its own // bounded one): once the record's index is past the bound attempt it answers // the zero attempt. func (p *movesProbeAttempts) CurrentAttempt(s OperationStart) hostclaim.WakeRetryAttempt { if p == nil { return hostclaim.WakeRetryAttempt{} } p.mu.Lock() cur, ok := p.current[probeKey(s)] p.mu.Unlock() if !ok { return hostclaim.WakeRetryAttempt{} } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if k, recoveryKey, err := p.bound(ctx, s); err != nil || k != cur.k || startKey(s, k, recoveryKey) != cur.key { return hostclaim.WakeRetryAttempt{} } return cur.attempt } // Failed settles start n, already stopped and drained, as failed. func (p *movesProbeAttempts) Failed(ctx context.Context, s OperationStart, n int) error { recovery, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() return p.settle(recovery, s, n, probeFailed) } func (p *movesProbeAttempts) settle(ctx context.Context, s OperationStart, n int, state string) error { k, recoveryKey, err := p.bound(ctx, s) if err != nil { return err } p.mu.Lock() defer p.mu.Unlock() j, err := p.journal(s) if err != nil { return err } if err := bindSession(j, s, k, false); err != nil { return err } st, ok, err := readStart(j, startKey(s, k, recoveryKey)) if err != nil { return err } cur := p.current[probeKey(s)] if !ok || st.Session != s.Owner.Session || st.N != n || st.State != probeReserved || cur.k != k || st.Attempt != cur.attempt { return hostbox.ErrChanged } st.State = state if err := writeStart(p.DataDir, j, startKey(s, k, recoveryKey), st); err != nil { return err } if k < 0 && state == probeFailed { count, err := recoveryAttemptCount(j, recoveryKey) if err != nil { return err } if count >= recoveryProbeAttempts { return ErrRecoveryProbeExhausted } } return nil } // Healthy accepts start n after readiness, replication and publication: // the 20 s settle with the process alive, then the probe's own readiness // check again (again, the claim's ClaimProbeReadiness against the started // role), with the process still alive after it (moves ยง7.7). func (p *movesProbeAttempts) Healthy(ctx context.Context, s OperationStart, n int, cmd *exec.Cmd, again func(context.Context) error) error { k, _, err := p.bound(ctx, s) if err != nil { return err } p.mu.Lock() j, err := p.journal(s) if err == nil { err = bindSession(j, s, k, false) } p.mu.Unlock() if err != nil { return err } if k == 0 { // A committed wake: serving's readiness has passed, with // replication and publication; there is no probe settle. return p.settle(ctx, s, n, probeHealthy) } if again == nil || !probeAlive(cmd) { return hostbox.ErrChanged } settle, ready := p.Settle, p.Ready if settle <= 0 { settle = time.Duration(moves.SettleSeconds * float64(time.Second)) } if ready <= 0 { ready = time.Duration(moves.ProbeSeconds * float64(time.Second)) } timer := time.NewTimer(settle) select { case <-ctx.Done(): timer.Stop() return ctx.Err() case <-timer.C: } if !probeAlive(cmd) { return fmt.Errorf("%w: the app exited during the settle", hostbox.ErrChanged) } asked, cancel := context.WithTimeout(ctx, ready) err = again(asked) cancel() if err != nil { return err } if !probeAlive(cmd) { return fmt.Errorf("%w: the app exited after the second request", hostbox.ErrChanged) } return p.settle(ctx, s, n, probeHealthy) } // probeAlive reports whether cmd's process has not exited. Signal 0 checks // existence without delivering anything; once the activator's Wait has // reaped the process it reports os.ErrProcessDone. func probeAlive(cmd *exec.Cmd) bool { return cmd != nil && cmd.Process != nil && cmd.Process.Signal(syscall.Signal(0)) == nil } // RecoveryProbeAttemptsReady is checked before the adapter records inputs. func (p *movesProbeAttempts) RecoveryProbeAttemptsReady() bool { return p.valid() } func (p *movesProbeAttempts) boundRecovery(ctx context.Context, s OperationStart) (string, *provision.Record, error) { a := s.Admitted if a.Claim.Holder == nil || !s.Policy.Probe { return "", nil, hostbox.ErrChanged } h := a.Claim.Holder if s.Owner.Host != a.Host || s.Owner.Incarnation != h.Incarnation || s.Owner.Stamp != h.Stamp || s.Owner.Session != h.Session || s.Owner.Box != a.Claim.Op.Exec { return "", nil, hostbox.ErrChanged } fresh, err := hostclaim.Read(ctx, p.Claims, a.Host) if err != nil { return "", nil, err } if !claimSameWriter(a, fresh, hostclaim.SessionContinues) || fresh.Claim.Op.Phase != hostclaim.PhaseProbing || fresh.Claim.Op.Exec != s.Owner.Box { return "", nil, hostbox.ErrChanged } rec, err := p.Records(ctx, h.Account, h.App, a.Host) if err != nil { return "", nil, err } if rec == nil || rec.Host != a.Host || rec.Move == nil || rec.Move.ID != a.Claim.Op.ID || rec.Move.Incarnation != h.Incarnation { return "", nil, hostbox.ErrChanged } m := rec.Move if a.Claim.Op.Kind == hostclaim.KindReverting { if m.State != provision.MoveReverting || m.ReopenedAt != nil || fresh.Claim.Op.Journal[hostclaim.JournalRevPhase] != hostclaim.PhaseProbing { return "", nil, hostbox.ErrChanged } } else if m.State != provision.MoveRollingBack || m.Rollback == nil || m.Rollback.ID == "" || m.Rollback.ID != a.Claim.Op.Intent["rollback"] || m.Rollback.ID != fresh.Claim.Op.Intent["rollback"] || m.Rollback.SourcePhase != provision.RbSourceProbing { return "", nil, hostbox.ErrChanged } return recoveryAttemptKey(s, m.RecoveryRetryEpoch), rec, nil } // The move has its own journal directory. JSON preserves the rollback and // incarnation boundaries; sessions share this counter across controller restarts. func recoveryAttemptKey(s OperationStart, retryEpoch uint64) string { c := s.Admitted.Claim rollback := c.Op.ID if c.Op.Kind == hostclaim.KindRestoring { rollback = c.Op.Intent["rollback"] } parts := []string{c.Op.ID, rollback, c.Holder.Incarnation} // Keep epoch zero's key so an upgrade preserves existing counts. if retryEpoch != 0 { parts = append(parts, strconv.FormatUint(retryEpoch, 10)) } raw, _ := json.Marshal(parts) return "recovery_attempts_" + string(raw) } func recoveryAttemptCount(j *moves.Journal, key string) (int, error) { raw, ok := j.Steps[key] if !ok { return 0, nil } n, err := strconv.Atoi(raw) if err != nil || n < 0 || n > recoveryProbeAttempts { return 0, hostbox.ErrChanged } return n, nil } // recoveryAttempted tells RecoveryAttempt of start n of a recovery probe, // with the record the start was bound against (its retry epoch): the read // the binding already made, so a counted start is never lost to a second // read. The hook's own failed write goes to the outbox. func (p *movesProbeAttempts) recoveryAttempted(ctx context.Context, rec *provision.Record, n int) { if p.RecoveryAttempt == nil || rec == nil || rec.Move == nil { return } p.RecoveryAttempt(ctx, *rec, rec.Move.RecoveryRetryEpoch, n) } // probeAttempts is the attempt journal a box builds for its probe wakes // (Plan 3 Task 5), over the deployment store s, its records and newID, with // this worker's data directory and clock. Its RecoveryAttempt is the // worker's move log hook (recoveryAttemptLogger), so every recovery probe // start it counts is logged (fix round 2, ruling 2). Production's journal // is the registered wake source's (movesWakeSources.Open, Plan 5 Task 9b), // built the same way before this worker is armed; it reaches this worker's // hook through armedMovesWorker (movesWorkerRecoveryAttempt). func (w *movesWorker) probeAttempts(s release.Store, records MovesRecordReader, newID func() string) *movesProbeAttempts { return &movesProbeAttempts{Claims: s, Records: records, DataDir: w.cfg.DataDir, Admitted: MovesRecordAdmittedAttempts, NewID: newID, Now: w.cfg.Now, RecoveryAttempt: w.recoveryAttemptLogger(s)} }