package host import ( "context" "errors" "fmt" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // The source's recovery probe adapter (moves §7.8 source probing; claim // Plan 10 J2). It is the production recoveryProbe hook: movesrecover.go's // probing calls it once the record says probing and the claim is this // rollback's restoring op at probing with this box executing. // // One call is one probe-shaped claim wake, in this order: // // 1. the request is checked against the record (this rollback at probing, // nine attempts, the coverage and generation of its return; a declared // return, whose record cites the target's declared loss or fence, asks // no coverage, Q6) and the claim is read fresh (restoring/probing, this // op, this rollback, this box executing, the record's incarnation, a // clean return's src_gen, the target's loss the record cites); // 2. the inputs the probe will render are selected (the box's current // inputs, as the cut reads them) and recorded through req.Record, which // is provision.MoveRecordRecoveryInputs under the move and rollback ids, // legal only at probing. A clean return records here: it has no // configured phase. A dirty return records again, replacing what its // install recorded at configured, because this render is the probe's. // A failed record write stops the attempt: nothing is rendered or run; // 3. the activator is asked for a probe-shaped claim wake (StartPolicy // Probe, carrying exactly the recorded inputs). That wake is the claim's // Group D path unchanged: Prerequisites and admitsOperation admit a // recovery probe only at probing with the moves C1 owners bound; Admit, // the pre-launch record and Prepare bind OperationAttempts and the // role-cgroup probe readiness to the admitted session; the moves Inputs // owner selects and materializes (the held render) the recorded inputs, // and moves' policy admits only those (MovesClaimPolicy). // // The adapter writes nothing besides req.Record. The restore, install and // evidence remain the moves owners'; the nine-attempt budget is journalled // by moves' OperationAttempts, which refuses a start past it with // ErrRecoveryProbeExhausted. // ClaimRecoveryProbeWake is the activator's recovery probe entry: one // claim wake of rt under p (p.Probe set), through ClaimRuntime's // Prerequisites, Admit and Prepare and the activator's own start loop. It // returns nil once a start was healthy (OperationAttempts.Healthy accepted // it). Production binds Service.RecoveryProbe, which retains the process // in the activator and refuses missing claim-runtime dependencies. type ClaimRecoveryProbeWake func(ctx context.Context, rt registry.Route, p activator.StartPolicy) error // ErrRecoveryProbeExhausted is the moves attempt journal's refusal once the // recovery's nine starts are spent. The recovery stops for an operator at // probing (StuckRecoveryProbeAttempts). var ErrRecoveryProbeExhausted = errors.New("moves: the recovery probe's attempts are spent") // errRecoveryProbeRequest is a request that does not match its record: a // caller bug, never retried into a wake. var errRecoveryProbeRequest = errors.New("moves: the recovery probe request does not match its record") // errRecoveryInputsNoRelease: the selected inputs name no release, which the // record transition would refuse; the adapter refuses first. var errRecoveryInputsNoRelease = errors.New("moves: the recovery probe's inputs name no release") // recoveryProbeStartsPerWake is the starts one probe wake may run. The // activator allows at most five per wake; the recovery's nine are a budget // across wakes, journalled by moves. const recoveryProbeStartsPerWake = 3 // recoveryProbeWindow is the probe's readiness window from its first // app start (§7.7's 30 s). const recoveryProbeWindow = 30 * time.Second // recoveryProbeAdapter is the source's recovery probe. Claims is the // captured CARLOS_HOST_CLAIM; Route reads this box's registry row; Current // selects the inputs the probe will render; Wake is the activator's entry. type recoveryProbeAdapter struct { Box string Claims bool Route func(host string) (*registry.Route, error) Current func(ctx context.Context, s release.Store, rec provision.Record) (provision.MoveInputs, error) Wake ClaimRecoveryProbeWake Ready func() bool } // armRecoveryProbe binds the production recoveryProbe hook: the adapter // over this worker's configuration, read when the probe runs. wake is the // activator's recovery probe entry; tests may leave it nil to prove refusal. func armRecoveryProbe(cfg *moveCfg, w *movesWorker, wake ClaimRecoveryProbeWake, ready func() bool) { cfg.recoveryProbe = func(ctx context.Context, s release.Store, rec provision.Record, req recoveryProbeRequest) (recoveryProbeOutcome, error) { c := w.cfg a := recoveryProbeAdapter{Box: c.Box, Claims: c.claimRecords, Route: c.Route, Current: w.recoveryInputs, Wake: wake, Ready: ready} return a.probe(ctx, s, rec, req) } } // probe runs one recovery probe wake for rec (see the file comment). func (a recoveryProbeAdapter) probe(ctx context.Context, s release.Store, rec provision.Record, req recoveryProbeRequest) (recoveryProbeOutcome, error) { var missing []string if !a.Claims { missing = append(missing, "CARLOS_HOST_CLAIM Group D recovery admission") } if a.Wake == nil || (a.Ready != nil && !a.Ready()) { missing = append(missing, "activator recovery probe entry") } if a.Box == "" || a.Route == nil || a.Current == nil { missing = append(missing, "moves recovery probe route and inputs") } if len(missing) > 0 { return recoveryProbeWaiting, &ClaimDependencyError{Contracts: missing} } if err := recoveryProbeMatches(rec, req); err != nil { return recoveryProbeWaiting, err } obs, err := hostclaim.Read(ctx, s, rec.Host) if err != nil { return recoveryProbeWaiting, fmt.Errorf("read the host claim: %w", err) } if err := a.admits(obs, rec, req); err != nil { return recoveryProbeWaiting, err } rt, err := a.Route(rec.Host) if err != nil { return recoveryProbeWaiting, fmt.Errorf("read the registry row: %w", err) } if rt == nil || rt.Host != rec.Host { return recoveryProbeWaiting, fmt.Errorf("%w: %s has no registry row on this box", hostbox.ErrChanged, rec.Host) } in, err := a.Current(ctx, s, rec) if err != nil { return recoveryProbeWaiting, fmt.Errorf("select the recovery inputs: %w", err) } if in.Release == "" { return recoveryProbeWaiting, errRecoveryInputsNoRelease } // The record precedes every render and launch of this wake. if err := req.Record(ctx, in); err != nil { return recoveryProbeWaiting, fmt.Errorf("record the recovery inputs: %w", err) } recorded := in recorded.BundleETags = make(map[string]string, len(in.BundleETags)) for k, v := range in.BundleETags { recorded.BundleETags[k] = v } err = a.Wake(ctx, *rt, activator.StartPolicy{RecoveryFence: moveFenceGeneration(rec.Move), MaxStarts: recoveryProbeStartsPerWake, Window: recoveryProbeWindow, Probe: true, Inputs: &recorded}) switch { case err == nil: return recoveryProbeAnswered, nil case errors.Is(err, ErrRecoveryProbeExhausted): return recoveryProbeExhausted, nil case errors.Is(err, moves.ErrAttemptOver), errors.Is(err, hostbox.ErrChanged): // This wake's window lapsed or its admission no longer holds: the // next pass probes again, and only the probe. return recoveryProbeWaiting, nil } return recoveryProbeWaiting, fmt.Errorf("the recovery probe wake: %w", err) } // recoveryProbeMatches checks the request against its record: this // rollback, rolling back with the source at probing, the nine-attempt // budget and the coverage and generation of its return. func recoveryProbeMatches(rec provision.Record, req recoveryProbeRequest) error { m := rec.Move if m != nil && m.ID != "" && m.State == provision.MoveReverting { if m.ReopenedAt != nil || req.RollbackID != "" || req.Coverage != "" || req.Generation != "" || !req.Clean || req.Declared || req.Attempts != recoveryProbeAttempts || req.Record == nil { return errRecoveryProbeRequest } return nil } if m == nil || m.ID == "" || m.State != provision.MoveRollingBack || m.Rollback == nil || m.Rollback.ID != req.RollbackID || m.Rollback.SourcePhase != provision.RbSourceProbing { return fmt.Errorf("%w: not rollback %s at probing", errRecoveryProbeRequest, req.RollbackID) } if req.Record == nil || req.Attempts != recoveryProbeAttempts || req.Clean == m.Rollback.Dirty { return fmt.Errorf("%w: budget %d, clean %v", errRecoveryProbeRequest, req.Attempts, req.Clean) } // The declared kind (Q6): a return whose record cites a declared loss // or a fence of the target is clean and asks no coverage. declared := m.Rollback.LossRef != "" switch { case req.Declared != declared: return fmt.Errorf("%w: the request's declared return is not the record's", errRecoveryProbeRequest) // SourceGeneration may be "" (whole-branch fixes, item 4): the claim's // legacy encoding of a source whose sessions predate admission records, // which recoverRun.probing passes through from the journal and the // owners normalise as hostclaim.RootGeneration; admits holds it to the // claim's. case declared && (!req.Clean || req.Coverage != "" || req.Generation != ""): return fmt.Errorf("%w: a declared return admits a fresh generation restored from src_gen, uncovered", errRecoveryProbeRequest) case declared: case req.Clean: if req.Coverage != recoveryCoverageClean || req.Generation != "" { return fmt.Errorf("%w: a clean return admits a fresh generation restored from src_gen", errRecoveryProbeRequest) } case req.Coverage != recoveryCoverageSealed || req.Generation != req.RollbackID: return fmt.Errorf("%w: a dirty return admits its sealed return generation", errRecoveryProbeRequest) } return nil } // admits checks the fresh claim: this move's restoring op for this // rollback, at probing, executed by this box under the record's // incarnation, naming the request's src_gen. Anything else waits. func (a recoveryProbeAdapter) admits(obs hostclaim.Observed, rec provision.Record, req recoveryProbeRequest) error { c := obs.Claim if obs.Absent || c == nil || c.Holder == nil || c.Op.ID != rec.Move.ID || c.Op.Phase != hostclaim.PhaseProbing || c.Op.Exec != a.Box || c.Holder.Incarnation != rec.Move.Incarnation { return hostbox.ErrChanged } if rec.Move.State == provision.MoveReverting { if c.Op.Kind != hostclaim.KindReverting || c.Op.Journal[hostclaim.JournalRevPhase] != hostclaim.PhaseProbing || c.Holder.Generation != req.SourceGeneration { return hostbox.ErrChanged } return nil } if c.Op.Kind != hostclaim.KindRestoring || c.Op.Intent["rollback"] != req.RollbackID { return hostbox.ErrChanged } // The fresh claim must still agree with the record on the target's // loss, and with the request's kind (Q6). declared, err := recoveryDeclared(c, rec.Move.Rollback.LossRef) if err != nil { return err } if declared != req.Declared { return fmt.Errorf("%w: the claim's loss of the target is not the request's", hostbox.ErrChanged) } // The claim's src_gen and the request's, normalised as the owners // normalise them (selectedGeneration: "" is the legacy root), must agree. if selectedGeneration(c.Op.Journal[hostclaim.JournalSrcGen]) != selectedGeneration(req.SourceGeneration) { return fmt.Errorf("%w: the claim's src_gen is not the request's", hostbox.ErrChanged) } return nil }