package activator import ( "context" "errors" "os/exec" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) type ServingRetries struct { BeforeReserve func(context.Context, registry.Route, hostclaim.Observed) error Claims release.Store Box, Enrollment, BootID, ControllerID string Now func() time.Time NewID func() string Inputs func(context.Context, registry.Route) (string, error) StopAttempt func(context.Context, hostclaim.Observed, hostclaim.WakeRetryAttempt) error ExactHealth func(context.Context, *exec.Cmd, hostclaim.WakeRetryAttempt) error } func (d ServingRetries) valid() bool { return d.Claims != nil && d.Box != "" && d.Enrollment != "" && d.BootID != "" && d.ControllerID != "" && d.Now != nil && d.NewID != nil && d.Inputs != nil && d.StopAttempt != nil && d.ExactHealth != nil } func (d ServingRetries) attempt(inputs string, session uint64) hostclaim.WakeRetryAttempt { return hostclaim.WakeRetryAttempt{ID: d.NewID(), Box: d.Box, Enrollment: d.Enrollment, BootID: d.BootID, ControllerID: d.ControllerID, Session: session, InvocationID: d.NewID(), LaunchInputs: inputs} } func retryOf(o hostclaim.Observed) *hostclaim.WakeRetry { if o.Claim == nil || o.Claim.Holder == nil { return nil } return o.Claim.Holder.WakeRetry } func (d ServingRetries) settle(ctx context.Context, expected hostclaim.Observed, writes ...func(hostclaim.Observed, hostclaim.Observed) error) error { r := retryOf(expected) if r == nil || r.State != "attempting" || r.Attempt == nil { return nil } o, err := hostclaim.Read(ctx, d.Claims, expected.Host) if err != nil { return err } current := retryOf(o) if o.Claim == nil || o.Claim.Holder == nil || o.Claim.Holder.Incarnation != expected.Claim.Holder.Incarnation || current == nil || current.EpisodeID != r.EpisodeID || current.Attempt == nil || *current.Attempt != *r.Attempt { return hostbox.ErrChanged } if err := d.StopAttempt(ctx, o, *r.Attempt); err != nil { return err } // A stop can change bookkeeping. Never use its earlier ETag for settlement. o, err = hostclaim.Read(ctx, d.Claims, expected.Host) if err != nil { return err } written, err := hostclaim.Update(ctx, d.Claims, o, hostclaim.FailWakeRetryAttempt(hostclaim.WakeRetryResult{HolderIncarnation: expected.Claim.Holder.Incarnation, EpisodeID: r.EpisodeID, Attempt: *r.Attempt, Now: d.Now()})) if err == nil { for _, write := range writes { if write != nil { if err := write(o, written); err != nil { return err } } } } return err } func (d ServingRetries) Wrap(base *ClaimRuntime) (*ClaimRuntime, error) { if !d.valid() || base == nil || base.Admit == nil || base.Prepare == nil || base.Prerequisites == nil || base.FailedAdmission == nil { return nil, hostwrite.ErrUnavailable } out := *base out.Prerequisites = func(ctx context.Context, rt registry.Route, o hostclaim.Observed, p StartPolicy) error { if o.Claim != nil && o.Claim.Op.Kind == hostclaim.KindServing && !p.Probe { if r := retryOf(o); r != nil { if r.State == "attempting" { if err := d.settle(ctx, o); err != nil { return err } return hostbox.ErrChanged } if r.State != "open" || d.Now().Before(r.NotBefore) { return hostclaim.ErrWrongState } } } if o.Claim != nil && o.Claim.Op.Kind == hostclaim.KindServing && !p.Probe && d.BeforeReserve != nil { if err := d.BeforeReserve(ctx, rt, o); err != nil { return err } } return base.Prerequisites(ctx, rt, o, p) } out.Admit = func(ctx context.Context, rt registry.Route, o hostclaim.Observed, p StartPolicy) (result hostclaim.Observed, resultErr error) { if o.Claim == nil || o.Claim.Op.Kind != hostclaim.KindServing || p.Probe { return base.Admit(ctx, rt, o, p) } cleanupObservation := o defer func() { if resultErr != nil { recovery, cancel := CleanupContext(ctx, 10*time.Second) defer cancel() resultErr = errors.Join(resultErr, base.FailedAdmission(recovery, cleanupObservation, resultErr)) } }() if d.BeforeReserve != nil { if err := d.BeforeReserve(ctx, rt, o); err != nil { return hostclaim.Observed{}, err } } inputs, err := d.Inputs(ctx, rt) if err != nil || inputs == "" { return hostclaim.Observed{}, errors.Join(err, hostwrite.ErrUnavailable) } episode := d.NewID() if r := retryOf(o); r != nil { episode = r.EpisodeID } attempt := d.attempt(inputs, 0) if d.BeforeReserve != nil { if err := d.BeforeReserve(ctx, rt, o); err != nil { return hostclaim.Observed{}, err } } reserved, err := hostclaim.Update(ctx, d.Claims, o, hostclaim.ReserveWakeRetryAttempt(hostclaim.WakeRetryReservation{HolderIncarnation: o.Claim.Holder.Incarnation, EpisodeID: episode, Attempt: attempt, Now: d.Now()})) if err != nil { return hostclaim.Observed{}, err } cleanupObservation = reserved admitted, err := base.Admit(ctx, rt, reserved, p) if err == nil && admitted.Claim != nil && admitted.Claim.Holder != nil { admitted, err = hostclaim.Update(ctx, d.Claims, admitted, hostclaim.BindWakeRetrySession(o.Claim.Holder.Incarnation, episode, attempt.ID, admitted.Claim.Holder.Session)) } else if err == nil { err = hostbox.ErrChanged } if err != nil { recovery, cancel := CleanupContext(ctx, 10*time.Second) defer cancel() // An ambiguous admission or bind is settled using the stored attempt, never refunded. fresh, readErr := hostclaim.Read(recovery, d.Claims, o.Host) if readErr == nil && fresh.Claim != nil && fresh.Claim.Holder != nil && fresh.Claim.Holder.Incarnation == o.Claim.Holder.Incarnation { r := retryOf(fresh) if r != nil && r.EpisodeID == episode && r.Attempt != nil && r.Attempt.ID == attempt.ID { readErr = d.settle(recovery, fresh) } else { readErr = hostbox.ErrChanged } } return hostclaim.Observed{}, errors.Join(err, readErr) } return admitted, nil } out.FailedAdmission = func(ctx context.Context, o hostclaim.Observed, cause error) error { err := base.FailedAdmission(ctx, o, cause) fresh, readErr := hostclaim.Read(ctx, d.Claims, o.Host) if readErr != nil { return errors.Join(err, readErr) } if o.Claim == nil || o.Claim.Holder == nil || fresh.Claim == nil || fresh.Claim.Holder == nil || o.Claim.Holder.Incarnation != fresh.Claim.Holder.Incarnation { return errors.Join(err, hostbox.ErrChanged) } expected, current := retryOf(o), retryOf(fresh) if expected == nil || current == nil || expected.EpisodeID != current.EpisodeID { return err } if current.Attempt != nil && (!sameAdmission(fresh, o) || current.Attempt.Box != d.Box || current.Attempt.Enrollment != d.Enrollment || current.Attempt.BootID != d.BootID || current.Attempt.ControllerID != d.ControllerID) { return errors.Join(err, hostbox.ErrChanged) } return errors.Join(err, d.settle(ctx, fresh)) } out.Prepare = func(ctx context.Context, rt registry.Route, o hostclaim.Observed, p StartPolicy) (ClaimStart, error) { s, err := base.Prepare(ctx, rt, o, p) if err != nil || p.Probe || o.Claim == nil || o.Claim.Op.Kind != hostclaim.KindServing { return s, err } r := retryOf(o) if r == nil || r.Attempt == nil || r.Attempt.Session == 0 || r.Attempt.Session != o.Claim.Holder.Session { return ClaimStart{}, hostbox.ErrChanged } episode, inc := r.EpisodeID, o.Claim.Holder.Incarnation attempt := *r.Attempt s.CurrentAttempt = func() hostclaim.WakeRetryAttempt { return attempt } s.Reserve = func(ctx context.Context, n int) error { if err := ctx.Err(); err != nil { return err } fresh, err := hostclaim.Read(ctx, d.Claims, o.Host) if err != nil { return err } if !sameAdmission(fresh, o) { return hostbox.ErrChanged } if n == 0 { rr := retryOf(fresh) if rr == nil || rr.EpisodeID != episode || rr.Attempt == nil || *rr.Attempt != attempt { return hostbox.ErrChanged } return nil } inputs, err := d.Inputs(ctx, rt) if err != nil { return err } if inputs != attempt.LaunchInputs { return hostbox.ErrChanged } next := d.attempt(inputs, o.Claim.Holder.Session) written, err := hostclaim.Update(ctx, d.Claims, fresh, hostclaim.ReserveWakeRetryAttempt(hostclaim.WakeRetryReservation{HolderIncarnation: inc, EpisodeID: episode, Attempt: next, Now: d.Now()})) if err == nil { attempt = next if s.RetryClaimWrite != nil { return s.RetryClaimWrite(fresh, written) } } return err } s.Failed = func(ctx context.Context, _ int) error { recovery, cancel := CleanupContext(ctx, 10*time.Second) defer cancel() fresh, err := hostclaim.Read(recovery, d.Claims, o.Host) if err != nil { return err } rr := retryOf(fresh) if rr == nil || rr.EpisodeID != episode || rr.Attempt == nil || *rr.Attempt != attempt { return hostbox.ErrChanged } return d.settle(recovery, fresh, s.RetryClaimWrite) } s.Healthy = func(ctx context.Context, _ int, cmd *exec.Cmd) error { if cmd == nil || cmd.Process == nil { return hostbox.ErrChanged } if err := d.ExactHealth(ctx, cmd, attempt); err != nil { return err } inputs, err := d.Inputs(ctx, rt) if err != nil { return err } if inputs != attempt.LaunchInputs { return hostbox.ErrChanged } fresh, err := hostclaim.Read(ctx, d.Claims, o.Host) if err != nil { return err } if !sameAdmission(fresh, o) { return hostbox.ErrChanged } _, err = hostclaim.Update(ctx, d.Claims, fresh, hostclaim.ClearWakeRetryHealthy(hostclaim.WakeRetryHealth{WakeRetryResult: hostclaim.WakeRetryResult{HolderIncarnation: inc, EpisodeID: episode, Attempt: attempt, Now: d.Now()}, ReadinessPassed: true})) return err } return s, nil } return &out, nil } // SettleAbandoned is the seed rerun's settlement boundary. Its caller holds the // host exclusion. It spends no new attempt and remains usable below disk floor. func (d ServingRetries) SettleAbandoned(ctx context.Context, expected hostclaim.Observed) (hostclaim.Observed, error) { if !d.valid() || ctx == nil || expected.Absent || expected.Claim == nil || expected.Claim.Holder == nil || expected.Claim.Op.Kind != hostclaim.KindServing || expected.Claim.Op.Exec != d.Box { return hostclaim.Observed{}, hostwrite.ErrUnavailable } fresh, err := hostclaim.Read(ctx, d.Claims, expected.Host) if err != nil { return hostclaim.Observed{}, err } if fresh.Absent || fresh.ETag != expected.ETag || !hostclaim.SameWriterIdentity(fresh, expected) { return hostclaim.Observed{}, hostbox.ErrChanged } if err := d.settle(ctx, fresh); err != nil { return hostclaim.Observed{}, err } return hostclaim.Read(ctx, d.Claims, expected.Host) }