package hostexec import ( "context" "crypto/rand" "encoding/hex" "errors" "sync" "time" ) type RootClientConfig struct { Project func(State) error Binding Binding Clock Clock Observe func(context.Context) (ExecutionRequest, error) Call func(context.Context, ExecutionRequest) (ExecutionResponse, error) CloseAdmission func() Cleanup func(context.Context) error } type RootClient struct { cfg RootClientConfig mu sync.Mutex gate chan struct{} request ExecutionRequest uncertain bool state State err error next int64 cancel context.CancelFunc done chan struct{} watchDone chan struct{} autoDone chan struct{} closeMu sync.Mutex cleaned, settled bool settleID string } func requestIdentity() (string, error) { var raw [24]byte if _, err := rand.Read(raw[:]); err != nil { return "", err } return hex.EncodeToString(raw[:]), nil } func OpenRootClient(ctx context.Context, cfg RootClientConfig) (*RootClient, error) { if cfg.Clock == nil || cfg.Call == nil || cfg.Observe == nil || cfg.CloseAdmission == nil || cfg.Cleanup == nil { return nil, ErrState } worker, cancel := context.WithCancel(ctx) c := &RootClient{cfg: cfg, gate: make(chan struct{}, 1), cancel: cancel, done: make(chan struct{}), watchDone: make(chan struct{}), autoDone: make(chan struct{})} if err := c.refresh(worker, true); err != nil { cancel() close(c.done) close(c.watchDone) close(c.autoDone) return c, err } go func() { defer close(c.watchDone) for { select { case <-worker.Done(): return case <-cfg.Clock.After(100 * time.Millisecond): if _, err := c.Ready(worker); err != nil { c.cancel() return } } } }() go func() { defer close(c.done) for { select { case <-worker.Done(): c.lose(worker.Err()) return case <-cfg.Clock.After(Renewal): if err := c.Refresh(worker); err != nil { c.lose(err) return } } } }() go func() { defer close(c.autoDone) <-worker.Done() cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer cancel() _ = c.close(cleanup) }() return c, nil } func (c *RootClient) lose(err error) { c.mu.Lock() if c.err == nil { c.err = errors.Join(ErrLost, err) } c.mu.Unlock() c.cancel() c.cfg.CloseAdmission() } func (c *RootClient) Refresh(ctx context.Context) error { return c.refresh(ctx, false) } func (c *RootClient) refresh(ctx context.Context, begin bool) error { select { case c.gate <- struct{}{}: case <-ctx.Done(): return ctx.Err() } defer func() { <-c.gate }() c.mu.Lock() prior, next, previousErr := c.state, c.next, c.err c.mu.Unlock() if previousErr != nil { return previousErr } _, start, err := c.cfg.Clock.Now() if err != nil { return err } if !begin && start >= prior.SafeUntil { c.lose(ErrLost) return ErrLost } q, err := c.cfg.Observe(ctx) if err != nil { c.lose(err) return err } if q.Host != c.cfg.Binding.Host || q.Incarnation != c.cfg.Binding.Incarnation || q.Op != c.cfg.Binding.Op { c.lose(ErrLost) return ErrLost } q.Schema = 1 q.ID, err = requestIdentity() if err != nil { return err } q.Operation = "executionView" q.Token = prior.ControllerID if begin { q.Operation = "executionBegin" } else if start >= next { q.Operation = "executionRenew" } c.mu.Lock() old := c.request c.mu.Unlock() if !begin && (old.Stamp != q.Stamp || old.Session != q.Session) { q.Operation = "executionRenew" } c.mu.Lock() c.request = q c.uncertain = true c.mu.Unlock() out, err := c.cfg.Call(ctx, q) if err != nil { c.lose(err) return err } _, now, clockErr := c.cfg.Clock.Now() if clockErr != nil || (!begin && now >= prior.SafeUntil) || out.Schema != 1 || out.Host != q.Host || out.Incarnation != q.Incarnation || out.Stamp != q.Stamp || out.Session != q.Session || out.Op != q.Op || out.BootID != c.cfg.Binding.BootID || out.Token == "" || out.Status != "active" || out.SafeUntil <= now || out.LeaseETag == "" || out.Expires.IsZero() { c.lose(ErrLost) return ErrLost } binding := c.cfg.Binding binding.ControllerID = out.Token c.mu.Lock() if c.err != nil { err := c.err c.mu.Unlock() return err } c.state = State{V: 1, Binding: binding, State: out.Status, SafeUntil: out.SafeUntil, Expires: out.Expires, LeaseETag: out.LeaseETag} c.request = q c.uncertain = false if q.Operation != "executionView" { c.next = start + int64(Renewal) } state := c.state c.mu.Unlock() if c.cfg.Project != nil { if err := c.cfg.Project(state); err != nil { c.lose(err) return err } } return nil } func (c *RootClient) Ready(ctx context.Context) (State, error) { if ctx.Err() != nil { return State{}, ctx.Err() } c.mu.Lock() state, err := c.state, c.err c.mu.Unlock() if err != nil { return state, err } _, now, err := c.cfg.Clock.Now() if err != nil || state.State != "active" || now >= state.SafeUntil { c.lose(ErrLost) return state, ErrLost } return state, nil } func (c *RootClient) Check(ctx context.Context, q ExecutionRequest) error { if _, err := c.Ready(ctx); err != nil { return err } c.mu.Lock() defer c.mu.Unlock() if c.err != nil { return c.err } old := c.request if c.uncertain || old.Host != q.Host || old.Incarnation != q.Incarnation || old.Stamp != q.Stamp || old.Session != q.Session || old.Op != q.Op { return ErrLost } return nil } func (c *RootClient) Close(ctx context.Context) error { err := c.close(ctx) select { case <-c.autoDone: return err case <-ctx.Done(): return errors.Join(err, ctx.Err()) } } func (c *RootClient) close(ctx context.Context) error { c.closeMu.Lock() defer c.closeMu.Unlock() if c.settled { return nil } c.lose(ErrLost) c.cancel() select { case <-c.done: case <-ctx.Done(): return ctx.Err() } select { case <-c.watchDone: case <-ctx.Done(): return ctx.Err() } select { case c.gate <- struct{}{}: case <-ctx.Done(): return ctx.Err() } defer func() { <-c.gate }() if !c.cleaned { if err := c.cfg.Cleanup(ctx); err != nil { return err } c.cleaned = true } c.mu.Lock() q, token, uncertain := c.request, c.state.ControllerID, c.uncertain c.mu.Unlock() if uncertain { if !q.Valid() { return ErrLost } out, err := c.cfg.Call(ctx, q) if err != nil { return err } if !executionReplyMatches(q, c.cfg.Binding.BootID, out) || out.Status != "active" { return ErrLost } token = out.Token c.mu.Lock() c.state.Binding = c.cfg.Binding c.state.ControllerID = token c.uncertain = false c.mu.Unlock() } if token == "" { return ErrLost } q.Token, q.Operation = token, "executionSettle" var err error if c.settleID == "" { c.settleID, err = requestIdentity() if err != nil { return err } } q.ID = c.settleID settle, err := c.cfg.Call(ctx, q) if err != nil { return err } if !executionReplyMatches(q, c.cfg.Binding.BootID, settle) || settle.Status != "settled" || settle.Token != token { return ErrLost } if c.cfg.Project != nil { c.mu.Lock() state := c.state c.mu.Unlock() state.State = "settled" if err := c.cfg.Project(state); err != nil { return err } } c.settled = true return nil } func executionReplyMatches(q ExecutionRequest, boot string, out ExecutionResponse) bool { return out.Schema == 1 && out.Host == q.Host && out.Incarnation == q.Incarnation && out.Stamp == q.Stamp && out.Session == q.Session && out.Op == q.Op && out.BootID == boot && out.Token != "" }