package hostops import ( "context" "errors" "os" "slices" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) // These callbacks are authority providers, not optional best-effort hints. // Production construction waits for the decisions named in this plan. type Authority interface { // SelectExecutor chooses the new host's executor once, before the // create CAS (claim P5-D2, region HA ยง5.2). For a non-production // authority, selected false means no choice at create: Attempt runs // today's BeginCreate and Executor names the box at completion. // Production is ExecutorChooser.Choose behind CARLOS_REGION_HA, // consulted inside StartCreate's own selection; with it off, claim // Plan 5's selectExecutor chooses as before. SelectExecutor(context.Context, hostclaim.Observed, Record) (sel hostclaim.ExecutorSelection, selected bool, err error) Create(context.Context, hostclaim.Observed, Record, string) error Executor(context.Context, hostclaim.Observed, Record) (string, error) StartDelete(context.Context, hostclaim.Observed, Record) error FirstPhase(context.Context, hostclaim.Observed) error Inventory(context.Context, hostclaim.Observed) ([]string, []string, error) Adoption(context.Context, Record) (region, box string, otherRegions, otherBoxes []string, err error) PurgeRetirement(context.Context, hostclaim.Observed) error } type Controller struct { Store release.Store // Control is the admission engine's typed writer (Task 7). nil in the // C5-only slice; every admission/journal transition requires it. Control *ControlWriter Authority Authority Now func() time.Time NewID func() string } func (c Controller) ready() error { if c.Store == nil || c.Authority == nil || c.Now == nil || c.NewID == nil { return ErrContract } return nil } // Attempt consumes the claim observation taken BEFORE app registration. // Create's caller owns registration renewal/removal. A failed CAS is never retried here. func (c Controller) Attempt(ctx context.Context, before hostclaim.Observed, r Record, id, by string) (hostclaim.Observed, error) { if err := c.ready(); err != nil { return hostclaim.Observed{}, err } if before.Host != r.Holder.Host || r.Holder.CreatedAt.IsZero() || id == "" { return hostclaim.Observed{}, ErrIdentity } // With region HA on, the executor is selected once, before the CAS, // and recorded in it; off, the create is today's. The production // authority makes that one selection inside StartCreate. production, isProduction := c.Authority.(productionCreateAuthority) var sel hostclaim.ExecutorSelection var selected bool if !isProduction { var err error if sel, selected, err = c.Authority.SelectExecutor(ctx, before, r); err != nil { return hostclaim.Observed{}, err } } // Last pre-CAS read of registration, open admission and purge reservation. if err := c.Authority.Create(ctx, before, r, id); err != nil { return hostclaim.Observed{}, err } var obs hostclaim.Observed var err error switch { case isProduction: obs, err = production.StartCreate(ctx, before, r, id, by) case selected: obs, err = hostclaim.Update(ctx, c.Store, before, hostclaim.BeginCreateOn(r.Holder, r.Body, id, by, c.Now(), sel)) default: obs, err = hostclaim.Update(ctx, c.Store, before, hostclaim.BeginCreate(r.Holder, r.Body, id, by, c.Now())) } if err != nil { return hostclaim.Observed{}, err } if isProduction { if err := production.RecordEffects(ctx, obs, r); err != nil { return obs, err } return c.Complete(ctx, obs.Host, id) } // No raw record effect precedes the durable claim intent. if _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), c.Store, r.Holder.Key, []byte(`{}`), r.Body, ""); err != nil { if !errors.Is(err, release.ErrCASConflict) { return obs, err } existing, rerr := ReadRecord(ctx, c.Store, r.Holder.Key) if rerr != nil { return obs, rerr } if !matches(existing, obs.Claim.Holder) { removed, rerr := RemoveOrphan(ctx, c.Store, existing) if rerr != nil { return obs, rerr } if !removed { return obs, ErrPending } if _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), c.Store, r.Holder.Key, []byte(`{}`), r.Body, ""); err != nil { return obs, err } } } return c.Complete(ctx, obs.Host, id) } func (c Controller) Complete(ctx context.Context, host, id string) (hostclaim.Observed, error) { if err := c.ready(); err != nil { return hostclaim.Observed{}, err } obs, err := hostclaim.Read(ctx, c.Store, host) if err != nil { return obs, err } if obs.Absent || obs.Claim.Op.Kind != hostclaim.KindCreating || obs.Claim.Op.ID != id { return obs, ErrPending } r, err := ReadRecord(ctx, c.Store, obs.Claim.Holder.Key) if err != nil { return obs, err } if !matches(r, obs.Claim.Holder) { return obs, ErrIdentity } box, err := c.Authority.Executor(ctx, obs, r) if err != nil { return obs, err } return hostclaim.Update(ctx, c.Store, obs, hostclaim.CompleteCreate(id, box)) } func (c Controller) RecoverCreate(ctx context.Context, host string) error { if err := c.ready(); err != nil { return err } obs, err := hostclaim.Read(ctx, c.Store, host) if err != nil { return err } if obs.Absent || obs.Claim.Op.Kind != hostclaim.KindCreating { return nil } if obs.Claim.Op.StartedAt.IsZero() || !c.Now().After(obs.Claim.Op.StartedAt.Add(5*time.Minute)) { return ErrPending } r, err := ReadRecord(ctx, c.Store, obs.Claim.Holder.Key) if errors.Is(err, os.ErrNotExist) { if _, production := c.Authority.(productionCreateAuthority); production { // The five-minute age is inspection eligibility. An absent record // cannot settle a delayed ownership, store, or record effect. return ErrPending } _, err = hostclaim.Update(ctx, c.Store, obs, hostclaim.CancelCreate(obs.Claim.Op.ID, c.Now())) return err } if err != nil { return err } if !matches(r, obs.Claim.Holder) { return ErrPending } _, err = c.Complete(ctx, host, obs.Claim.Op.ID) return err } // Adopt is the shared IO wrapper around the one hostclaim.Adopt transition. func (c Controller) Adopt(ctx context.Context, r Record) (hostclaim.Observed, error) { if err := c.ready(); err != nil { return hostclaim.Observed{}, err } obs, err := hostclaim.Read(ctx, c.Store, r.Holder.Host) if err != nil { return obs, err } if !obs.Absent { if matches(r, obs.Claim.Holder) { return obs, nil } if obs.Claim.Holder == nil || obs.Claim.Holder.Acquisition != "" || r.Holder.Acquisition == "" { return obs, ErrIdentity } // #232 also repairs already-claimed legacy holders while globally // frozen. Adoption authority verifies the exact manifest/ownership. prior := *obs.Claim.Holder prior.Acquisition = r.Holder.Acquisition if !matches(r, &prior) { return obs, ErrIdentity } _, box, _, _, err := c.Authority.Adoption(ctx, r) if err != nil { return obs, err } if obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.Exec != box || !slices.Contains(obs.Claim.Touched.Boxes, box) { return obs, ErrIdentity } fresh, err := ReadRecord(ctx, c.Store, r.Holder.Key) if err != nil { return obs, err } if fresh.ETag != r.ETag { return obs, ErrPending } return hostclaim.Update(ctx, c.Store, obs, func(next *hostclaim.Claim) error { if next.Holder == nil || next.Holder.Acquisition != "" { return ErrPending } next.Holder.Acquisition = r.Holder.Acquisition return nil }) } region, box, regions, boxes, err := c.Authority.Adoption(ctx, r) if err != nil { return obs, err } // Re-read the exact record after discovery; never adopt a stale listing body. fresh, err := ReadRecord(ctx, c.Store, r.Holder.Key) if err != nil { return obs, err } if fresh.ETag != r.ETag { return obs, ErrPending } n := fresh.Holder n.Region = region return hostclaim.Update(ctx, c.Store, obs, hostclaim.Adopt(n, box, regions, boxes, c.NewID(), c.NewID)) } // BeginRetire only starts the claim operation. Its successful return does not // mean the record is gone or the host is free; the caller must run/retry Retire. func (c Controller) BeginRetire(ctx context.Context, key, by, expectedInc string) (hostclaim.Observed, error) { if err := c.ready(); err != nil { return hostclaim.Observed{}, err } r, err := ReadRecord(ctx, c.Store, key) if err != nil { return hostclaim.Observed{}, err } inc, ierr := r.Incarnation() if ierr != nil { return hostclaim.Observed{}, ierr } if expectedInc == "" || inc != expectedInc { return hostclaim.Observed{}, ErrIdentity } obs, err := hostclaim.Read(ctx, c.Store, r.Holder.Host) if err != nil { return obs, err } if obs.Absent { return obs, ErrPending } if !matches(r, obs.Claim.Holder) { return obs, ErrIdentity } if err := c.Authority.StartDelete(ctx, obs, r); err != nil { return obs, err } regions, boxes, err := c.Authority.Inventory(ctx, obs) if err != nil { return obs, err } if production, ok := c.Authority.(interface { StartDeleteClaim(context.Context, hostclaim.Observed, Record, string) (hostclaim.Observed, error) }); ok { return production.StartDeleteClaim(ctx, obs, r, by) } return hostclaim.Update(ctx, c.Store, obs, hostclaim.BeginDelete(c.NewID(), by, r.ETag, c.Now(), regions, boxes, "")) } func (c Controller) Retire(ctx context.Context, host, id string) error { if err := c.ready(); err != nil { return err } obs, err := hostclaim.Read(ctx, c.Store, host) if err != nil { return err } if obs.Absent || obs.Claim.Op.ID != id { return ErrPending } op := obs.Claim.Op if op.Kind != hostclaim.KindDeleting && op.Kind != hostclaim.KindPurging { return ErrPending } if obs.Claim.Holder == nil || op.Intent["key"] != obs.Claim.Holder.Key || op.Intent["etag"] == "" { return ErrIdentity } if err := c.Authority.FirstPhase(ctx, obs); err != nil { return err } if op.Kind == hostclaim.KindPurging { if err := c.Authority.PurgeRetirement(ctx, obs); err != nil { return err } } regions, boxes, err := c.Authority.Inventory(ctx, obs) if err != nil { return err } // Validate record identity before deleting even the stored version. r, rerr := ReadRecord(ctx, c.Store, op.Intent["key"]) if rerr != nil && !errors.Is(rerr, os.ErrNotExist) { return rerr } if rerr == nil { if !matches(r, obs.Claim.Holder) { return ErrIdentity } if r.ETag != op.Intent["etag"] { if authority, ok := c.Authority.(interface { RefreshDeleteRecord(context.Context, hostclaim.Observed, Record) error }); ok { return authority.RefreshDeleteRecord(ctx, obs, r) } return ErrPending } if err := release.DeleteIfMatch(ctx, c.Store, r.Holder.Key, r.ETag); err != nil { return err } } // A success/412 alone never certifies absence. Re-read before parking. _, _, err = c.Store.Get(ctx, op.Intent["key"]) if !errors.Is(err, os.ErrNotExist) { if err != nil { return err } return ErrPending } // The old observation binds the entire sequence to this operation. transition := hostclaim.ToParking(id, regions, boxes, c.NewID) if _, production := c.Authority.(productionCreateAuthority); production { transition = hostclaim.ToScopedParking(id, regions, boxes, c.NewID) } _, err = hostclaim.Update(ctx, c.Store, obs, transition) if err != nil { return err } return ErrPending } func (c Controller) Free(ctx context.Context, host, id string) error { if err := c.ready(); err != nil { return err } obs, err := hostclaim.Read(ctx, c.Store, host) if err != nil { return err } if obs.Absent { return ErrPending } _, err = hostclaim.Update(ctx, c.Store, obs, hostclaim.FreeAfterPark(id)) return err }