package hostops import ( "context" "errors" "fmt" "time" "github.com/carlosframework/platform/internal/boxchoice" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/park" ) // ErrNoEligibleBox is a create refused because no box in the placement is // eligible (region HA §5.2). The console maps it to today's refusal. var ErrNoEligibleBox = errors.New("hostops: region has no box that can take this") // ErrEvidenceIncomplete is a placement read that could not judge every // box: an edges file unread or malformed, the status listing unread, or a // fresh box with no edges file (boxchoice.Snapshot.Complete false). It is // never ErrNoEligibleBox: a door answers it as temporarily unavailable, // and nothing pages "no room" for it (region HA 2a, Task 11 ruling). var ErrEvidenceIncomplete = errors.New("hostops: the placement read was incomplete, so no box could be judged") // ExecutorChooser is create's executor selection: claim P5-D2's choose() // over the complete eligible set (region HA spec 1 §5.2, H12, L8). It is // made once, before the create CAS, and recorded in the create intent. type ExecutorChooser struct { Directory hostactors.Directory // Edges is one read of every box's edges file with the caller's // liveness verdict per box. An incomplete snapshot selects nothing: // every question answers ErrEvidenceIncomplete (judged). Edges func(context.Context) (boxchoice.Snapshot, error) // Loads counts claimed hosts per box and per "acct/app", from the // claim-recovery listing (claim §5.1 step 5). A stale count only // affects balance, never safety. Loads func(context.Context) (map[string]boxchoice.Load, error) SecretsKeyFP string HomeName string Now func() time.Time // Namespace and Pool are the scope the chooser chooses within, claim // Plan 5's create scope: only roster boxes in Namespace serving Pool // are candidates, and only their fences are read. Pool "" is the // shared pool (the roster's "default"). An empty Namespace matches no // roster box, so it chooses nothing. ProductionAuthority.SelectExecutor // sets both from its own namespace and the record's pool. Namespace, Pool string } // sharedPool is the roster's name for the shared "" pool (claim Plan 5's // selectExecutor reads a record's empty pool as it). const sharedPool = "default" // Choose returns the box, and its roster enrollment, that a new host in // r's placement is created on. Identity, placement and AWS region come from // the roster; a box with a valid termination fence, or not active, is never // chosen; a box on the claim's lost list, or owing a park-local chore on // it, is never chosen. Among the rest, region HA spec 2's policy (§7) // decides: placeable boxes only (placeable), then boxchoice.ChooseRoomiest. func (x ExecutorChooser) Choose(ctx context.Context, obs hostclaim.Observed, r Record) (hostclaim.ExecutorSelection, error) { o, cands, err := x.judged(ctx, obs, r) if err != nil { return hostclaim.ExecutorSelection{}, err } cands = x.placeable(o, cands) loads, err := x.Loads(ctx) if err != nil { return hostclaim.ExecutorSelection{}, err } best, ok := boxchoice.ChooseRoomiest(r.Holder.Host, cands, loads, r.Holder.Account+"/"+r.Holder.App) if !ok { return hostclaim.ExecutorSelection{}, ErrNoEligibleBox } return hostclaim.ExecutorSelection{Box: best.BoxID, Enrollment: o.Roster.Boxes[best.BoxID].Enrollment}, nil } // Permits reports whether box is in the complete eligible set for a host // in r's placement now (spec 1 §2, §5.8 "The start CAS ... refused ... // when `to` is not eligible for the host"): the same read Choose makes, // without choosing. A stale edges file, a missing capability or secrets // key, a draining box, a fence, or an incomplete snapshot refuses, and so // does a box spec 2's policy does not place on (placeable): a legacy or // platform role, too little memory or disk, or stale headroom. func (x ExecutorChooser) Permits(ctx context.Context, obs hostclaim.Observed, r Record, box string) error { o, cands, err := x.judged(ctx, obs, r) if err != nil { return err } for _, c := range x.placeable(o, cands) { if c.BoxID == box { return nil } } return fmt.Errorf("%w: %s", ErrNoEligibleBox, box) } // PlaceableBoxes is every box in placement that could take a new // instance in the chooser's namespace and pool now, sorted: Choose's set, // before the choice. The console's move door asks it only whether there // is room (region HA spec 2 §7, H27); it never takes a destination from // it. An incomplete read is ErrEvidenceIncomplete, never an empty set. func (x ExecutorChooser) PlaceableBoxes(ctx context.Context, placement string) ([]string, error) { r := Record{Holder: hostclaim.NewHolder{Region: placement}} o, cands, err := x.judged(ctx, hostclaim.Observed{Absent: true}, r) if err != nil { return nil, err } boxes := []string{} for _, c := range x.placeable(o, cands) { boxes = append(boxes, c.BoxID) } return boxes, nil } // Capacity is how many boxes PlaceableBoxes names, and the AWS region the // signed roster names for placement. The console publishes the count per // region as the eligible-capacity metric (region HA spec 2 §4, A1-11), in // that AWS region: the roster's mapping, the one eligible reads, never an // edges file's own claim (Astra 2a round 2 finding 3). An incomplete read // is ErrEvidenceIncomplete, never a count of 0. func (x ExecutorChooser) Capacity(ctx context.Context, placement string) (int, string, error) { r := Record{Holder: hostclaim.NewHolder{Region: placement}} o, cands, err := x.judged(ctx, hostclaim.Observed{Absent: true}, r) if err != nil { return 0, "", err } // eligible's own home mapping, so the region is the one it read. if placement == park.HomeRegion && x.HomeName != "" && park.PlacementRegion(x.HomeName) == x.HomeName { placement = x.HomeName } return len(x.placeable(o, cands)), o.Roster.Regions[placement].AWSRegion, nil } // judged is eligible's read, refused with ErrEvidenceIncomplete when the // edges snapshot it read was incomplete: eligible's empty set then means // "could not tell", never "no room". eligible itself (an accepted-legacy // census unit) is unchanged; the snapshot's completeness is taken from // the same Edges read it makes, through a wrapper on this copy of x. func (x ExecutorChooser) judged(ctx context.Context, obs hostclaim.Observed, r Record) (hostactors.Observed, []boxchoice.Candidate, error) { complete := false if edges := x.Edges; edges != nil { x.Edges = func(ctx context.Context) (boxchoice.Snapshot, error) { snap, err := edges(ctx) complete = err == nil && snap.Complete return snap, err } } o, cands, err := x.eligible(ctx, obs, r) if err != nil { return o, cands, err } if !complete { return hostactors.Observed{}, nil, ErrEvidenceIncomplete } return o, cands, nil } // placeable is region HA spec 2's policy over the eligible set (§7): // each box's roster role, then boxchoice.Placeable's thresholds at // x.Now(). Identity was Eligible's; nothing here reads a provider. func (x ExecutorChooser) placeable(o hostactors.Observed, cands []boxchoice.Candidate) []boxchoice.Candidate { roles := make(map[string]string, len(cands)) for _, c := range cands { roles[c.BoxID] = o.Roster.Boxes[c.BoxID].Role } return boxchoice.Placeable(cands, roles, x.Now()) } // eligible is one read of the roster, the fences and the edges files, // reduced to boxchoice.Eligible's candidates. func (x ExecutorChooser) eligible(ctx context.Context, obs hostclaim.Observed, r Record) (hostactors.Observed, []boxchoice.Candidate, error) { placement := r.Holder.Region // A record placed in the home names it park.HomeRegion when no home // name is configured on the console (main's claimPlacementRegion); the // roster, its boxes and their edges files name it by the home's roster // name. The one mapping is main's StartCreate's: park.HomeRegion is // HomeName when HomeName is a placement code. if placement == park.HomeRegion && x.HomeName != "" && park.PlacementRegion(x.HomeName) == x.HomeName { placement = x.HomeName } if x.Edges == nil || x.Loads == nil || x.Now == nil || placement == "" { return hostactors.Observed{}, nil, ErrContract } o, err := x.Directory.Read(ctx) if err != nil { return hostactors.Observed{}, nil, err } region, ok := o.Roster.Regions[placement] if !ok || region.AWSRegion == "" { return hostactors.Observed{}, nil, ErrContract } pool := x.Pool if pool == "" { pool = sharedPool } roster := map[string]boxchoice.RosterBox{} for ref, b := range o.Roster.Boxes { // Scope first, so a fence is read only for a box this create could // name: another namespace's unreadable fence never blocks it. if b.State != "active" || b.Region != placement || b.Namespace != x.Namespace || b.Pool != pool { continue } fenced, err := x.Directory.TerminationFence(ctx, o, ref, x.Now()) if err != nil { return hostactors.Observed{}, nil, err // C6: a read error never means "not fenced" } if !fenced { roster[ref] = boxchoice.RosterBox{Placement: b.Region, Cloud: b.Cloud} } } snap, err := x.Edges(ctx) if err != nil { return hostactors.Observed{}, nil, err } if pool != sharedPool { // serves_shared_pool is a box's answer for the shared pool only. A // named pool's membership is the roster's, already applied above, // so the shared-pool answer is not asked of its boxes. yes := true boxes := make([]boxchoice.Box, len(snap.Boxes)) for i, b := range snap.Boxes { b.ServesSharedPool = &yes boxes[i] = b } snap.Boxes = boxes } // Never a box on the claim's lost list, nor one that still owes a // park-local chore for this host (spec 1 §2: its old installation is // not yet retired). excluded := map[string]bool{} if !obs.Absent && obs.Claim != nil { for _, box := range obs.Claim.Lost { excluded[box] = true } for _, box := range hostclaim.LocalChoreBoxes(obs.Claim) { excluded[box] = true } } return o, boxchoice.Eligible(snap, roster, placement, region.AWSRegion, x.HomeName, x.SecretsKeyFP, excluded), nil }