package console import ( "context" "errors" "fmt" "log" "sort" "time" "github.com/carlosframework/platform/internal/boxchoice" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/park" ) // The eligible-capacity metric (region HA spec 2 §4, A1-11): once a // minute the console counts, for each region its boxes publish, the boxes // that could take a new instance now (hostops.ExecutorChooser.Capacity: // enrolled, active, unfenced and placeable), and publishes the counts. An // alarm on the count catches what the cloud's own group metrics cannot: a // box that is in service but never enrolled, and a launch loop. Where the // counts go is an adapter's business (awsregionmetric.go); a console with // none publishes nothing. // capacityInterval is how often the counts are published. const capacityInterval = time.Minute // regionCapacity is one region's count. Placement is the CARLOS placement // code the metric is keyed by; AWSRegion is where its boxes run. type regionCapacity struct { Placement, AWSRegion string Eligible int } // capacityPublisher sends the counts to a metrics service. type capacityPublisher interface { PublishCapacity(ctx context.Context, counts []regionCapacity) error } // regionCapacities counts each placement the snapshot's files name, sorted // by placement, and takes each one's AWS region from capacity's answer: // the signed roster's, never the files' own claim, so a stale or // misconfigured file naming another AWS region cannot send a region's // count elsewhere (Astra 2a round 2 finding 3). A placement the roster // does not know (no region entry, so no box there can be chosen) is left // out. A placement whose count fails, or whose read is incomplete // (hostops.ErrEvidenceIncomplete), is left out too and its error returned // with the counts that did succeed: publishing 0 for it would trip the // eligible-capacity alarm on a transient read failure, whereas a missing // data point is the alarm's own signal if the failure lasts (spec §4, // A2-8). Only a complete read that finds no placeable box is a 0. // // A home box publishes the home's placement code, "". It is counted under // home, the home's roster name, as the chooser reads it (Capacity's and // eligible's mapping of park.HomeRegion to HomeName), so the home region's // count goes to the AWS region the signed roster gives home (Astra 2a // final review, P2). With no home name the chooser places nothing in the // home, and its boxes are left out. func regionCapacities(ctx context.Context, snap boxchoice.Snapshot, home string, capacity func(context.Context, string) (int, string, error)) ([]regionCapacity, error) { named := map[string]bool{} for _, b := range snap.Boxes { if p := capacityPlacement(home, b.Placement); p != "" { named[p] = true } } placements := make([]string, 0, len(named)) for p := range named { placements = append(placements, p) } sort.Strings(placements) var out []regionCapacity var errs []error for _, p := range placements { n, aws, err := capacity(ctx, p) if errors.Is(err, hostops.ErrContract) { continue } if err != nil { errs = append(errs, fmt.Errorf("count %s: %w", p, err)) continue } out = append(out, regionCapacity{Placement: p, AWSRegion: aws, Eligible: n}) } return out, errors.Join(errs...) } // capacityPlacement is the region a box's placement code is counted under: // the code itself, or for the home's "" the home's name, exactly when the // chooser maps park.HomeRegion to it (home is a placement code). "" is // counted nowhere. func capacityPlacement(home, code string) string { if region := park.PlacementRegion(code); region == park.HomeRegion && home != "" && park.PlacementRegion(home) == home { return home } return code } // publishCapacity counts and publishes once. func (s *Server) publishCapacity(ctx context.Context, pub capacityPublisher) error { x := s.placementChooser("") if x == nil { return hostops.ErrContract } snap, err := s.liveEdges(ctx) if err != nil { return err } counts, countErr := regionCapacities(ctx, boxSnapshot(snap), x.HomeName, x.Capacity) if len(counts) == 0 { return countErr } // The regions that counted are published even when another failed: // the failed one has no data point, which is the alarm's signal. return errors.Join(pub.PublishCapacity(ctx, counts), countErr) } // runCapacityPass publishes every capacityInterval until ctx ends. A // failed pass is logged on its first failure and every tenth; the alarm // treats missing data as breaching, so a console that stops publishing is // itself an alert. func (s *Server) runCapacityPass(ctx context.Context, pub capacityPublisher) { log.Printf("console: eligible-capacity metric armed (every %s)", capacityInterval) t := time.NewTicker(capacityInterval) defer t.Stop() fails := 0 for { pass, cancel := context.WithTimeout(ctx, capacityInterval) err := s.publishCapacity(pass, pub) cancel() switch { case err != nil: fails++ if fails == 1 || fails%10 == 0 { log.Printf("console: eligible-capacity metric (failure #%d): %v", fails, err) } case fails > 0: log.Printf("console: eligible-capacity metric recovered after %d failure(s)", fails) fails = 0 } select { case <-ctx.Done(): return case <-t.C: } } }