package console import ( "context" "errors" "fmt" "log" "net/http" "strings" "sync" "time" "github.com/carlosframework/platform/internal/alerting" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostops" ) // No box has room (region HA spec 2 ยง7, ruling H27). Every door that // chooses a box (instance create, domain attach's first route, standalone // sidecar create, and the move door's destination) refuses with a 503 when // the placement policy finds no eligible box, and the operator hears of it // at most once per region an hour per console process: a restart may send // one extra alert, which H27 accepts rather than persist the limit. A // create door chooses before it registers anything (chooseExecutorFirst), // so a refusal leaves nothing registered, nothing written to the claim and // nothing for recovery to settle. Only a complete placement read with no // placeable box is "no room": an incomplete one (hostops.ErrEvidenceIncomplete) // is answered as temporarily unavailable and pages nobody. // noCapacityAlertEvery is how often one region's alert may repeat. const noCapacityAlertEvery = time.Hour // noCapacityRetryAfter is the Retry-After (seconds) a no-room 503 carries: // RHA-C17 tells the caller to try again in a few minutes. Every other // claim refusal keeps its own. const noCapacityRetryAfter = "60" func noCapacityRefusal(region string) string { return fmt.Sprintf("No box in %s has room right now. Try again in a few minutes.", region) } func noCapacityAlert(region string) (subject, body string) { return fmt.Sprintf("No box in %s has room", region), fmt.Sprintf("A request in %s was refused because no box there can take it. Every box is full, stale or excluded by its role. Add a box, or free memory or disk on one, and check the eligible-capacity alarm for %s.", region, region) } // noCapacityError is a request refused because no box in Region (its // display name) can take it. It is hostops.ErrNoEligibleBox, and // claimResponseRefusal answers it with noCapacityRefusal. type noCapacityError struct{ Region string } func (e *noCapacityError) Error() string { return "no eligible box in " + e.Region + ": " + hostops.ErrNoEligibleBox.Error() } func (e *noCapacityError) Is(target error) bool { return target == hostops.ErrNoEligibleBox } // pageNoCapacity sends the alert; tests replace it. var pageNoCapacity = alerting.Page // noCapacityAlerts remembers when each region's alert was last sent, in // this console process. An alert counts as sent only once its send // succeeds: due reserves the region's slot so concurrent refusals do not // page twice while one send is in flight, and failed gives the slot back, // so the next refusal tries again rather than the region going quiet for // an hour. The zero value is ready. type noCapacityAlerts struct { mu sync.Mutex last map[string]time.Time } // due reports whether region's alert should be sent at now, and if so // records it as sent. func (a *noCapacityAlerts) due(region string, now time.Time) bool { a.mu.Lock() defer a.mu.Unlock() if at, ok := a.last[region]; ok && now.Sub(at) < noCapacityAlertEvery { return false } if a.last == nil { a.last = map[string]time.Time{} } a.last[region] = now return true } // failed gives back the slot due reserved at for region, because that send // failed. A later reservation is left alone. func (a *noCapacityAlerts) failed(region string, at time.Time) { a.mu.Lock() defer a.mu.Unlock() if last, ok := a.last[region]; ok && last.Equal(at) { delete(a.last, region) } } // isNoCapacityRefusal reports whether ref is the no-room refusal // (claimResponseRefusal's answer to a noCapacityError), which the attach // doors carry as a doorRefusal. func isNoCapacityRefusal(ref *doorRefusal) bool { if ref == nil || ref.Status != http.StatusServiceUnavailable { return false } before, after, _ := strings.Cut(noCapacityRefusal("\x00"), "\x00") region, ok := strings.CutPrefix(ref.Msg, before) if !ok { return false } region, ok = strings.CutSuffix(region, after) return ok && region != "" } // setNoCapacityRetryAfter stamps noCapacityRetryAfter on w when ref is the // no-room refusal, and leaves every other refusal's headers as its door // writes them. func setNoCapacityRetryAfter(w http.ResponseWriter, ref *doorRefusal) { if isNoCapacityRefusal(ref) { w.Header().Set("Retry-After", noCapacityRetryAfter) } } // noRoom is the refusal for region (record vocabulary, "" for the home): // it logs, sends the operator alert when one is due, and returns the // error the door answers with. func (s *Server) noRoom(ctx context.Context, region string) error { name := s.homeRegionOut(region) log.Printf("console: refused: no eligible box in %s", name) if now := time.Now(); s.noCapacity.due(name, now) { subject, body := noCapacityAlert(name) go func() { ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer cancel() if err := pageNoCapacity(ctx, alerting.LevelWarning, subject, body); err != nil { // Not sent: the next refusal in name tries again. s.noCapacity.failed(name, now) log.Printf("console: no-capacity alert for %s: %v", name, err) } }() } return &noCapacityError{Region: name} } // placementChooser is the create door's chooser scoped as // ProductionAuthority.SelectExecutor scopes it, for pool ("" the shared // pool), or nil with region HA off or no claim runtime. func (s *Server) placementChooser(pool string) *hostops.ExecutorChooser { a := s.cfg.ClaimAuthority if !s.cfg.RegionHA || a == nil || a.Executors == nil { return nil } x := *a.Executors x.Namespace, x.Pool = a.Namespace, pool return &x } // roomIn is H27's question about region (record vocabulary) for pool, asked // with region HA on before a door's own placement checks (Astra 2a round 2 // finding 1): a region the roster places boxes in, whose placeable set is // empty (no edges file, no public address, stale, full or excluded by // role), is the no-room refusal, never a 400. It only refuses: it chooses // nothing and changes nothing the door goes on to do. It answers nil with // region HA off, for a region the roster does not place (the door's own // checks answer that), and when a box has room. An incomplete read is // hostops.ErrEvidenceIncomplete, which the door answers as temporarily // unavailable, with no alert. func (s *Server) roomIn(ctx context.Context, region, pool string) error { x := s.placementChooser(pool) if x == nil { return nil } boxes, err := x.PlaceableBoxes(ctx, s.cfg.claimPlacementRegion(region)) switch { case errors.Is(err, hostops.ErrContract): return nil case err != nil: return err case len(boxes) == 0: return s.noRoom(ctx, region) } return nil } // errChoiceStale is an early choice whose claim observation is no longer // the claim: a write landed between the choice and beginClaimCreate's own // read. It wraps hostclaim.ErrConflict, so errors.Is(errChoiceStale, // hostclaim.ErrConflict) holds on the sentinel itself and the door answers // main's retryable busy refusal; nothing was registered or written. var errChoiceStale = fmt.Errorf("%w: the host's claim changed after its executor was chosen", hostclaim.ErrConflict) // chosenOnKey carries the claim observation an early choice was decided // on, beside the choice itself (hostops.WithChosenExecutor). type chosenOnKey struct{} type chosenOn struct { host, etag string absent bool } // chooseExecutorFirst is beginClaimCreate's first step with region HA on, // given beginClaimCreate's own read of the claim (before). It chooses the // executor for the record build would make, before anything is // registered, and carries the choice to the create // (hostops.WithChosenExecutor). It takes no lease and writes nothing: the // choice is SelectExecutor's read of the roster and the edges files. No // eligible box on a complete read is noRoom's refusal; an incomplete read // (hostops.ErrEvidenceIncomplete) is returned as it is. A choice ctx already carries for host // (chooseCreateFirst's) is kept only when before is the claim observation // it was decided on (census ruling region-ha-2a, item 5): otherwise the // create is refused with errChoiceStale, and the old choice is never used // and nothing is chosen again. With region HA off it returns ctx // unchanged, and the create is main's. func (s *Server) chooseExecutorFirst(ctx context.Context, host string, before hostclaim.Observed, build func(time.Time, string) (hostops.Record, error)) (context.Context, error) { a := s.cfg.ClaimAuthority if !s.cfg.RegionHA || a == nil || a.Executors == nil { return ctx, nil } if _, ok := hostops.ChosenExecutor(ctx, host); ok { on, ok := ctx.Value(chosenOnKey{}).(chosenOn) if !ok || on.host != host || on.etag != before.ETag || on.absent != before.Absent { return ctx, fmt.Errorf("%w: %s", errChoiceStale, host) } return ctx, nil } probe, err := build(time.Now().UTC(), "") if err != nil { return ctx, err } sel, chosen, err := a.SelectExecutor(ctx, before, probe) if errors.Is(err, hostops.ErrNoEligibleBox) { return ctx, s.noRoom(ctx, probe.Holder.RecordRegion) } if err != nil || !chosen { return ctx, err } return hostops.WithChosenExecutor(ctx, host, sel), nil } // chooseCreateFirst is the instance create door's choice, made before the // door's first write (Astra 2a round 2 finding 2): the environment binding // (bindEnvironmentCreate) can move a seed's store to the new instance, so // a create refused for want of room must be refused before it. It reads // host's claim and chooses as chooseExecutorFirst does, and returns the // choice together with the claim observation it decided on (its ETag, or // its absence). beginClaimCreate keeps the choice only if its own read is // that same observation. With region HA off it returns ctx unchanged and // reads nothing. func (s *Server) chooseCreateFirst(ctx context.Context, host string, build func(time.Time, string) (hostops.Record, error)) (context.Context, error) { a := s.cfg.ClaimAuthority if !s.cfg.RegionHA || a == nil || a.Executors == nil { return ctx, nil } before, err := hostclaim.Read(ctx, s.cfg.Releases, host) if err != nil { return ctx, err } chosen, err := s.chooseExecutorFirst(ctx, host, before, build) if err != nil { return ctx, err } if _, ok := hostops.ChosenExecutor(chosen, host); !ok { return chosen, nil } return context.WithValue(chosen, chosenOnKey{}, chosenOn{host: host, etag: before.ETag, absent: before.Absent}), nil }