package hostops import ( "context" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "os" "sort" "strconv" "strings" "github.com/carlosframework/platform/internal/domains" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/provision" ) func (a ProductionAuthority) admission(ctx context.Context, account, app string) (AdmissionObserved, error) { raw, _, err := a.Store.Get(ctx, "apps/"+account+"/"+app+"/admission.json") if err != nil { return AdmissionObserved{}, err } var id struct { AppID string `json:"app_id"` } if err = json.Unmarshal(raw, &id); err != nil { return AdmissionObserved{}, err } return ReadAdmission(ctx, a.Store, account, app, id.AppID) } func (a ProductionAuthority) Create(ctx context.Context, before hostclaim.Observed, r Record, id string) error { if err := CheckCreateFreeze(ctx, a.Store, a.Directory.PublicKey); err != nil { return err } adm, err := a.admission(ctx, r.Holder.Account, r.Holder.App) if err != nil { return err } reg, ok := adm.Value.Registrations[id] inc, err := r.Incarnation() if err != nil { return err } if !ok || adm.Value.State != "open" || adm.Value.Op != nil || reg.Kind != "create" || reg.Host != r.Holder.Host || reg.AppID != adm.Value.AppID || reg.Epoch != adm.Value.Epoch || reg.Owner.Console != a.ConsoleID || reg.Owner.Enrollment != a.Enrollment || reg.Claim != SavedObservation(before) || reg.Incarnation != inc || !reg.CreatedAt.Equal(r.Holder.CreatedAt) || !RegistrationLive(reg, a.Now()) { return ErrPending } owner, ownerErr := domains.GetClaim(ctx, a.Store, r.Holder.Host) if ownerErr != nil && !errors.Is(ownerErr, os.ErrNotExist) { return ownerErr } if ownerErr == nil && owner != nil { legacy := owner.Acquisition == "" && owner.RecordIncarnation == "" if legacy && owner.Pending && !r.VerifiedOwnership { return ErrPending } bound := owner.Acquisition == r.Holder.Acquisition && owner.RecordIncarnation == inc if owner.Account != r.Holder.Account || owner.App != r.Holder.App || (!legacy && !bound) { return ErrPending } if legacy { if _, _, e := a.Store.Get(ctx, r.Holder.Key); !errors.Is(e, os.ErrNotExist) { if e != nil { return e } return ErrPending } } } _, _, err = a.Store.Get(ctx, "apps/"+r.Holder.Account+"/"+r.Holder.App+"/purge.json") if !errors.Is(err, os.ErrNotExist) { if err != nil { return err } return ErrPending } return nil } func operationToken() (string, error) { var b [16]byte if _, err := rand.Read(b[:]); err != nil { return "", err } return hex.EncodeToString(b[:]), nil } type executorCandidate struct { box, enrollment, hash string sameApp bool hosts int } // selectExecutor is claim Plan 5's create-time selection over this // console's namespace, the record's region and pool, active and unfenced. // With a chooser (ProductionAuthority.Executors, CARLOS_REGION_HA on) the // one choice is the chooser's, over its edges evidence; this scope still // binds it, so a chosen box outside it refuses the create. func (a ProductionAuthority) selectExecutor(ctx context.Context, before hostclaim.Observed, r Record) (string, string, string, error) { var shape struct { Pool string `json:"pool"` Replicas []string `json:"replicas"` } if err := json.Unmarshal(r.Body, &shape); err != nil { return "", "", "", err } if len(shape.Replicas) != 0 { return "", "", "", ErrContract } everywhere := false if r.Holder.Family == "instance" { var rec provision.Record if err := json.Unmarshal(r.Body, &rec); err != nil { return "", "", "", err } everywhere = provision.ClaimEverywhere(rec) } pool := shape.Pool if pool == "" { pool = "default" } roster, err := a.Directory.Read(ctx) if err != nil { return "", "", "", err } if !roster.Roster.MatchesHome(a.HomeRegion) { return "", "", "", ErrIdentity } candidates := map[string]*executorCandidate{} servingScope := 0 for ref, b := range roster.Roster.Boxes { if b.Namespace == a.Namespace && b.Pool == pool && b.State == "active" { servingScope++ } if b.Namespace != a.Namespace || !park.SameRegion(a.HomeRegion, b.Region, r.Holder.Region) || b.Pool != pool || b.State != "active" { continue } fenced, err := a.Directory.TerminationFence(ctx, roster, ref, a.Now()) if err != nil { return "", "", "", err } if fenced { continue } sum := sha256.Sum256([]byte(r.Holder.Host + "\x00" + ref)) candidates[ref] = &executorCandidate{box: ref, enrollment: b.Enrollment, hash: hex.EncodeToString(sum[:])} } if everywhere && servingScope != 1 { return "", "", "", ErrContract } if sel, selected, err := a.SelectExecutor(ctx, before, r); err != nil { return "", "", "", err } else if selected { if c := candidates[sel.Box]; c == nil || c.enrollment != sel.Enrollment { return "", "", "", fmt.Errorf("%w: %s is not in this console's namespace and the record's pool", ErrNoEligibleBox, sel.Box) } return sel.Box, sel.Enrollment, pool, nil } hosts, err := ListClaims(ctx, a.Store) if err != nil { return "", "", "", err } for _, host := range hosts { obs, err := hostclaim.Read(ctx, a.Store, host) if err != nil { return "", "", "", err } if obs.Absent || obs.Claim.Holder == nil { continue } box := obs.Claim.Op.Exec if box == "" { box = obs.Claim.Op.Intent["selected_executor"] } c := candidates[box] if c == nil { continue } c.hosts++ h := obs.Claim.Holder if h.Account == r.Holder.Account && h.App == r.Holder.App { c.sameApp = true } } var ordered []*executorCandidate for _, c := range candidates { ordered = append(ordered, c) } sort.Slice(ordered, func(i, j int) bool { x, y := ordered[i], ordered[j] if x.sameApp != y.sameApp { return !x.sameApp } if x.hosts != y.hosts { return x.hosts < y.hosts } if x.hash != y.hash { return x.hash > y.hash } return x.box < y.box }) if len(ordered) == 0 { return "", "", "", ErrPending } return ordered[0].box, ordered[0].enrollment, pool, nil } func (a ProductionAuthority) StartCreate(ctx context.Context, before hostclaim.Observed, r Record, id, by string) (hostclaim.Observed, error) { box, enrollment, pool, err := a.selectExecutor(ctx, before, r) if err != nil { return hostclaim.Observed{}, err } adm, err := a.admission(ctx, r.Holder.Account, r.Holder.App) if err != nil { return hostclaim.Observed{}, err } region := r.Holder.Region if region == park.HomeRegion { region = a.HomeRegion } scope := hostactors.Scope{Namespace: a.Namespace, Regions: []string{region}, Pools: []string{pool}, Participants: []string{box}, ConsoleID: a.ConsoleID} // Final admission check is after census/reservation, immediately before // the one CAS against the observation saved before registration. if err = a.Create(ctx, before, r, id); err != nil { return hostclaim.Observed{}, err } ownershipEffect, err := operationToken() if err != nil { return hostclaim.Observed{}, err } recordEffect, err := operationToken() if err != nil { return hostclaim.Observed{}, err } transition := hostclaim.BeginSelectedCreate(r.Holder, r.Body, id, by, a.Now(), box, enrollment, adm.Value.AppID, adm.Value.Epoch, ownershipEffect, recordEffect) return a.startHeldClaim(ctx, before, id, scope, func() error { return a.Create(ctx, before, r, id) }, func(c *hostclaim.Claim) error { if err := transition(c); err != nil { return err } if r.VerifiedOwnership { c.Op.Intent["ownership_verified"] = "true" } return nil }) } func (a ProductionAuthority) Executor(ctx context.Context, obs hostclaim.Observed, r Record) (string, error) { if obs.Absent || obs.Claim.Holder == nil || !matches(r, obs.Claim.Holder) { return "", ErrIdentity } op := obs.Claim.Op box, enrollment := op.Intent["selected_executor"], op.Intent["executor_enrollment"] if box == "" || enrollment == "" { return "", ErrContract } if err := a.Directory.CheckHold(ctx, obs.Claim, box, enrollment); err != nil { return "", err } actor, err := a.Directory.Lookup(ctx, box) if err != nil { return "", err } if actor.State != "active" || actor.Enrollment != enrollment { return "", ErrPending } adm, err := ReadAdmission(ctx, a.Store, r.Holder.Account, r.Holder.App, op.Intent["app_id"]) if err != nil { return "", err } epoch, err := strconv.ParseUint(op.Intent["admission_epoch"], 10, 64) if err != nil || epoch != adm.Value.Epoch { return "", ErrIdentity } foundRecord := false for _, e := range adm.Value.Effects { if e.OperationID != op.ID { continue } if e.AppID != adm.Value.AppID || e.State != "complete" { return "", ErrPending } if e.Resource == r.Holder.Key && e.Action == "record-put" { foundRecord = true } } if !foundRecord { return "", ErrPending } if err = a.FirstPhase(ctx, obs); err != nil { return "", err } return box, nil } func (a ProductionAuthority) FirstPhase(ctx context.Context, obs hostclaim.Observed) error { if obs.Absent || obs.Claim == nil || obs.Claim.Op.ID == "" { return ErrIdentity } roster, err := a.Directory.Read(ctx) if err != nil { return err } hold, ok := roster.Roster.Holds[obs.Claim.Op.Intent["actors_hold"]] if !ok || hold.Host != obs.Host || hold.OpID != obs.Claim.Op.ID { return ErrPending } floor, err := hostfloor.Read(ctx, a.Store, a.Directory.PublicKey, a.Directory.Bucket) if err != nil { return err } if floor.Digest != hold.FloorSHA256 || roster.Roster.FloorSHA256 != floor.Digest { return ErrPending } for actor, enrollment := range hold.Actors { report, ok := hold.Releases[actor] if !ok { return ErrPending } instance := "" if ref, ok := strings.CutPrefix(actor, "box:"); ok { box := roster.Roster.Boxes[ref] if box.State != "active" || box.Enrollment != enrollment { return ErrPending } fenced, err := a.Directory.TerminationFence(ctx, roster, ref, a.Now()) if err != nil { return err } if fenced { return ErrPending } instance = box.Cloud.InstanceID } else if ref, ok := strings.CutPrefix(actor, "console:"); ok { console := roster.Roster.Consoles[ref] if console.Enrollment != enrollment { return ErrPending } instance = console.InstanceID } else { return ErrIdentity } if err = hostactors.VerifyReport(ctx, a.Evidence, a.Directory.PublicKey, floor, report, enrollment, instance, a.Now()); err != nil { return err } } if obs.Claim.Op.Kind == hostclaim.KindPurging { if err := a.StopBeforeTeardown(ctx, obs); err != nil { return err } } for _, box := range obs.Claim.Op.Participants { var ack hostclaim.ExecutionAcknowledgement if json.Unmarshal([]byte(obs.Claim.Op.Journal["exec_ack."+box]), &ack) != nil || ack.Op != obs.Claim.Op.ID || ack.Enrollment != hold.Actors["box:"+box] || ack.BootID == "" || ack.ControllerID == "" || ack.LeaseETag == "" || !a.Now().Before(ack.Expires.Add(-RegistrationSkew)) { return ErrPending } } return nil } func (a ProductionAuthority) StartDelete(ctx context.Context, obs hostclaim.Observed, r Record) error { if obs.Absent || obs.Claim.Holder == nil || !matches(r, obs.Claim.Holder) { return ErrIdentity } if obs.Claim.Op.Kind != hostclaim.KindServing { return ErrPending } adm, err := a.admission(ctx, r.Holder.Account, r.Holder.App) if err != nil { return err } if adm.Value.State != "closed" || adm.Value.Op == nil || adm.Value.Op.Kind != "deleting" || len(adm.Value.Registrations) != 0 { return ErrPending } for _, effect := range adm.Value.Effects { if effect.State != "complete" { return ErrPending } } return nil } func (a ProductionAuthority) PurgeRetirement(ctx context.Context, obs hostclaim.Observed) error { if obs.Absent || obs.Claim.Holder == nil || obs.Claim.Op.Kind != hostclaim.KindPurging { return ErrIdentity } h := obs.Claim.Holder journal, _, err := ReadPurge(ctx, a.Store, h.Account, h.App) if err != nil { return err } if journal.ID != obs.Claim.Op.ID || journal.Phase != "torn-down" { return ErrPending } return nil }