package hostactors import ( "context" "crypto/x509" "encoding/base64" "encoding/json" "errors" "fmt" "os" "slices" "sort" "strings" "time" "github.com/carlosframework/platform/internal/boxid" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/release" ) // The console's automatic enrollment pass (region HA spec 2 §5 step 5, // A2-1, H16, H28). A box its region's group launched publishes its signed // identity record (control/boxes/.json) and joins. For each such box // not yet in the roster, the pass takes the next step its state allows: // // 1. no invitation: begin an enrollment attempt (BeginEnrollmentAttempt), // then, on the next pass, confirm the instance's group membership with // the console's own read (checkConfirmed, Members) and invite the box // (Invite) while that read is under a minute old; // 2. invited: wait for the box to arm its guard and publish its // guard-ready report (VerifyReport, the same check membership.go makes // when the box is admitted); // 3. guard-ready: enroll it (EnrollAutomatic), which reads membership // again, itself, immediately before the enrolling write. // // A read that fails is unknown: the pass asks again next time, with the // same attempt, and its wait does not grow. Only the cloud's positive // answer that the instance is not a current member (ErrNotMember), or a // read that was over a minute old at its write (ErrMembershipStale), // counts against the attempt: the pass then waits Patience before it // begins another, doubling each time, so the record of a box the group // has already replaced stops costing roster writes and describe calls. // The invitation never changes. // // The box takes its group's role in the enrolling write (enroll.go). Only // a deployment whose host floor is published is enrolled automatically: // a box can arm its guard only under one. // // Nothing here names a provider beyond the identity the roster already // rests on (H26, §25): the record is AWS's signed instance document, as // for every enrolled box on main, and Members is an interface. // AutoEnroller is the pass's configuration and its memory of attempts. type AutoEnroller struct { Directory Directory Signer release.Signer // Evidence reads boxes' reports and release identities (the console's // ExactEvidence). Evidence Evidence // Members is the console's own read of the cloud's group membership // (awsmembership in production). Members MembershipReader // Certs verify identity records (boxid.EmbeddedCerts in production). Certs map[string]*x509.Certificate // Namespace and Pool are what a box is enrolled into: the console's // own namespace, and the default pool. Namespace, Pool string Now func() time.Time // Patience is how long the pass waits after an attempt is refused // before it begins another. Each refused attempt doubles the wait, up // to 32 times Patience, so a record whose instance is gone (a box the // group replaced while it was joining) costs the roster a write an // hour, not one a minute. An invitation resets it. Patience time.Duration // Logf records each membership read's request ids (C2's audit: the // roster has no field for them). nil is silent. Logf func(format string, args ...any) attempts map[string]attemptSeen } // attemptSeen is what the pass remembers of a box's current attempt: its // id, when the pass first saw it, how many attempts it has begun for the // box since the last invitation, and why the cloud refused the current // one ("" while it has not). Memory only: a console restart starts every // box afresh. type attemptSeen struct { id string since time.Time tries int refused string } // wait is how long a refused attempt is kept after tries attempts. func (e *AutoEnroller) wait(tries int) time.Duration { return e.Patience << min(max(tries-1, 0), 5) } // EnrollStep is what one pass did, or is waiting for, for one box. type EnrollStep struct { Box string Step string } // ErrNoFloor is a deployment with no published host floor: no box can arm // its guard, so none is enrolled automatically. var ErrNoFloor = errors.New("hostactors: no host floor is published, so no box is enrolled automatically") // EnrollPass takes one step for every box whose identity record is // published and that the roster does not list. It returns the steps in // reference order, and the first error that stopped the pass as a whole. func (e *AutoEnroller) EnrollPass(ctx context.Context) ([]EnrollStep, error) { o, floor, err := e.inputs(ctx) if err != nil { return nil, err } var steps []EnrollStep keys, err := e.Directory.Store.List(ctx, boxid.Prefix) if err != nil { return nil, err } var refs []string for _, k := range keys { ref, ok := strings.CutSuffix(strings.TrimPrefix(k, boxid.Prefix), ".json") if !ok || strings.Contains(ref, "/") { continue } if _, listed := o.Roster.Boxes[ref]; !listed { refs = append(refs, ref) } } sort.Strings(refs) if e.attempts == nil { e.attempts = map[string]attemptSeen{} } for _, ref := range refs { next, step := e.step(ctx, o, floor, ref) if next.ETag != "" { o = next } steps = append(steps, EnrollStep{Box: ref, Step: step}) } for ref := range e.attempts { if !slices.Contains(refs, ref) { delete(e.attempts, ref) } } return steps, nil } // inputs reads the roster and the host floor it was published under. func (e *AutoEnroller) inputs(ctx context.Context) (Observed, hostfloor.Verified, error) { o, err := e.Directory.Read(ctx) if err != nil { return Observed{}, hostfloor.Verified{}, err } if o.Roster.Publication != "published" { return Observed{}, hostfloor.Verified{}, ErrNoFloor } floor, err := hostfloor.Read(ctx, e.Directory.Store, e.Directory.PublicKey, e.Directory.Bucket) if err != nil { return Observed{}, hostfloor.Verified{}, err } if floor.Digest != o.Roster.FloorSHA256 { return Observed{}, hostfloor.Verified{}, ErrRoster } return o, floor, nil } // step takes one box's next step. It returns the roster it wrote, if it // wrote one, and what it did or waits for. func (e *AutoEnroller) step(ctx context.Context, o Observed, floor hostfloor.Verified, ref string) (Observed, string) { d, now := e.Directory, e.Now() raw, err := boxid.Load(ctx, d.Store, ref) if err != nil { return Observed{}, fmt.Sprintf("its identity record could not be read: %v", err) } name, region, err := groupRegion(o.Roster, raw) if err != nil { return Observed{}, err.Error() } if !slices.Contains(region.Namespaces, e.Namespace) || !slices.Contains(region.Pools, e.Pool) { return Observed{}, fmt.Sprintf("%s does not serve namespace %s and pool %s", name, e.Namespace, e.Pool) } v := boxid.Verifier{Account: region.Account, Certs: e.Certs} expect := boxid.Expect{Box: ref, AWSRegion: region.AWSRegion} for other, b := range o.Roster.Boxes { expect.Roster = append(expect.Roster, boxid.Enrolled{Box: other, Cloud: b.Cloud}) } ev, err := v.Enroll(raw, expect) if err != nil { return Observed{}, fmt.Sprintf("its identity record does not verify: %v", err) } instance := ev.Cloud.InstanceID // A group box publishes under the reference its instance derives; // the pass derives it again and never takes the key the box chose // (census ruling region-ha-2b). if ref != GroupBoxReference(region.Account, instance) { return Observed{}, "its record is not under its group reference, so it is not enrolled automatically" } in := EnrollmentRequest{Reference: ref, Namespace: e.Namespace, Label: ref, Region: name, Pool: e.Pool, AWSRegion: region.AWSRegion} // The invitation is only read here, once, and verified for this // record (enrollmentNonce writes nothing). Invite is get-or-create, so // it runs only on the bound store after a membership read // (confirmAndWrite): an invitation that vanished between two reads is // never minted again without one. nonce, err := d.enrollmentNonce(ctx, in, raw) invited := err == nil switch { case err == nil, errors.Is(err, os.ErrNotExist): case errors.Is(err, ErrRoster): return Observed{}, fmt.Sprintf("its invitation is not for this record: %v", err) default: return Observed{}, fmt.Sprintf("its invitation could not be read: %v", err) } if invited { // The report is read under the reference derived from the // attempt's proven instance (PrivateBoxReport, Group set). rep, err := e.Evidence.BoxReport(ctx, ref, Box{Namespace: e.Namespace, Label: ref, Group: region.GroupName, Cloud: ev.Cloud}) if err != nil { return Observed{}, "waiting for its guard-ready report" } if err := VerifyReport(ctx, e.Evidence, d.PublicKey, floor, rep, nonce, instance, now); err != nil { return Observed{}, "waiting for its guard-ready report" } } a, ok := o.Roster.EnrollmentAttempts[ref] usable := ok && !a.Consumed && a.InstanceID == instance && a.EvidenceSHA256 == hostfloor.Digest(raw) && a.Region == name && a.Account == region.Account && a.AWSRegion == region.AWSRegion && a.GroupName == region.GroupName && a.GroupARN == region.GroupARN seen := e.attempts[ref] if usable { if seen.id != a.AttemptID { seen = attemptSeen{id: a.AttemptID, since: now, tries: seen.tries} e.attempts[ref] = seen } if seen.refused != "" { // The cloud refused this attempt: keep it until its wait is // over, asking nothing more, then begin another. if now.Sub(seen.since) < e.wait(seen.tries) { return Observed{}, "not confirmed (" + seen.refused + "); waiting before another attempt" } } else { in.AttemptID, in.ConfirmationID = a.AttemptID, a.ConfirmationID next, step, err := e.confirmAndWrite(ctx, o, v, in, raw, instance, invited, name, region) if err == nil { if !invited { seen.tries = 0 e.attempts[ref] = seen } else { delete(e.attempts, ref) } return next, step } if !errors.Is(err, ErrNotMember) && !errors.Is(err, ErrMembershipStale) { // Unknown: a failed read, or a write that failed. The same // attempt is asked again next time; the wait does not grow. return Observed{}, step } seen.refused, seen.since = err.Error(), now e.attempts[ref] = seen return Observed{}, "not confirmed (" + seen.refused + "); waiting before another attempt" } } next, a, err := d.BeginEnrollmentAttempt(ctx, e.Signer, o, in, raw, instance) if err != nil { return Observed{}, fmt.Sprintf("an attempt could not be begun: %v", err) } e.attempts[ref] = attemptSeen{id: a.AttemptID, since: now, tries: seen.tries + 1} return next, "began an attempt" } // confirmAndWrite reads the box's membership and makes the write it // allows: the invitation, or, for a box already invited and guard-ready, // the enrollment. The membership read is the console's own, made here, // immediately before the write. func (e *AutoEnroller) confirmAndWrite(ctx context.Context, o Observed, v boxid.Verifier, in EnrollmentRequest, raw []byte, instance string, invited bool, name string, region Region) (Observed, string, error) { d := e.Directory if invited { next, ev, err := d.EnrollAutomatic(ctx, e.Signer, o, v, in, raw, instance, e.Members, e.Now) // A write the deadline left unknown (ErrWriteDeadline) is settled // by the next pass's roster read: a box the roster lists is // enrolled, and the pass has nothing more to do for it; one it // does not list goes on with the same attempt, whose wait has not // grown. e.logRead(in, instance, ev, "enrolled", err) if err != nil { return Observed{}, fmt.Sprintf("it could not be enrolled: %v", err), err } return next, fmt.Sprintf("enrolled in %s with the group's role, %s", name, RoleName(region.Role)), nil } // checkConfirmed returns the confirmation its read built, and the // write is bound to that, never to one rebuilt from exported evidence. _, c, err := d.checkConfirmed(ctx, o, v, in, raw, instance, e.Members, e.Now) ev := c.evidence() if err != nil { e.logRead(in, instance, ev, "", err) return Observed{}, fmt.Sprintf("its membership could not be confirmed: %v", err), err } // Invite is main's and sealed. It runs on the store bound to the // read's deadline (writeBound), the single enforcement point: its // record read, its signature's Put and any read-back are refused once // the read is a minute old; a call in flight then, or still unanswered // 15 seconds after it began, is cut off as unknown, and so is one that // answers only after the deadline. The next pass settles it from the // bucket. // Nothing that waits sits between the read and this call. bd, bctx, cancel := writeBound(ctx, d, c, e.Now) defer cancel() _, err = bd.Invite(bctx, e.Signer, in.Reference, raw) e.logRead(in, instance, ev, "invited", err) if err != nil { return Observed{}, fmt.Sprintf("it could not be invited: %v", err), err } return Observed{}, "invited", nil } // logRead records one membership read: the box, its attempt, its instance, // the cloud's two request ids and the read's time, and what came of it // (C2's audit: the roster has no field for them): done on success, // "refused: " and the reason for the cloud's positive answer that counts // against the attempt (ErrNotMember, ErrMembershipStale), and "unknown: " // and the reason for everything else. A call that failed before the cloud // answered has no request ids and says so. A check that failed before // any read was made (no read time) is not a membership read and is not // logged here. func (e *AutoEnroller) logRead(in EnrollmentRequest, instance string, ev MembershipEvidence, done string, err error) { if ev.ObservedAt.IsZero() { return } outcome := done if err != nil { outcome = "unknown: " + err.Error() if errors.Is(err, ErrNotMember) || errors.Is(err, ErrMembershipStale) { outcome = "refused: " + err.Error() } } e.logOutcome(in, instance, ev, outcome) } // logOutcome is one membership log line (S5): the box, its attempt, its // instance, both request ids and the read's time, and the outcome. func (e *AutoEnroller) logOutcome(in EnrollmentRequest, instance string, ev MembershipEvidence, outcome string) { if e.Logf == nil { return } ids := "none" if len(ev.RequestIDs) > 0 { ids = strings.Join(ev.RequestIDs, ", ") } e.Logf("console: automatic box enrollment: membership read for %s (attempt %s, instance %s, request ids %s, read at %s): %s", in.Reference, in.AttemptID, instance, ids, ev.ObservedAt.Format(time.RFC3339), outcome) } // groupRegion is the one roster region whose group could have launched // the box the record names: its account and AWS region, with a group. // The record is read here only to choose; Verifier.Enroll then proves it. func groupRegion(r Roster, raw []byte) (string, Region, error) { var rec boxid.Record if json.Unmarshal(raw, &rec) != nil { return "", Region{}, errors.New("its identity record is not JSON") } doc, err := base64.StdEncoding.DecodeString(rec.IdentityDocument) if err != nil { return "", Region{}, errors.New("its identity document is not base64") } var id struct { AccountID string `json:"accountId"` Region string `json:"region"` } if json.Unmarshal(doc, &id) != nil { return "", Region{}, errors.New("its identity document is not JSON") } var names []string for name, g := range r.Regions { if g.GroupName != "" && g.Account == id.AccountID && g.AWSRegion == id.Region { names = append(names, name) } } switch len(names) { case 0: return "", Region{}, fmt.Errorf("no region's group runs in %s for account %s", id.Region, id.AccountID) case 1: return names[0], r.Regions[names[0]], nil } sort.Strings(names) return "", Region{}, fmt.Errorf("the regions %s all have a group in %s", strings.Join(names, ", "), id.Region) }