package host import ( "context" "encoding/base64" "encoding/json" "errors" "fmt" "net/http" "os" "runtime" "strings" "sync/atomic" "time" "github.com/carlosframework/platform/internal/boxid" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/status" "github.com/carlosframework/platform/internal/version" ) // Joining and hosting (region HA spec 2 ยง5, A1-2, A2-4). // // With CARLOS_HOST_CLAIM and CARLOS_REGION_HA both on, the agent chooses // its mode at every start, from three facts and never from one file's // presence. It hosts only when all three hold: // // - /etc/carlos/host-actor.json names this box; // - the signed roster has an active entry for that enrollment and // instance; // - the host lifecycle guard is armed under the roster's floor, with // this process's own binary one it admits. // // Anything else is joining. A joining box hosts nothing, publishes no // edges file, and answers the load balancer 503 (joining.go). Joining // resumes from whatever it finds, and each stage names the next step: // // - no invitation: the console has not invited the box yet; // - invited: root arms the guard (carlos system join, joinroot.go); // - guard installed: the box publishes its guard-ready report; // - guard-ready reported: the console enrolls the box; // - held: the roster names this box, but not as an active entry for its // enrollment and armed guard, or, for a box its region's group // launched, the instance was stopped and started since the identity // record it was enrolled on (C3: it must prove its group membership // again, and never hosts on its old enrollment). Nothing automatic // can fix that: a held group box answers 503 and the group replaces // it; an operator sorts out any other. // // A fact that cannot be read is never taken as its negative: the mode is // then unknown (errModeUnknown), and the caller asks again later. type agentMode string const ( modeHosting agentMode = "hosting" modeJoining agentMode = "joining" ) type joinStage string const ( stageNoInvitation joinStage = "no invitation" stageInvited joinStage = "invited" stageGuardInstalled joinStage = "guard installed" stageGuardReady joinStage = "guard-ready reported" stageHeld joinStage = "held" ) // modeDecision is one start's answer. Stage and Why are joining's. type modeDecision struct { Mode agentMode Stage joinStage Box string Why string } // errModeUnknown is a fact that could not be read: no mode is chosen. var errModeUnknown = errors.New("agent mode unknown") func modeUnknown(what string, err error) error { return fmt.Errorf("%w: %s could not be read: %v", errModeUnknown, what, err) } // reportFreshness is how old a guard-ready report may be and still count: // the console's own limit (hostactors.VerifyReport). const reportFreshness = 15 * time.Minute // hostActorPath is the box's enrollment, written by guard-arm. const hostActorPath = "/etc/carlos/host-actor.json" // modeFacts are where selectMode reads from; production's are // productionModeFacts. type modeFacts struct { // box is this box's reference: the label its identity record and its // status report are published under. box string // enrollment reads host-actor.json; os.ErrNotExist when there is none. enrollment func() (hostfloor.Enrollment, error) // roster reads the signed roster. roster func(context.Context) (hostactors.Roster, error) // guard is the guard's view of this process (hostfloor.Disk.Running). // hostfloor.ErrFloor or os.ErrNotExist: not armed, or not for this // binary. guard func() (hostfloor.Running, error) // invited reports whether the console has written box's invitation. invited func(context.Context, string) (bool, error) // report reads box's own published status report; os.ErrNotExist when // there is none. report func(context.Context, string) (status.Box, error) // relaunch says why this instance is not the launch the roster entry // row was enrolled on ("" when it is): asked only of a box its // region's group launched (row.Group set). relaunch func(context.Context, hostactors.Box) (string, error) // marked is host.env's group marker (CARLOS_BOX_REFERENCE=group), and // region its CARLOS_REGION: values from the parameter store, which // select this box's authority, so they are checked against the signed // roster before the box hosts (crossCheck). marked bool region string now func() time.Time } // selectMode chooses this start's mode. func selectMode(ctx context.Context, f modeFacts) (modeDecision, error) { d := modeDecision{Mode: modeJoining, Box: f.box} e, err := f.enrollment() if errors.Is(err, os.ErrNotExist) { invited, err := f.invited(ctx, f.box) if err != nil { return d, modeUnknown("the invitation", err) } d.Stage = stageNoInvitation if invited { d.Stage = stageInvited } return d, nil } if err != nil { return d, modeUnknown(hostActorPath, err) } if e.Box != f.box { d.Stage, d.Why = stageHeld, fmt.Sprintf("%s names %s, not this box (%s)", hostActorPath, e.Box, f.box) return d, nil } r, err := f.roster(ctx) if err != nil { return d, modeUnknown("the roster", err) } g, err := f.guard() if err != nil && !errors.Is(err, hostfloor.ErrFloor) && !errors.Is(err, os.ErrNotExist) { return d, modeUnknown("the host lifecycle guard", err) } armed := err == nil && g.GuardReady && g.FloorSHA256 == r.FloorSHA256 row, listed := r.Boxes[e.Box] switch { case listed && row.Enrollment == e.Enrollment && row.Cloud.InstanceID == e.InstanceID && row.State == "active" && armed: if why := crossCheck(f, row); why != "" { d.Stage, d.Why = stageHeld, why return d, nil } if row.Group != "" { why, err := f.relaunch(ctx, row) if err != nil { return d, modeUnknown("this instance's launch", err) } if why != "" { d.Stage, d.Why = stageHeld, why return d, nil } } d.Mode, d.Stage = modeHosting, "" case listed: d.Stage = stageHeld d.Why = fmt.Sprintf("the roster's entry for %s is %s, enrollment %s on %s; this box holds enrollment %s on %s, guard armed %t", e.Box, row.State, row.Enrollment, row.Cloud.InstanceID, e.Enrollment, e.InstanceID, armed) case !armed: // guard-arm writes host-actor.json before it persists the armed // policy: an arm that was cut off is still the invited stage. d.Stage = stageInvited default: rep, err := f.report(ctx, f.box) if err != nil && !errors.Is(err, os.ErrNotExist) { return d, modeUnknown("this box's status report", err) } d.Stage = stageGuardInstalled if err == nil && guardReadyReport(rep, e, g, f.now()) { d.Stage = stageGuardReady } } return d, nil } // crossCheck says why host.env's authority-selecting values disagree with // the roster entry the box would host under ("" when they agree). The // group marker must match whether the console enrolled the box from a // group, and a group box's CARLOS_REGION must name the region it was // enrolled in. It is made here, at the agent's use, and not when // host-env write assembles the file: at boot the box has no roster entry // to check against (it is not enrolled yet), and before enrollment // neither value grants anything, since the console derives the box's // reference and region from its proven instance itself. A box an operator // enrolled keeps its own CARLOS_REGION, unchecked, as on main. func crossCheck(f modeFacts, row hostactors.Box) string { switch { case f.marked && row.Group == "": return "host.env marks this box as one its region's group launched, but the roster enrolled it by hand; it does not host" case !f.marked && row.Group != "": return "the roster enrolled this box from group " + row.Group + ", but host.env does not mark it so; it does not host" case f.marked && f.region == "": return "host.env names no region, but the roster enrolled this box in " + row.Region + "; it does not host" case f.marked && f.region != row.Region: return "host.env names region " + f.region + ", but the roster enrolled this box in " + row.Region + "; it does not host" } return "" } // relaunched says why current, this instance's identity record read from // IMDS now, is not the launch row was enrolled on: the record published // under the box's reference (enrolled) must be the one the roster entry // binds (its digest), and the two identity documents must name the same // pendingTime. EC2 sets pendingTime each time an instance enters the // pending state: at launch, and again at every start after a stop, but // not at a reboot. "" when they agree. func relaunched(row hostactors.Box, enrolled []byte, current boxid.Record) string { if hostfloor.Digest(enrolled) != row.EvidenceSHA256 { return "the identity record under this box's reference is not the one it was enrolled on" } var rec boxid.Record if json.Unmarshal(enrolled, &rec) != nil { return "the identity record it was enrolled on cannot be read" } then, now := recordPendingTime(rec), recordPendingTime(current) switch { case then == "" || now == "": return "an identity document names no pendingTime, so this launch cannot be matched to its enrollment" case then != now: return fmt.Sprintf("this instance was stopped and started since it was enrolled (launched %s, enrolled on the launch of %s): it must prove its group membership again, so it does not host", now, then) } return "" } // recordPendingTime is the pendingTime of a record's identity document, "" // when it has none. func recordPendingTime(rec boxid.Record) string { doc, err := base64.StdEncoding.DecodeString(rec.IdentityDocument) if err != nil { return "" } var d struct { PendingTime string `json:"pendingTime"` } if json.Unmarshal(doc, &d) != nil { return "" } return d.PendingTime } // guardReadyReport reports whether rep is this box's fresh guard-ready // report for enrollment e under the guard g. func guardReadyReport(rep status.Box, e hostfloor.Enrollment, g hostfloor.Running, now time.Time) bool { m := rep.Metrics return m != nil && m.Enrollment == e.Enrollment && m.FloorGuardReady && m.FloorSHA256 == g.FloorSHA256 && m.BinarySHA256 == g.Hash && !rep.GeneratedAt.IsZero() && now.Sub(rep.GeneratedAt) <= reportFreshness } // productionModeFacts reads the deployment store, the status store and // this box's own files. func productionModeFacts(ctx context.Context, getenv func(string) string) (modeFacts, error) { store, err := openDeployStoreEnv(ctx, getenv) if err != nil { return modeFacts{}, err } statusStore, _, err := openStatusStore(ctx, getenv) if err != nil { return modeFacts{}, err } pub, err := release.ParsePublicKey(getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY") if err != nil { return modeFacts{}, err } disk, err := floorDisk(release.OptDir()) if err != nil { return modeFacts{}, err } bucket := getenv("CARLOS_DEPLOYMENT_BUCKET") box := metricsBoxLabel(getenv, readMachineID()) if box == "" { return modeFacts{}, errors.New("this box's group reference could not be read") } imds := &http.Client{Timeout: 2 * time.Second} return modeFacts{ box: box, marked: groupReferenced(getenv), region: strings.TrimSpace(getenv("CARLOS_REGION")), enrollment: func() (hostfloor.Enrollment, error) { return hostfloor.ReadEnrollment(hostActorPath) }, roster: func(ctx context.Context) (hostactors.Roster, error) { return readGuardRoster(ctx, store, pub, bucket) }, guard: func() (hostfloor.Running, error) { return disk.Running(version.Build, runtime.GOOS+"-"+runtime.GOARCH) }, invited: func(ctx context.Context, box string) (bool, error) { _, _, err := store.Get(ctx, invitationKey(box)) if errors.Is(err, os.ErrNotExist) { return false, nil } return err == nil, err }, report: func(ctx context.Context, box string) (status.Box, error) { raw, _, err := statusStore.Get(ctx, "status/boxes/"+box+".json") if err != nil { return status.Box{}, err } var b status.Box if err := json.Unmarshal(raw, &b); err != nil { return status.Box{}, err } return b, nil }, relaunch: func(ctx context.Context, row hostactors.Box) (string, error) { enrolled, err := boxid.Load(ctx, store, box) if err != nil { return "", err } current, err := readIdentityRecord(imds, imdsBase, box, time.Now()) if err != nil { return "", err } return relaunched(row, enrolled, current), nil }, now: time.Now, }, nil } // invitationKey is where the console writes box's invitation // (hostactors.Directory.Invite). func invitationKey(box string) string { return "control/host-actor-invitations/" + box + ".json" } // agentHosting is this process's mode, decided once by RunAgent before // the agent starts (hostingGate): true for the claim agent, false while // joining. The claim wiring reports it to the load balancer's health // answer (ClaimWiring.startRegional). var agentHosting atomic.Bool