package host import ( "context" "errors" "flag" "io" "log" "net/http" "os" "os/signal" "path/filepath" "runtime" "strings" "syscall" "time" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/lbhealth" "github.com/carlosframework/platform/internal/status" "github.com/carlosframework/platform/internal/version" ) // The agent's joining mode (region HA spec 2 ยง5 step 4) and the gate that // chooses it (joinmode.go). // // A joining box publishes its signed identity record (boxRecordLoop) and, // once its guard is armed, its guard-ready report: the private status // report every hosting box publishes, carrying the guard's facts // (claimReport). It never publishes an edges file: a box an edges file // names is referenced, and automatic enrollment refuses a referenced box // (hostactors references.go, enroll.go). It opens no registry, starts no // activator and serves no instance. Its only listener is the load // balancer's health check on the edge's HTTP address, which answers 503: // lbhealth.Default is never marked hosting here. // // When the box is invited it asks root to arm the guard by touching // joinRequestMarker (carlos-system-join.path). Once a start check finds it // enrolled and armed, it returns, and RunAgent ends the process so systemd // starts the agent again, now as the claim agent through the guard. // joinEvery is how often joining checks its mode. const joinEvery = 15 * time.Second // joinNudgeEvery is how often joining re-publishes its report or re-asks // root, at most: the report stays well inside reportFreshness, and root // is not asked again while it is still arming. While the box stays // invited with its guard not armed, root is asked again at this interval, // once a minute: carlos-system-join.path fires only when the marker // changes, so a carlos system join that failed (an IMDS or store read, // say) is retried by the next touch, and by nothing else. const joinNudgeEvery = time.Minute // joinRequestMarker is the file carlos-system-join.path watches, under the // data dir. const joinRequestMarker = "system-join-requested" // joining is one joining run's steps; production's are productionJoining. type joining struct { facts modeFacts every time.Duration // serve answers the load balancer until ctx ends. serve func(ctx context.Context) error // record publishes the box's identity record, retrying until it is in // place. record func(ctx context.Context) // report publishes the guard-ready report. report func(ctx context.Context) error // requestArm asks root to arm the guard. requestArm func() error logf func(format string, args ...any) } // run joins until the box can host (the decision is returned) or ctx // ends (the last decision, and ctx's error). func (j joining) run(ctx context.Context) (modeDecision, error) { lbhealth.Default.SetHosting(false) go func() { if err := j.serve(ctx); err != nil && !errors.Is(err, http.ErrServerClosed) { j.logf("agent: joining: the load balancer health check is not answered: %v", err) } }() go j.record(ctx) var last modeDecision var lastErr string var nudged time.Time for { d, err := selectMode(ctx, j.facts) now := j.facts.now() switch { case err != nil: if err.Error() != lastErr { j.logf("agent: joining: %v; checking again", err) lastErr = err.Error() } case d.Mode == modeHosting: j.logf("agent: joining: %s is enrolled and its guard is armed; restarting as the claim agent", d.Box) return d, nil default: lastErr = "" if d.Stage != last.Stage || d.Why != last.Why { j.logf("agent: joining: %s: %s %s", d.Box, d.Stage, d.Why) nudged = time.Time{} } last = d if now.Sub(nudged) >= joinNudgeEvery { switch d.Stage { case stageInvited: if err := j.requestArm(); err != nil { j.logf("agent: joining: could not ask root to arm the guard: %v", err) } case stageGuardInstalled, stageGuardReady: if err := j.report(ctx); err != nil { j.logf("agent: joining: the guard-ready report was not published: %v", err) } } nudged = now } } select { case <-ctx.Done(): return last, ctx.Err() case <-time.After(j.every): } } } // productionJoining is joining on a real box. args are the agent's own // (edge.Run's flags): the health check listens on -http, as the edge will. func productionJoining(args []string, facts modeFacts, getenv func(string) string) joining { addr := joiningHTTPAddr(args) return joining{ facts: facts, every: joinEvery, serve: func(ctx context.Context) error { srv := &http.Server{Addr: addr, ReadHeaderTimeout: 10 * time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !lbhealth.Serve(w, r, lbhealth.Default, time.Now()) { http.NotFound(w, r) } })} go func() { <-ctx.Done(); _ = srv.Close() }() return srv.ListenAndServe() }, record: func(context.Context) { boxRecordLoop() }, report: func(ctx context.Context) error { store, _, err := openStatusStore(ctx, getenv) if err != nil { return err } p := status.Probes{MachineID: readMachineID(), InstanceID: cachedInstanceID(), Region: strings.TrimSpace(getenv("CARLOS_REGION")), Release: version.Build, Platform: runtime.GOOS + "-" + runtime.GOARCH, SystemChannel: boxSystemChannel(getenv)} claimReport(&p) box := status.CollectBox(nil, p, time.Now()) if groupReferenced(getenv) { // Under the group reference the console derives (census // ruling region-ha-2b), as statusLoop publishes it. box.Label = facts.box } return status.WriteBox(ctx, store, box) }, requestArm: func() error { return os.WriteFile(filepath.Join(dataDir(), joinRequestMarker), []byte(time.Now().UTC().Format(time.RFC3339Nano)+"\n"), 0o644) }, logf: log.Printf, } } // joiningHTTPAddr is the agent's -http flag, ":80" when it is not given. func joiningHTTPAddr(args []string) string { fs := flag.NewFlagSet("agent", flag.ContinueOnError) fs.SetOutput(io.Discard) addr := fs.String("http", ":80", "") fs.String("registry", "", "") fs.String("acme-cache", "", "") fs.String("https", "", "") fs.String("fallback", "", "") fs.Bool("dev", false, "") _ = fs.Parse(args) return *addr } // decideAgentMode is RunAgent's gate. With claim mode or region HA off // there is no joining mode, and the agent hosts as on main. Otherwise it // selects the mode, waiting out an unknown one, and on joining runs join // until the box can host or ctx ends; it reports whether this process // should go on to start the claim agent. func decideAgentMode(ctx context.Context, getenv func(string) string, open func(context.Context, func(string) string) (modeFacts, error), join func(context.Context, modeFacts, modeDecision) error, wait time.Duration, logf func(string, ...any)) bool { if !hostwrite.Enabled(getenv) || !regionHAEnabled(getenv) { return true } var last string note := func(err error) bool { if err.Error() != last { logf("agent: mode: %v; checking again", err) last = err.Error() } select { case <-ctx.Done(): return false case <-time.After(wait): return true } } for { facts, err := open(ctx, getenv) if err != nil { if !note(modeUnknown("the deployment's stores and this box's guard", err)) { return false } continue } d, err := selectMode(ctx, facts) if err != nil { if !note(err) { return false } continue } if d.Mode == modeHosting { return true } logf("agent: %s is joining (%s): it hosts nothing until it is enrolled", d.Box, d.Stage) if err := join(ctx, facts, d); err != nil && !errors.Is(err, context.Canceled) { logf("agent: joining: %v", err) } return false } } // hostingGate decides, for RunAgent, whether this process starts the claim // agent, and records the answer in agentHosting. SIGTERM or SIGINT while // joining ends the process cleanly. func hostingGate(args []string) bool { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt) defer stop() hosting := decideAgentMode(ctx, os.Getenv, productionModeFacts, func(ctx context.Context, facts modeFacts, _ modeDecision) error { _, err := productionJoining(args, facts, os.Getenv).run(ctx) return err }, joinEvery, log.Printf) agentHosting.Store(hosting) return hosting } // groupReferenceBackoff is how long agentEntry waits after the n-th // failed read of a group box's reference (n from 1): a second, doubling, // at most 30 seconds. func groupReferenceBackoff(n int) time.Duration { d := time.Second for ; n > 1 && d < 30*time.Second; n-- { d *= 2 } return min(d, 30*time.Second) } // agentEntry is RunAgent's order. A box its region's group launched (the // CARLOS_BOX_REFERENCE=group marker) first resolves its group reference, // whatever the claim and region HA switches say: metricsBoxLabel answers // "" until it resolves, and the loops runAgent starts (the edges file, // the registry snapshot, metrics, usage, the layout box, the moves pass) // read the label once and keep it for the process. It asks IMDS again // with backoff, logging each failure's cause, until it answers or the // process is told to stop. Any other box resolves nothing. Then the gate // (hostingGate) decides whether this process starts the agent (run). func agentEntry(getenv func(string) string, resolve func() (string, error), backoff func(int) time.Duration, logf func(string, ...any), gate func() bool, run func()) { if groupReferenced(getenv) { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt) ok := awaitGroupReference(ctx, resolve, backoff, logf) stop() if !ok { return } } if !gate() { return } run() } // awaitGroupReference asks resolve until it answers, waiting backoff(n) // after the n-th failure; false when ctx ends first. func awaitGroupReference(ctx context.Context, resolve func() (string, error), backoff func(int) time.Duration, logf func(string, ...any)) bool { for n := 1; ; n++ { ref, err := resolve() if err == nil { logf("agent: this box's group reference is %s", ref) return true } wait := backoff(n) logf("agent: this box's group reference could not be read from its identity document (attempt %d): %v; the agent starts once it is, asking again in %s", n, err, wait) select { case <-ctx.Done(): return false case <-time.After(wait): } } }