package main // The control daemon: the box's localhost-only provisioning surface, anticipated // in docs/instance-router.md as "the god surface … never exposed on a served // hostname". It executes the same creation core as `lchat add-instance` // (addInstance in control.go) on behalf of the unprivileged cloud provisioner // service (cloud.go), which is the public, policy-enforcing face at // cloud.elevenmessenger.com. The split exists because creation needs root // (systemctl, /opt/lchat writes) while everything network-facing runs sandboxed // as the lchat user. See docs/cloud-provisioner.md. // // Auth: every unit on the box shares the lchat uid, so socket permissions can't // distinguish the cloud service from a compromised instance. The gate is a // bearer token in a root-only file (/etc/lchat/control.token, 0600) that the // cloud unit alone receives via systemd LoadCredential. import ( "crypto/subtle" "encoding/json" "errors" "flag" "fmt" "io" "log" "net" "net/http" "os" "os/user" "path/filepath" "strconv" "strings" "sync" "syscall" "time" ) // controlTokenPath is root-owned (0600): readable by controld and loadable into // the cloud unit by systemd, but never by anything running as the lchat user. const controlTokenPath = "/etc/lchat/control.token" // controlSocketPath is the control daemon's unix socket: where controld listens // (runControld's -socket default below) AND where the control self-update dials // /healthz to confirm a freshly-swapped control binary came up. One source of // truth for both, and a var so tests can point the health probe at a temp socket. var controlSocketPath = envOr("LCHAT_CONTROL_SOCKET", "/run/lchat/control.sock") // freeDiskBytes reports the free space where instance DBs live. A var so tests // can simulate a full disk without filling one. var freeDiskBytes = func(path string) (uint64, error) { var st syscall.Statfs_t if err := syscall.Statfs(path, &st); err != nil { return 0, err } return uint64(st.Bavail) * uint64(st.Bsize), nil } // controld is the daemon's state: the registry it counts and creates into, the // capacity policy, and the token gate. Creation is serialized (mu) so the // capacity check and the create it guards are atomic. type controld struct { reg *Registry tokenHash string maxInstances int minFreeGB int // hibernateCreates (env LCHAT_CONTROL_HIBERNATE_CREATES=1) makes every // provisioner-created space born hibernating (#550): registered + routed, // env file + staged binary in place, but no systemd unit and no resident // process — the router's activator wakes it on demand. Off by default: the // flip to hibernate-by-default is this one env var on the box, not a code // deploy. Direct `lchat add-instance` is unaffected either way (its // --hibernate flag stays an explicit per-invocation choice). Requires the // router to run with LCHAT_HIBERNATE_BUCKET, or created spaces never answer. hibernateCreates bool // maxRegistered caps TOTAL instance rows (materialized + hibernating) when // hibernateCreates is on — a sanity ceiling on registry rows/env files, not // a resource cap (hibernating rows hold no process). <=0 defaults to // 10x maxInstances. maxRegistered int // homeOrigin (env LCHAT_CONTROL_HOME_ORIGIN) is the box's default // LCHAT_HOME_ORIGIN for provisioner creates that don't name one. The home // origin is an environment property (which home server fronts this fleet), // not per-space policy — without this, a non-production box's spaces point // at the compiled production default, so their backup nudges and the demo // banner's live status read ask the wrong home. Empty = keep the compiled // default (production boxes need nothing). homeOrigin string mu sync.Mutex } // controlCreateReq mirrors addInstanceOpts across the socket. AdminToken is the // migration path (import an existing token) — unused by the cloud provisioner, // kept so the control API can do everything the CLI can. type controlCreateReq struct { Host string `json:"host"` AdminToken string `json:"admin_token,omitempty"` AppStoreURL string `json:"app_store_url,omitempty"` HomeOrigin string `json:"home_origin,omitempty"` PushRelayURL string `json:"push_relay_url,omitempty"` PushRelayToken string `json:"push_relay_token,omitempty"` Entitlement string `json:"entitlement,omitempty"` // Name seeds the space's display name (→ LCHAT_GROUP_NAME; the create flow // asks for it up front). Sanitized + capped by addInstance; optional. Name string `json:"name,omitempty"` // Replication classes the space at birth (registry.go's durability ladder) // so a free space never spends a single awake window on the streaming rung. Replication string `json:"replication,omitempty"` // HomeRegion is the space's placement region (multi-edge Phase 1, // docs/multi-edge.md): the mesh owns a space on a box whose LCHAT_MESH_REGION // matches its home_region (mesh.candidatesFor), so relocating a space is just // a home_region flip. The cloud provisioner stamps its own LCHAT_CLOUD_REGION // here at create; "" (the default) keeps the space region-agnostic — hashed // across the whole fleet exactly as before regions existed. HomeRegion string `json:"home_region,omitempty"` } // controlErr is the wire error shape: a stable machine code plus a human // message, so callers never parse message strings. type controlErr struct { Code string `json:"error"` Message string `json:"message"` } func writeControlErr(w http.ResponseWriter, status int, code, msg string) { writeJSON(w, status, controlErr{Code: code, Message: msg}) } func (c *controld) handler() http.Handler { mux := http.NewServeMux() // Unauthenticated, state-free liveness+build probe. The box's own control // self-update (controlSelfUpdate in updater.go) swaps THIS binary in place and // then dials /healthz to confirm the new build actually came up before it // commits — rolling the swap back if it can't. It has to answer the instant the // HTTP listener binds, so it deliberately touches no DB/registry state a // half-started process might still lack; it reveals only the compiled-in build // string, and only over the root-only control socket. No token: the swap probe // runs in the same already-root updater and there's nothing here worth gating. mux.HandleFunc("GET /healthz", c.healthzHandler) mux.HandleFunc("POST /instances", c.auth(c.createHandler)) mux.HandleFunc("DELETE /instances/{host}", c.auth(c.deleteHandler)) mux.HandleFunc("GET /instances/{host}/stats", c.auth(c.statsHandler)) mux.HandleFunc("POST /instances/{host}/quota", c.auth(c.quotaHandler)) mux.HandleFunc("POST /instances/{host}/meter", c.auth(c.meterHandler)) mux.HandleFunc("POST /instances/{host}/bot", c.auth(c.botHandler)) mux.HandleFunc("POST /instances/{host}/replication", c.auth(c.replicationHandler)) mux.HandleFunc("POST /instances/{host}/materialize", c.auth(c.materializeHandler)) return mux } // healthzHandler answers the control self-update's health probe with the running // control binary's build id. It touches nothing on c (no registry, no disk) so a // process that has only just bound its listener still answers — see the route // registration in handler() for the full why, including why it's unauthenticated. func (c *controld) healthzHandler(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, map[string]any{"build": buildVersion}) } // replicationHandler records one instance's replication class on its registry // route (the durability ladder — registry.go) and reloads the router so the // next wake reads it. The class is wake-time config, so nothing is restarted // and no sleeping space is woken: an awake instance keeps its current sync // tick until it next sleeps. Idempotent, and deliberately reload-free when the // class already matches — the cloud sweep re-pushes every pass, and a router // reload per space per sweep would be pointless churn. func (c *controld) replicationHandler(w http.ResponseWriter, r *http.Request) { host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) if !validHost(host) { writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") return } var req struct { Class string `json:"class"` } if err := json.NewDecoder(io.LimitReader(r.Body, 4<<10)).Decode(&req); err != nil { writeControlErr(w, http.StatusBadRequest, "bad_request", "invalid JSON body") return } class := strings.ToLower(strings.TrimSpace(req.Class)) if class == "" || !validReplication(class) { writeControlErr(w, http.StatusBadRequest, "bad_request", fmt.Sprintf("class must be %s, %s or %s", replicationStream, replicationBatched, replicationDaily)) return } rt, err := c.reg.get(host) if err != nil { log.Printf("controld: replication %s: lookup: %v", host, err) writeControlErr(w, http.StatusInternalServerError, "lookup_failed", "db error") return } if rt == nil || rt.Kind != "instance" { writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance") return } if rt.Replication == class { writeJSON(w, http.StatusOK, map[string]any{"ok": true, "changed": false}) return } if err := c.reg.setReplication(host, class); err != nil { log.Printf("controld: replication %s: %v", host, err) writeControlErr(w, http.StatusInternalServerError, "replication_failed", err.Error()) return } // Best-effort: the registry write is durable either way, and the sweep's // next push finds changed=false and stops — so a failed reload here must be // loud (it's the only convergence signal left until something else reloads // the router). reloaded := true if err := systemctl("reload", "lchat-router"); err != nil { reloaded = false log.Printf("controld: replication %s → %s: router reload FAILED: %v (class is stored but inert until a reload)", host, class, err) } log.Printf("controld: replication %s → %s (reloaded=%v)", host, class, reloaded) writeJSON(w, http.StatusOK, map[string]any{"ok": true, "changed": true, "reloaded": reloaded}) } // materializeHandler stages the running control binary and points host's live // symlink at it, if not already done. This is the privileged half of a route's // first wake on a box that never locally created it (control.go's // materializeInstance normally does this at add-instance time, but a // mesh-replicated route — meshreplicate.go — only ever inserts a registry row, // never a symlink, since the receiving box didn't create the space). // activator.go's s3Backend.start can't do this itself: the router runs // sandboxed (ProtectSystem=strict, no write access outside /data and // /run/lchat — see docs/instance-router.md § Hardening), so only this // always-unsandboxed root daemon can. Idempotent: safe to call even if the // host is already materialized (pointLive happily repoints at the same or a // newer version), and safe to call for a host this box didn't create — the // row already exists (checked below) is the only prerequisite. func (c *controld) materializeHandler(w http.ResponseWriter, r *http.Request) { host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) if !validHost(host) { writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") return } if c.reg != nil { rt, err := c.reg.get(host) if err != nil { log.Printf("controld: materialize %s: lookup: %v", host, err) writeControlErr(w, http.StatusInternalServerError, "lookup_failed", "db error") return } if rt == nil { writeControlErr(w, http.StatusNotFound, "unknown_host", "no such route") return } } ver, err := ensureSelfStaged() if err != nil { log.Printf("controld: materialize %s: stage: %v", host, err) writeControlErr(w, http.StatusInternalServerError, "stage_failed", err.Error()) return } if err := pointLive(host, ver); err != nil { log.Printf("controld: materialize %s: point live: %v", host, err) writeControlErr(w, http.StatusInternalServerError, "point_live_failed", err.Error()) return } writeJSON(w, http.StatusOK, map[string]any{"ok": true, "version": ver}) } // quotaHandler applies a storage cap to one instance: upsert // LCHAT_BLOB_QUOTA_BYTES in its env file (0 = back to the compiled default), // mirror the billing entitlement alongside it when the caller sends one, and // bounce a unit-managed instance so it applies now. A hibernating instance's // env is read at spawn, so its cap simply applies on next wake — never wake a // space to limit it. func (c *controld) quotaHandler(w http.ResponseWriter, r *http.Request) { host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) if !validHost(host) { writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") return } var req struct { Bytes int64 `json:"bytes"` Entitlement string `json:"entitlement"` } if err := json.NewDecoder(io.LimitReader(r.Body, 4<<10)).Decode(&req); err != nil || req.Bytes < 0 { writeControlErr(w, http.StatusBadRequest, "bad_request", "bytes required (>= 0; 0 restores the default)") return } if req.Entitlement != "" && req.Entitlement != "free" && req.Entitlement != "paid" { writeControlErr(w, http.StatusBadRequest, "bad_request", `entitlement must be "free" or "paid" (or omitted)`) return } envPath := filepath.Join(instanceDataDir, host+".env") if err := setInstanceQuota(envPath, req.Bytes, req.Entitlement); err != nil { if os.IsNotExist(err) { writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance") return } log.Printf("controld: quota %s: %v", host, err) writeControlErr(w, http.StatusInternalServerError, "quota_failed", err.Error()) return } // The router (service user) reads a hibernating instance's env at wake; the // file was just rewritten as root, so hand it back (same rule as creation). if err := matchOwner(envPath, instanceDataDir); err != nil { log.Printf("controld: quota %s: env ownership: %v", host, err) } restarted := false hibernating := false if c.reg != nil { if rt, err := c.reg.get(host); err == nil && rt != nil && rt.Hibernate { hibernating = true } } if !hibernating { // try-restart: bounce only if it's running (data-safe; seconds). if err := systemctl("try-restart", "lchat@"+host); err == nil { restarted = true } else { log.Printf("controld: quota %s: restart: %v (cap applies on next start)", host, err) } } log.Printf("controld: quota %s → %d bytes (restarted=%v)", host, req.Bytes, restarted) writeJSON(w, http.StatusOK, map[string]any{"ok": true, "restarted": restarted}) } // meterHandler hands one instance its usage-pipe credentials (#857): upsert // LCHAT_METER_URL + LCHAT_METER_TOKEN in the env and bounce the unit so they // take effect — the same shape as quotaHandler, minus the registry-quota // bookkeeping. Both empty clears the pair (metering off). func (c *controld) meterHandler(w http.ResponseWriter, r *http.Request) { host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) if !validHost(host) { writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") return } var req struct { URL string `json:"url"` Token string `json:"token"` } if err := json.NewDecoder(io.LimitReader(r.Body, 4<<10)).Decode(&req); err != nil || (req.URL == "") != (req.Token == "") { writeControlErr(w, http.StatusBadRequest, "bad_request", "url and token required together (both empty clears)") return } envPath := filepath.Join(instanceDataDir, host+".env") if err := setInstanceMeter(envPath, req.URL, req.Token); err != nil { if os.IsNotExist(err) { writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance") return } log.Printf("controld: meter %s: %v", host, err) writeControlErr(w, http.StatusInternalServerError, "meter_failed", err.Error()) return } // Root just rewrote the env; the router (service user) reads it at wake. if err := matchOwner(envPath, instanceDataDir); err != nil { log.Printf("controld: meter %s: env ownership: %v", host, err) } restarted := false hibernating := false if c.reg != nil { if rt, err := c.reg.get(host); err == nil && rt != nil && rt.Hibernate { hibernating = true // picks the env up on its next wake } } if !hibernating { if err := systemctl("try-restart", "lchat@"+host); err == nil { restarted = true } else { log.Printf("controld: meter %s: restart: %v (applies on next start)", host, err) } } log.Printf("controld: meter %s → %v (restarted=%v)", host, req.URL != "", restarted) writeJSON(w, http.StatusOK, map[string]any{"ok": true, "restarted": restarted}) } // botHandler attaches a bot sidecar to one instance — the provisioner's half // of the AI opt-in (#858, docs/ai-provisioning.md phase 2). Cloud spaces get // **Eleven only** (the product voice; admins add further agents through the // admin console, which the managed sidecar picks up). Idempotent: a host // whose bot env is already staged answers ok without touching anything, so // re-opt-ins and admin toggle flips are safe to signal every time. // // The model key comes from controld's own environment (ANTHROPIC_API_KEY) — // the sidecar holds it, never the instance (CLAUDE.md's bot invariant). With // no key configured the sidecar still attaches deterministic-only and the // response says so, so the operator gap is visible rather than silent. func (c *controld) botHandler(w http.ResponseWriter, r *http.Request) { host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) if !validHost(host) { writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") return } hibernating := false if c.reg != nil { rt, err := c.reg.get(host) if err != nil || rt == nil || rt.Kind != "instance" { writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance") return } hibernating = rt.Hibernate } apiKey := strings.TrimSpace(os.Getenv("ANTHROPIC_API_KEY")) msg, err := attachBot(host, hibernating, []botIdentitySpec{elevenIdentity}, apiKey, "", envOr("LCHAT_BOT_UNIT", "lchat-bot@")) if err != nil { if errors.Is(err, os.ErrNotExist) { // No instance env file: a registered-but-never-materialized host. writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance") return } log.Printf("controld: attach-bot %s: %v", host, err) writeControlErr(w, http.StatusInternalServerError, "attach_failed", err.Error()) return } log.Printf("controld: attach-bot %s — %s (ai=%v)", host, msg, apiKey != "") writeJSON(w, http.StatusOK, map[string]any{"ok": true, "detail": msg, "ai": apiKey != ""}) } // statsHandler reports the reaper's guards for a space — setup_complete, member // count, and has_content (any message or thread). Read-only against the live // instance DB. validHost guards the DB path against traversal before it's joined // with the data dir. func (c *controld) statsHandler(w http.ResponseWriter, r *http.Request) { host := strings.ToLower(strings.TrimSpace(r.PathValue("host"))) if !validHost(host) { writeControlErr(w, http.StatusBadRequest, "invalid_host", "not a valid hostname") return } members, people, setupComplete, hasContent, err := instanceStatus(host) if err != nil { if os.IsNotExist(err) { // A hibernating space that has never woken has no local DB at all — // the first wake boots it fresh, and the setup-invite mint seeds the // admin user. Its stats are therefore DEFINITIONAL, answered from // the registry row alone: no users, never set up, no content. No // socket is dialed and no wake is triggered — a reaper sweep over // thousands of dormant spaces must stay a pure metadata read, never // a wave of reaper-induced wakes. (A missing DB on a NON-hibernating // row stays a 404: a systemd-managed instance creates its DB at // first boot, so "no DB" there means a failed create — the reaper // skips those, as before.) if c.reg != nil { if rt, rerr := c.reg.get(host); rerr == nil && rt != nil && rt.Kind == "instance" && rt.Hibernate { writeJSON(w, http.StatusOK, map[string]any{"host": host, "members": 0, "people": 0, "setup_complete": false, "has_content": false, "storage_bytes": 0}) return } } writeControlErr(w, http.StatusNotFound, "unknown_host", "no such instance") return } log.Printf("controld: stats %s: %v", host, err) writeControlErr(w, http.StatusInternalServerError, "stats_failed", err.Error()) return } writeJSON(w, http.StatusOK, map[string]any{ "host": host, "members": members, "people": people, "setup_complete": setupComplete, "has_content": hasContent, // The storage meter's read (meter.go) — pure filesystem metadata, // measured alongside the DB stats so one endpoint serves both sweeps. "storage_bytes": instanceStorageBytes(host), }) } // deleteHandler deprovisions one instance (self-serve space deletion, reached // via the cloud provisioner which enforces ownership). The instance's data is // archived to /data/trash, not destroyed — see deleteInstance — unless the // caller passes ?discard=1, the demo tier's no-backups contract // (docs/demo-spaces.md): a demo's data dies with it, by disclosure. func (c *controld) deleteHandler(w http.ResponseWriter, r *http.Request) { host := r.PathValue("host") discard := r.URL.Query().Get("discard") == "1" c.mu.Lock() defer c.mu.Unlock() err := deleteInstance(c.reg, host, discard) switch { case errors.Is(err, errInvalidHost): writeControlErr(w, http.StatusBadRequest, "invalid_host", err.Error()) return case errors.Is(err, errUnknownHost): writeControlErr(w, http.StatusNotFound, "unknown_host", err.Error()) return case err != nil: log.Printf("controld: delete %s: %v", host, err) writeControlErr(w, http.StatusInternalServerError, "delete_failed", err.Error()) return } if discard { log.Printf("controld: deleted %s (data DISCARDED — demo contract)", host) } else { log.Printf("controld: deleted %s (data archived to trash)", host) } writeJSON(w, http.StatusOK, map[string]any{"ok": true}) } // auth gates a handler behind the control token (constant-time hash compare, // same pattern as the push relay's admin gate). func (c *controld) auth(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if !tokenHashEqual(bearerTokenHeader(r), c.tokenHash) { writeControlErr(w, http.StatusUnauthorized, "unauthorized", "bad or missing control token") return } next(w, r) } } // atCapacity applies the box capacity policy: an instance-count ceiling and a // free-disk floor (disk, not RAM, is the binding resource — see // docs/instance-router.md § Sizing). It returns a human reason when full. func (c *controld) atCapacity() (string, error) { routes, err := c.reg.list() if err != nil { return "", err } total, materialized := 0, 0 for _, r := range routes { if r.Kind != "instance" { continue } total++ if !r.Hibernate { materialized++ } } if c.hibernateCreates { // Hibernating rows hold no unit, no process, no RAM — the per-instance // resource maxInstances guards is what's MATERIALIZED (systemd-managed, // always-on), so only those count against it. Registrations still get a // much higher sanity ceiling (rows, env files, staged symlinks are not // free forever), and disk stays the real floor below. if materialized >= c.maxInstances { return fmt.Sprintf("materialized-instance limit reached (%d/%d)", materialized, c.maxInstances), nil } maxReg := c.maxRegistered if maxReg <= 0 { maxReg = 10 * c.maxInstances } if total >= maxReg { return fmt.Sprintf("registered-space limit reached (%d/%d)", total, maxReg), nil } } else if total >= c.maxInstances { return fmt.Sprintf("instance limit reached (%d/%d)", total, c.maxInstances), nil } free, err := freeDiskBytes(instanceDataDir) if err != nil { return "", err } if free < uint64(c.minFreeGB)<<30 { return fmt.Sprintf("low disk (%.1f GB free, floor %d GB)", float64(free)/(1<<30), c.minFreeGB), nil } return "", nil } func (c *controld) createHandler(w http.ResponseWriter, r *http.Request) { var req controlCreateReq if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&req); err != nil { writeControlErr(w, http.StatusBadRequest, "bad_request", "invalid JSON body") return } // One create at a time: the capacity check and the creation it authorizes // must not interleave with another create. c.mu.Lock() defer c.mu.Unlock() reason, err := c.atCapacity() if err != nil { writeControlErr(w, http.StatusInternalServerError, "create_failed", err.Error()) return } if reason != "" { log.Printf("controld: refusing %s: %s", req.Host, reason) writeControlErr(w, http.StatusServiceUnavailable, "at_capacity", reason) return } // Demos are NEVER activator-managed, even on hibernate-by-default boxes: // no lease, no litestream replica, no parked S3 prefix — "no backups" // (docs/demo-spaces.md) is structural. They hold a materialized slot for // their 24h life instead; the cloud's live-demo cap bounds how many. hibernate := c.hibernateCreates if strings.EqualFold(strings.TrimSpace(req.Entitlement), "demo") { hibernate = false } // The box default fills a create that names no home origin (see the // controld.homeOrigin comment); an explicit request value still wins. homeOrigin := req.HomeOrigin if homeOrigin == "" { homeOrigin = c.homeOrigin } res, err := addInstance(c.reg, addInstanceOpts{ Host: req.Host, AdminToken: req.AdminToken, AppStoreURL: req.AppStoreURL, HomeOrigin: homeOrigin, PushRelayURL: req.PushRelayURL, PushRelayToken: req.PushRelayToken, Entitlement: req.Entitlement, GroupName: req.Name, Replication: strings.ToLower(strings.TrimSpace(req.Replication)), // Placement region (multi-edge Phase 1): recorded on the registry row, // where the mesh reads it to place the space (control.go registerInstance // → registry.go). TrimSpace only — region is matched case-sensitively // against LCHAT_MESH_REGION (mesh.go), never lower-cased. HomeRegion: strings.TrimSpace(req.HomeRegion), // Hibernate-by-default (#550): with LCHAT_CONTROL_HIBERNATE_CREATES=1 // every provisioner-created space is born activator-managed — the // create's public-readiness probe is its one (first) wake, minting the // setup link and warming the cert, and the idle window then puts it to // sleep. A never-claimed space costs zero RAM from that point on. Off // by default so the flip stays a per-box config decision. Hibernate: hibernate, }) switch { case errors.Is(err, errInvalidHost): writeControlErr(w, http.StatusBadRequest, "invalid_host", err.Error()) return case errors.Is(err, errHostExists): writeControlErr(w, http.StatusConflict, "host_exists", err.Error()) return case err != nil: log.Printf("controld: create %s: %v", req.Host, err) writeControlErr(w, http.StatusInternalServerError, "create_failed", err.Error()) return } log.Printf("controld: created %s", res.Host) if res.SetupErr != nil { // The admin link still bootstraps, but a create whose setup link died // is a degraded first-run — it must be visible in the journal. log.Printf("controld: %s: setup link mint FAILED: %v", res.Host, res.SetupErr) } writeJSON(w, http.StatusCreated, map[string]any{ "host": res.Host, "url": "https://" + res.Host, "setup_link": res.SetupLink, // "" if the mint failed; admin_link still bootstraps "admin_link": "https://" + res.Host + "/admin#token=" + res.AdminToken, }) } // tokenHashEqual compares a presented token against a stored hash in constant // time (cf. the push relay's admin gate). func tokenHashEqual(token, wantHash string) bool { return token != "" && subtle.ConstantTimeCompare([]byte(hashToken(token)), []byte(wantHash)) == 1 } // bearerTokenHeader pulls the token out of "Authorization: Bearer ". func bearerTokenHeader(r *http.Request) string { const p = "Bearer " h := r.Header.Get("Authorization") if len(h) > len(p) && h[:len(p)] == p { return h[len(p):] } return "" } // runControld: `lchat controld` — serve the control API on a local unix socket. // Runs as root (the lchat-control.service unit): it's the one process allowed to // call systemctl and write the version store, and it is never routed publicly. func runControld(args []string) { fs := flag.NewFlagSet("controld", flag.ExitOnError) registryPath := fs.String("registry", envOr("LCHAT_REGISTRY", "/data/registry.db"), "router registry SQLite path") socketPath := fs.String("socket", controlSocketPath, "unix socket to serve the control API on") tokenFile := fs.String("token-file", envOr("LCHAT_CONTROL_TOKEN_FILE", controlTokenPath), "file holding the control bearer token") maxInstances := fs.Int("max-instances", envInt("LCHAT_MAX_INSTANCES", 40), "refuse creation beyond this many instances on the box (materialized only under -hibernate-creates)") minFreeGB := fs.Int("min-free-gb", envInt("LCHAT_MIN_FREE_GB", 5), "refuse creation when the data disk has less than this free (GB)") trashKeepDays := fs.Int("trash-keep-days", envInt("LCHAT_TRASH_KEEP_DAYS", 0), "purge archived (deleted) spaces from /data/trash after this many days; 0 keeps them forever") hibernateCreates := fs.Bool("hibernate-creates", os.Getenv("LCHAT_CONTROL_HIBERNATE_CREATES") == "1", "create provisioner spaces as hibernating (activator-managed, no systemd unit) — requires the router to run with LCHAT_HIBERNATE_BUCKET") maxRegistered := fs.Int("max-registered", envInt("LCHAT_MAX_REGISTERED", 0), "with -hibernate-creates: refuse registration beyond this many total spaces, hibernating included (0 = 10x -max-instances)") homeOrigin := fs.String("home-origin", os.Getenv("LCHAT_CONTROL_HOME_ORIGIN"), "default LCHAT_HOME_ORIGIN for provisioner creates that don't name one (empty = the compiled production default)") _ = fs.Parse(args) tok, err := os.ReadFile(*tokenFile) if err != nil { die("controld: read token file: %v (run `lchat bootstrap` to mint it)", err) } reg, err := openRegistry(*registryPath) if err != nil { die("controld: open registry: %v", err) } c := &controld{ reg: reg, tokenHash: hashToken(string(trimNL(tok))), maxInstances: *maxInstances, minFreeGB: *minFreeGB, hibernateCreates: *hibernateCreates, maxRegistered: *maxRegistered, homeOrigin: strings.TrimSpace(*homeOrigin), } if err := os.Remove(*socketPath); err != nil && !os.IsNotExist(err) { die("controld: remove stale socket: %v", err) } ln, err := net.Listen("unix", *socketPath) if err != nil { die("controld: listen %s: %v", *socketPath, err) } // 0660 + group lchat: the sandboxed services can connect (auth is the token, // not the socket), nobody else on the box can even dial. if err := os.Chmod(*socketPath, 0o660); err != nil { die("controld: chmod socket: %v", err) } if g, err := user.LookupGroup(serviceUser); err == nil { if gid, err := strconv.Atoi(g.Gid); err == nil { _ = os.Chown(*socketPath, -1, gid) // best-effort; needs root } } go runTrashPurge(time.Duration(*trashKeepDays) * 24 * time.Hour) mode := "" if c.hibernateCreates { maxReg := c.maxRegistered if maxReg <= 0 { maxReg = 10 * c.maxInstances } mode = fmt.Sprintf(", hibernate-creates on (max %d registered)", maxReg) } log.Printf("controld: serving on %s (registry %s, max %d instances, %d GB disk floor%s)", *socketPath, *registryPath, *maxInstances, *minFreeGB, mode) // Say the mesh posture out loud: on a mesh box this daemon MUST see the // router's mesh env (the unit reads /data/router.env), or every create's // route replication silently no-ops and peer-owned hosts are born dark — // the failure that hid for weeks precisely because nothing logged it. if hc, hcErr := hibernateConfigFromEnv(); hcErr != nil { log.Printf("controld: mesh route replication OFF (%v)", hcErr) } else if mesh, meshErr := meshFromEnv(hc); meshErr != nil { log.Printf("controld: mesh route replication OFF (%v)", meshErr) } else if mesh == nil { log.Printf("controld: mesh route replication off (no LCHAT_MESH_SELF — fine off-mesh; on a mesh box check the unit reads /data/router.env)") } else { log.Printf("controld: mesh route replication on (self %s, domains %v)", mesh.self, mesh.domains) } srv := &http.Server{Handler: c.handler(), ReadHeaderTimeout: 10 * time.Second} if err := srv.Serve(ln); err != nil { log.Fatal(err) } } // trimNL strips a single trailing newline a hand-edited token file may carry. func trimNL(b []byte) []byte { for len(b) > 0 && (b[len(b)-1] == '\n' || b[len(b)-1] == '\r') { b = b[:len(b)-1] } return b } // envInt reads an integer env var, falling back on absent or malformed values. func envInt(key string, def int) int { if v := os.Getenv(key); v != "" { if n, err := strconv.Atoi(v); err == nil { return n } } return def }