// prepare.go is everything a nested role needs before its unit starts // (isolated-exec design §6.1): the launch file's directories, the merged // environment, the control listener (run only) and the role's log // listener. It is the nested counterpart of what startChild/seedChild do // inline for a flat host. package activator import ( "context" "encoding/json" "errors" "fmt" "log" "net/http" "os" "path/filepath" "strings" "time" "github.com/carlosframework/platform/internal/edge" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/instancefs" "github.com/carlosframework/platform/internal/instancekey" "github.com/carlosframework/platform/internal/instanceupdates" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/schedule" ) // prepare is everything a nested role needs before its unit starts // (design §6.1): the two directories, the admin token and merged env (for // run and seed; a cmd's env is its generation's, and prepare returns none), the control listener (run only), the role's log listener, // and stale records cleared. It returns the listeners so the caller can // hand them to the proc or close them on a failed start. func (b *ExecBackend) prepare(rt registry.Route, role instancectl.Role) (instancePaths, *edge.ControlListener, *logListener, map[string]string, error) { return b.prepareAt(rt, role, rt.Socket, 0) } // prepareAt is prepare for a run role whose app socket is sock: rt.Socket, // or a generation socket (StartGen, zero-downtime spec §7.1). A generation // socket's own directory, g in the run dir, is made first (rev 19, rule // 3). Only the file at sock is removed, never a sibling generation's, and // the control listener binds sock + ".control". The seed and sidecar roles // ignore sock. // gen is the generation unit being prepared (0 for the original templates): // it keys the log socket and the exit records cleared (zero-downtime spec // §7.6). func (b *ExecBackend) prepareAt(rt registry.Route, role instancectl.Role, sock string, gen uint64) (instancePaths, *edge.ControlListener, *logListener, map[string]string, error) { p, ctl, logs, env, _, err := b.prepareKeyedAt(rt, role, sock, gen, nil) return p, ctl, logs, env, err } // prepareKeyedAt is prepareAt returning, with the env, the layer its // CARLOS_INSTANCE_KEY came from, decided by the same read (instance-key // ruling 29). The nested run spawn keeps both on the generation's handle, // for a run's command against that generation (StartCmd). // steps, when not nil, times prepare's parts for the caller's journal line. func (b *ExecBackend) prepareKeyedAt(rt registry.Route, role instancectl.Role, sock string, gen uint64, steps *wakeSteps) (instancePaths, *edge.ControlListener, *logListener, map[string]string, instancekey.Layer, error) { if steps == nil { steps = &wakeSteps{} } else { steps.begin(time.Now()) } p := b.pathsFor(rt) if err := os.MkdirAll(p.dataDir, 0o700); err != nil { return p, nil, nil, nil, instancekey.LayerNone, err } if err := os.MkdirAll(p.runDir, 0o750|os.ModeSetgid); err != nil { return p, nil, nil, nil, instancekey.LayerNone, err } if err := os.MkdirAll(filepath.Dir(b.logSocketFor(rt.Host, role, gen)), 0o700); err != nil { return p, nil, nil, nil, instancekey.LayerNone, err } steps.mark(time.Now(), "dirs") var env map[string]string var keyLayer instancekey.Layer // the layer env's key came from, read with it (ruling 29) switch role { case instancectl.RoleRun, instancectl.RoleSeed: // p.dataDir is the instance directory here, tenant-writable ground // (design §3.2), so the token file is minted and read through // instancefs rather than a plain path-based open. if err := EnsureAdminTokenBeneath(p.dataDir, rt.Host+".env"); err != nil { return p, nil, nil, nil, instancekey.LayerNone, fmt.Errorf("admin token for %s: %w", rt.Host, err) } steps.mark(time.Now(), "admin token") if _, err := b.ensureInstanceKeyFor(context.Background(), rt, p); err != nil { // prepareAt has no ctx to bound the mint's bucket read; configReadTimeout does return p, nil, nil, nil, instancekey.LayerNone, fmt.Errorf("instance key for %s: %w", rt.Host, err) } steps.mark(time.Now(), "instance key") var err error env, keyLayer, err = b.instanceEnvLayer(rt) if err != nil { return p, nil, nil, nil, instancekey.LayerNone, err } steps.mark(time.Now(), "env") case instancectl.RoleCmd: // A run's environment is the one its generation launched with // (merge ruling M3), which StartCmd has. The delivered files are // not read: config sync may have written a revision the serving // generation has not activated. default: var err error env, err = readEnvFileBeneath(p.dataDir, filepath.Base(p.sidecarEnv)) if err != nil { return p, nil, nil, nil, instancekey.LayerNone, err } keyLayer = sidecarKeyLayer(env) } // Stale exit records from earlier invocations of this role: a start // that reads one would mis-word the very first exit line this // invocation eventually gets, attributing a previous run's status to // this one. if entries, err := os.ReadDir(p.runDir); err == nil { for _, e := range entries { if strings.HasPrefix(e.Name(), exitPrefix(role, gen)) { _ = os.Remove(filepath.Join(p.runDir, e.Name())) } } } var ctl *edge.ControlListener if role == instancectl.RoleRun { // Nothing is running: either this is the first invocation of this // role for this host, or the unit is not active (Start would not // have reached prepare otherwise), so any file at rt.Socket is a // leftover from an earlier invocation, most likely one that was // SIGKILLed before its own bind was ever established. On the // nested path the agent itself never binds this socket (the app // does, inside its own namespace, once the trampoline execs it), // so unlike startChild's flat spawn there is no "our own live // process squatting on it" case this could ever race: removing it // here is safe by construction, and leaving it in place would fail // EVERY later wake's health check with nothing but a dial timeout, // no member-facing diagnosis at all, unlike the flat path's #129 // handling, which at least turns the bind failure into a named // error (memberSafeWakeReason). // // A generation's socket lives in a directory of its own (rev 19, // rule 1), made here with the run dir's mode. It inherits the run // dir's setgid carlos-sockets group, and it sits inside the run dir // the generation unit binds, so the app (uid carlos, as this agent // is) can bind its socket there. if dir := filepath.Dir(sock); dir != p.runDir { if err := os.MkdirAll(dir, 0o750|os.ModeSetgid); err != nil { return p, nil, nil, nil, instancekey.LayerNone, err } } _ = os.Remove(sock) ctl = b.listenControl(rt, sock+".control", p.envFile, gen) steps.mark(time.Now(), "control socket") } // A replica's roles are gated like a flat launch (ruling 26): the seed // and the sidecar have no control socket, the run role has the one just // bound or none. The sidecar's key, if its own file holds one, is // checked as the flat sidecar's is (sidecarKeyLayer). A cmd is gated in // StartCmd, on the environment its generation launched with: prepare // read none for it. if role != instancectl.RoleCmd { if err := b.replicaKeyGate(rt, env, keyLayer, ctl != nil); err != nil { if ctl != nil { ctl.Close() } return p, nil, nil, nil, instancekey.LayerNone, err } } steps.mark(time.Now(), "key gate") // The cmd role's output goes only to its own chunks, never to the // journal or the app log, so it has no app-log listener. var logs *logListener if role != instancectl.RoleCmd { var err error logs, err = listenLogs(b.logSocketFor(rt.Host, role, gen), b.logEmitter(rt)) if err != nil { if ctl != nil { ctl.Close() } return p, nil, nil, nil, instancekey.LayerNone, fmt.Errorf("log listener for %s %s: %w", rt.Host, role, err) } } steps.mark(time.Now(), "logs") return p, ctl, logs, env, keyLayer, nil } // exitPrefix is the exit records prepare clears before a start: this // role's for the original templates, this generation's alone otherwise. // A sibling generation's record is never touched (zero-downtime spec // §7.6). func exitPrefix(role instancectl.Role, gen uint64) string { if gen == 0 { return "exit." + string(role) + "." } return instancectl.GenExitPrefix(gen) } // listenControl binds the control socket exactly as startChild does for an // exec child (the tokenReader and schedule mount are unchanged, aside from // reading beneath the instance directory rather than following a plain // path, design §3.2); a bind failure is logged and the instance runs // without its control verbs, as today. // // gen is the generation the listener serves (0 for the original // templates): its schedule writes go through that generation's write fence // (writeFence, final review F8). func (b *ExecBackend) listenControl(rt registry.Route, control, envPath string, gen uint64) *edge.ControlListener { tokenReader := func() (string, error) { env, err := readEnvFileBeneath(filepath.Dir(envPath), filepath.Base(envPath)) if err != nil { return "", err } return env["CARLOS_ADMIN_TOKEN"], nil } var mount func(*http.ServeMux) if b.Timers != nil || b.UpdatesReader != nil { mount = func(mux *http.ServeMux) { if b.Timers != nil { schedule.MountGated(mux, rt.Host, b.Timers, tokenReader, time.Now, b.writeFence(rt.Host, gen)) } if b.UpdatesReader != nil { instanceupdates.Mount(mux, rt.Host, tokenReader, b.UpdatesReader) } } } ctl, err := edge.ListenControl(control, rt.Host, b.Cache, mount) if err != nil { log.Printf("activator: control listener for %s: %v (continuing without it)", rt.Host, err) return nil } return ctl } // logSocket is logSocketFor for the original templates. func (b *ExecBackend) logSocket(host string, role instancectl.Role) string { return b.logSocketFor(host, role, 0) } // logSocketFor is the role's log socket path, keyed by generation for a // generation unit, honouring the test override of the run directory // (RunDirOverride). func (b *ExecBackend) logSocketFor(host string, role instancectl.Role, gen uint64) string { if gen != 0 { if b.RunDirOverride != "" { return filepath.Join(b.RunDirOverride, "logs", instancectl.GenKey(host, gen)+".run.sock") } return instancectl.GenLogSocket(host, gen) } return b.logSocketNamed(host, string(role)) } // logSocketNamed is an original template's log socket for any listener // name: the cmd role has two, "cmd" and "cmd-err" (run spec §5.4). func (b *ExecBackend) logSocketNamed(host, name string) string { if b.RunDirOverride != "" { return filepath.Join(b.RunDirOverride, "logs", provision.InstanceID(host)+"."+name+".sock") } return provision.LogSocket(host, name) } // logEmitter is the same sink teeStd feeds, or nil when Logs is unset. func (b *ExecBackend) logEmitter(rt registry.Route) func(string) { if b.Logs == nil { return nil } return func(line string) { b.Logs.App(release.RouteAccount(rt), rt.App, rt.Host, line) } } // writeLaunch is writeLaunchFor for the original templates. func (b *ExecBackend) writeLaunch(p instancePaths, role instancectl.Role, host string, argv, env []string) error { return b.writeLaunchFor(p, role, 0, host, argv, env) } // writeLaunchFor writes the launch file atomically into the instance // directory (design §6.2): launch..json, or launch.gen..json for a // generation (zero-downtime spec §7.6). The env is childEnv's deduplicated // list; the trampoline execs it as is. func (b *ExecBackend) writeLaunchFor(p instancePaths, role instancectl.Role, gen uint64, host string, argv, env []string) error { raw, err := json.MarshalIndent(instancectl.Launch{Schema: instancectl.LaunchSchema, Host: host, Argv: argv, Env: env}, "", " ") if err != nil { return err } name := instancectl.LaunchFileName(role) if gen != 0 { name = instancectl.GenLaunchFileName(gen) } return instancefs.WriteAtomic(p.dataDir, name, append(raw, '\n'), 0o600) } // readEnvFileBeneath is readEnvFile through instancefs: a nested host's env // files are read without following a symlink (design §3.2). A missing file // is an empty env, as readEnvFile treats it (the caller's own // os.IsNotExist check would otherwise have nothing to check against). func readEnvFileBeneath(dir, name string) (map[string]string, error) { b, err := instancefs.ReadBeneathLimit(dir, name, 1<<20) if errors.Is(err, os.ErrNotExist) { return map[string]string{}, nil } if err != nil { return nil, err } return parseEnvBytes(b), nil }