// The on-box version store — ported from Eleven versions.go. // // Versioned, side-by-side deploys: the box runs one binary per host, but // they need not be the SAME binary. Each host's unit execs // /live/, a symlink into a content store at // /versions//. Swapping one link (and restarting one unit) // moves one host to a new build while every other host stays put — so a new // build can be canaried on one host before it's promoted fleet-wide, and // rolled back per host. // // The opt dir defaults from CARLOS_OPT_DIR (default /opt/carlos), but every // function takes the base dir as a parameter so tests never touch /opt. package release import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "net" "net/http" "os" "path/filepath" "regexp" "strings" "time" "github.com/carlosframework/platform/internal/safefile" "github.com/carlosframework/platform/internal/unixdial" ) // OptDir is the on-box install root: CARLOS_OPT_DIR, default /opt/carlos. // Resolved at call time (not init) so a test's Setenv is honoured. func OptDir() string { if d := os.Getenv("CARLOS_OPT_DIR"); d != "" { return d } return "/opt/carlos" } func versionsDir(optDir string) string { return filepath.Join(optDir, "versions") } func liveDir(optDir string) string { return filepath.Join(optDir, "live") } // VersionPath is where a staged build of app lives. App names are unique per // ACCOUNT, so two accounts can both have a "blog" — and, since versions are // caller-chosen strings, both can have a "v1" whose bytes differ. A single // /versions// would let one account's binary be staged over // another's and then executed under its host. // // The ops account deliberately keeps the pre-accounts path byte for byte // (/versions//): every existing box's live symlinks point // there, and moving them would restage and restart the whole fleet on // rollout for no benefit. Everyone else gets a level in between. func VersionPath(optDir, acct, ver, app string) string { if acct == "" || acct == OpsAccount { return filepath.Join(versionsDir(optDir), ver, app) } return filepath.Join(versionsDir(optDir), ver, acct, app) } // LiveLink is the per-host pointer the unit execs (/live/) — a // symlink to a VersionPath. Swapping it atomically is how one host moves // between versions independently of the others. func LiveLink(optDir, host string) string { return filepath.Join(liveDir(optDir), host) } // versionRe keeps a version usable as a directory name (no slashes, no // traversal). var versionRe = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`) func ValidVersion(v string) bool { return v != "" && len(v) <= 64 && v != "." && v != ".." && versionRe.MatchString(v) } // CopyFileAtomic copies src to dst via a temp file + rename, so a reader // never sees a half-written binary (and a crashed copy leaves the old one // intact). func CopyFileAtomic(src, dst string, mode os.FileMode) error { in, err := safefile.OpenFile(src, os.O_RDONLY, 0) if err != nil { return err } defer in.Close() return safefile.Write(dst, mode, func(out io.Writer) error { _, err := io.Copy(out, in); return err }) } // WriteFileAtomic is CopyFileAtomic for in-memory bytes (a downloaded, // sha256-verified artifact). func WriteFileAtomic(dst string, data []byte, mode os.FileMode) error { return safefile.WriteFile(dst, data, mode) } // StageArtifact puts verified artifact bytes into the store under ver, so // hosts can be pointed at it. Idempotent: re-staging overwrites. func StageArtifact(optDir, acct, ver, app string, data []byte) error { dst, err := stagePath(optDir, acct, ver, app) if err != nil { return err } return WriteFileAtomic(dst, data, 0o755) } // errArtifactMismatch is stageArtifactFrom refusing bytes that do not hash // to the manifest's sha256. Nothing is left behind when it is returned. var errArtifactMismatch = errors.New("sha256 mismatch") // stageArtifactFrom is StageArtifact for a build read from the store as a // stream (#269). The bytes go straight into the temp file beside the // destination and are hashed on the way, so memory stays flat whatever the // build's size: the root stage pass runs under MemoryMax=256M, and reading a // ~99 MB build whole peaked past it. Only bytes that hash to wantSHA are // moved into place. A mismatch or a failed read removes the temp file and // leaves whatever was staged under the name untouched, exactly as verifying // in memory first did. // // open is called only once the version dir is writable, so a sandboxed // agent fails on the mkdir (stageBlocked relays that to root) without // opening the build at all. func stageArtifactFrom(optDir, acct, ver, app, wantSHA string, open func() (io.ReadCloser, error)) error { dst, err := stagePath(optDir, acct, ver, app) if err != nil { return err } in, err := open() if err != nil { return fmt.Errorf("download %s: %w", ver, err) } defer in.Close() return safefile.Write(dst, 0o755, func(out io.Writer) error { h := sha256.New() if _, err := io.Copy(io.MultiWriter(out, h), in); err != nil { return fmt.Errorf("download %s: %w", ver, err) } if hex.EncodeToString(h.Sum(nil)) != wantSHA { return fmt.Errorf("download %s: %w (refusing to stage)", ver, errArtifactMismatch) } return nil }) } // stagePath validates the names, makes the version dir traversable, and // returns where the build goes. func stagePath(optDir, acct, ver, app string) (string, error) { if !ValidVersion(ver) { return "", fmt.Errorf("invalid version %q (allowed: letters, digits, . _ -)", ver) } if !ValidApp(app) { return "", fmt.Errorf("invalid app %q", app) } if acct != "" && !ValidAccount(acct) { return "", fmt.Errorf("invalid account %q", acct) } dst := VersionPath(optDir, acct, ver, app) dir := filepath.Dir(dst) if err := safefile.MkdirAll(dir, 0o755); err != nil { return "", err } // MkdirAll's mode is masked by the caller's umask, so a tight operator // umask (e.g. 077 from a hardened shell) leaves the version dir 0700 — // the sandboxed service user then can't traverse in to exec its binary // (203/EXEC "Permission denied", found staging a canary on Eleven). // Force it, like the write forces the binary's +x. if err := os.Chmod(dir, 0o755); err != nil { return "", err } return dst, nil } // StagedManifestPath is the staged binary's SIBLING file: // // /versions//[/].manifest.json // // A sibling, not a new directory, so an old box's tree stays valid and no // directory shape changes. This is what lets the converge pass read a // RUNNING version's capabilities without a bucket round-trip on every tick. func StagedManifestPath(optDir, acct, ver, app string) string { return VersionPath(optDir, acct, ver, app) + ".manifest.json" } // StageManifest writes the manifest bytes atomically beside the binary, // 0644 — root-writable, world-readable, never executable (it's JSON, not a // unit the sandboxed service execs). The write is temp-file-then-rename // (WriteFileAtomic), so a crashed pass never leaves a truncated sibling — // that atomicity is relied on but not itself exercised by this package's // suite (no test kills a fault injected between the write and the rename); // it is the same primitive StageArtifact already trusts for the binary. // // raw is the canonical manifest's bytes as the store returned them // (GetManifestRaw), so the sibling is lossless (zero-downtime deploys spec // §4.2). This is the ONE write path that skips StageArtifact, so it repeats // StageArtifact's three charset gates itself rather than relying on a caller // that staged the binary first: the back-fill branch calls this with no // StageArtifact call in the same pass, and the reserved-word/traversal // protection those gates give the binary's path must cover this sibling path // too. func StageManifest(optDir, acct, ver, app string, raw []byte) error { if !ValidVersion(ver) { return fmt.Errorf("invalid version %q (allowed: letters, digits, . _ -)", ver) } if !ValidApp(app) { return fmt.Errorf("invalid app %q", app) } if acct != "" && !ValidAccount(acct) { return fmt.Errorf("invalid account %q", acct) } // A version staged by a pre-capability binary has no / subdirectory // at all (issue #58): WriteFileAtomic's tmp-file create fails ENOENT the // first time a sibling is back-filled for it under a real account. Create // the parent dir first, exactly as StageArtifact does for the binary — // but only Chmod it when WE are the one creating it. The umask-masking // problem StageArtifact's Chmod exists for (a tight operator umask // leaving the dir non-traversable) only applies to a dir this call // creates; Chmod-ing a dir that already existed is a new failure mode of // its own — a non-owning uid that can still write into the dir (and so // can legitimately reach this branch) gets EPERM trying to chmod it, // and the sibling write below never runs. dir := filepath.Dir(StagedManifestPath(optDir, acct, ver, app)) _, statErr := os.Stat(dir) dirExisted := statErr == nil if err := safefile.MkdirAll(dir, 0o755); err != nil { return err } if !dirExisted { if err := os.Chmod(dir, 0o755); err != nil { return err } } return WriteFileAtomic(StagedManifestPath(optDir, acct, ver, app), raw, 0o644) } // ReadStagedManifest parses the sibling. It returns (nil, nil) — NOT an // error — when the file is absent: a box staged before this feature existed // has a binary and no sibling, and the converge pass must read that as // "unknown", not as "no capabilities". The distinction is load-bearing: // unknown LEAVES THE DROP-INS ALONE (spec §4d), none REMOVES them. // // A present-but-unreadable or unparsable sibling is a real error (fail // toward leave-alone, the same three-state rule): the caller cannot tell // "unknown" from "corrupt" if both return nil, and the converge pass must // not treat a half-written file as "nothing declared". // // The parsed manifest runs through the same validateManifest gate every // other manifest read does (GetManifest's), so a stale sibling written by a // NEWER binary — one that understood a capability this binary does not — // refuses cleanly here instead of being silently trusted. It also runs // GetManifest's identity cross-check: a sibling whose body names a // different app or version than the path it was read from is refused, // rather than trusted against the PATH's name — without this, a sibling at // keymail's path claiming app "console" would pass every charset and // reserved-word gate under keymail's own identity, and its capabilities // (host_role's User=, runtime_dir, ports) would land on keymail's unit. // // A sibling staged by this binary or later is the canonical manifest's exact // bytes. One staged by an older binary is a LOSSY re-marshal: it drops any // top-level key that binary's Manifest struct did not know, `swap` included. // So the bucket object stays the authority for anything a box acts on // (CanonicalDemand); this sibling exists so the converge pass can read a // running version's capabilities without a bucket round trip on every tick. func ReadStagedManifest(optDir, acct, ver, app string) (*Manifest, error) { raw, err := safefile.ReadFile(StagedManifestPath(optDir, acct, ver, app)) if err != nil { if os.IsNotExist(err) { return nil, nil } return nil, err } var m Manifest if err := json.Unmarshal(raw, &m); err != nil { return nil, fmt.Errorf("staged manifest %s/%s: %v", app, ver, err) } if err := validateManifest(&m); err != nil { return nil, fmt.Errorf("staged manifest %s/%s: %w", app, ver, err) } if m.App != app || m.Version != ver { return nil, fmt.Errorf("staged manifest at %s names %s/%s", StagedManifestPath(optDir, acct, ver, app), m.App, m.Version) } return &m, nil } // PointLive atomically repoints a host's live symlink at ver's build of app. // It verifies ver is staged first, so a typo can't leave a dangling link // that would fail the unit's ExecStart on the next restart. func PointLive(optDir, host, acct, ver, app string, runningVersion ...string) error { target := VersionPath(optDir, acct, ver, app) if _, err := os.Stat(target); err != nil { return fmt.Errorf("version %q of %s is not staged (%s)", ver, app, target) } if err := safefile.MkdirAll(liveDir(optDir), 0o755); err != nil { return err } link := LiveLink(optDir, host) if old, err := os.Readlink(link); err == nil && old != target { dir := filepath.Join(optDir, "previous") // Root may have crashed after the last repoint but before restart. // Keep the verified running build's pointer instead of replacing its // last authority with an intermediate build that never ran. if len(runningVersion) == 1 && runningVersion[0] != "" { if !ValidVersion(runningVersion[0]) { return os.ErrPermission } running := VersionPath(optDir, acct, runningVersion[0], app) current, err := filepath.EvalSymlinks(link) if err != nil { return err } if running != current { prior, err := filepath.EvalSymlinks(filepath.Join(dir, host)) if err != nil || prior != running { return os.ErrPermission } } old = running } if err := safefile.MkdirAll(dir, 0755); err != nil { return err } previous := filepath.Join(dir, host) pending := previous + ".tmp" _ = os.Remove(pending) if err := os.Symlink(old, pending); err != nil { return err } if err := os.Rename(pending, previous); err != nil { os.Remove(pending) return err } f, err := os.Open(dir) if err != nil { return err } err = f.Sync() f.Close() if err != nil { return err } } else if err != nil && !os.IsNotExist(err) { return err } tmp := link + ".tmp" _ = os.Remove(tmp) // a leftover temp link from a crashed swap if err := os.Symlink(target, tmp); err != nil { return err } if err := os.Rename(tmp, link); err != nil { os.Remove(tmp) return err } return nil } // LiveVersion reports the version a host's live symlink currently points at, // or "" (no error) if the host hasn't been put under version management yet. func LiveVersion(optDir, host string) (string, error) { target, err := os.Readlink(LiveLink(optDir, host)) if err != nil { if os.IsNotExist(err) { return "", nil } return "", err } return versionOfTarget(optDir, target), nil } // versionOfTarget is the version a live link's RAW target names, or "" when // the target is not under the version store. target is // /versions// (ops) or /versions/// // (everyone else), so the version is the FIRST segment under the versions // dir, not simply the parent directory, which would read an account sqid as // a version. A target outside the store, or a relative one, is not ours to // interpret: "" is the same answer an absent link gets. func versionOfTarget(optDir, target string) string { return versionUnder(versionsDir(optDir), target) } // versionUnder is versionOfTarget against a given versions dir: the first // segment of target under vdir, or "" when target is not below it. func versionUnder(vdir, target string) string { rel, err := filepath.Rel(vdir, target) if err != nil || strings.HasPrefix(rel, "..") { return "" } ver, _, ok := strings.Cut(filepath.ToSlash(rel), "/") if !ok { return "" } return ver } // WaitHealthyVersion waits until the host behind socket not only answers but // reports the expected build at /api/version — so a deploy fails loudly if // the process is still the old binary (a stale socket, or a restart that // didn't take), rather than silently "succeeding". func WaitHealthyVersion(socket, wantBuild string, timeout time.Duration) error { deadline := time.Now().Add(timeout) client := unixHTTPClient(socket) var last error for time.Now().Before(deadline) { resp, err := client.Get("http://carlos/api/version") if err == nil { var body struct { Build string `json:"build"` } derr := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&body) resp.Body.Close() if derr == nil && body.Build == wantBuild { return nil } if derr != nil { last = fmt.Errorf("read /api/version: %v", derr) } else { last = fmt.Errorf("live build %q != expected %q", body.Build, wantBuild) } } else { last = err } time.Sleep(150 * time.Millisecond) } return fmt.Errorf("host didn't report build %q within %s: %v", wantBuild, timeout, last) } // ProbeVersion reads the build the process behind a socket is ACTUALLY // serving (its /api/version), not the symlink target — how a status probe // catches an awake hibernating host that repointed its symlink but still // runs the old binary. "" + ok=false when the socket doesn't answer (asleep, // or no such host). func ProbeVersion(socket string) (build string, ok bool) { resp, err := unixHTTPClient(socket).Get("http://carlos/api/version") if err != nil { return "", false } defer resp.Body.Close() var body struct { Build string `json:"build"` } if json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&body) != nil { return "", false } return body.Build, body.Build != "" } // unixHTTPClient dials a unix socket over HTTP — the edge<->host transport, // reused by the health checks. func unixHTTPClient(socket string) *http.Client { return &http.Client{ Timeout: 3 * time.Second, Transport: &http.Transport{ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return unixdial.Checked(ctx, &net.Dialer{}, socket) }, }, } }