package layout import ( "crypto/rand" "encoding/hex" "errors" "fmt" "os" "path/filepath" "strings" "time" "github.com/carlosframework/platform/internal/instancefs" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" ) // Preflight is the ownership check of design §8.2 step 4, plus the socket // length check of §3.1. Under the flat rules .config.env is host's // rendered bundle UNLESS a route named .config exists, in which case // it is that route's instance env (SyncExec's own collision rule). Moving // it with host would take another route's file; reverting onto it would // overwrite one. Either is a refusal naming both owners. -sidecar.env // is shared the same way with a route named -sidecar. func Preflight(p *Plan, routes []registry.Route) error { hosts := map[string]bool{} for _, rt := range routes { hosts[rt.Host] = true } if other := p.Host + ".config"; hosts[other] { return fmt.Errorf("layout: %s: %s.config.env is owned by both %s (its bundle) and %s (its instance env); refusing to move it", p.Host, p.Host, p.Host, other) } if strings.HasSuffix(p.Host, ".config") { if owner := strings.TrimSuffix(p.Host, ".config"); hosts[owner] { return fmt.Errorf("layout: %s: its env file %s.env is owned by both %s (its instance env) and %s (its bundle); refusing to move it", p.Host, p.Host, p.Host, owner) } } // The sidecar env collides the same way: -sidecar.env is host's // sidecar env, and also the instance env of a route named // -sidecar. if other := p.Host + "-sidecar"; hosts[other] { return fmt.Errorf("layout: %s: %s-sidecar.env is owned by both %s (its sidecar env) and %s (its instance env); refusing to move it", p.Host, p.Host, p.Host, other) } if strings.HasSuffix(p.Host, "-sidecar") { if owner := strings.TrimSuffix(p.Host, "-sidecar"); hosts[owner] { return fmt.Errorf("layout: %s: its env file %s.env is owned by both %s (its instance env) and %s (its sidecar env); refusing to move it", p.Host, p.Host, p.Host, owner) } } // The activator binds a control socket beside every exec listener at // the listener's path plus ".control". A nested address that leaves no // room for it would make the host unwakeable after the move, so it is // refused before anything moves. if p.Mode == ModeNested { if n := len(p.To.Socket) + len(".control"); n > provision.MaxUnixPath { return fmt.Errorf("layout: %s: control socket %s.control is %d bytes, over the %d a unix socket address holds; refusing to move it", p.Host, p.To.Socket, n, provision.MaxUnixPath) } } return nil } // DestinationCheck is step 5, run before the journal so that at the moment // the journal is written nothing the pass will move has anything at its // destination, and, on a migration, so that the instance directory holds // nothing the pass did not put there. A database, or any file of one, // already at the destination is a refusal (two databases; refusing to // choose), and so is a litestream shadow (two replication states for one // database). An env file already there, which an older binary running // between a migration and a revert can have minted fresh, is moved aside // to the pass's own aside directory so the source, the file the instance // ran with for longer, takes its place; a quarantined restore is kept the // same way. The rendered bundle is simply cleared; sync re-renders it // anyway. // // The aside copy lives OUTSIDE the instance tree, in /layout/aside/, // where no instance's namespace reaches (design §4.1 binds the instance // its own directory and nothing else of /data). Inside the tree it would // be the tenant's to read, and a tenant that reclaims a hostname, or that // planted the file itself before its own migration, must not read what a // previous owner or a sibling left at that name. // // Every change goes through a directory fd that follows no symlink, so a // planted link at a destination directory cannot redirect it. func DestinationCheck(p *Plan, now func() time.Time) (aside []string, err error) { if p.Mode == ModeNested { moved, err := quarantineForeignEntries(p) aside = append(aside, moved...) if err != nil { return aside, err } } for _, a := range p.Artifacts { if a.Name == "socket" { continue } present, err := lstatPresent(a.Dst) if err != nil { return aside, err } if !present { continue } dir, name := filepath.Dir(a.Dst), filepath.Base(a.Dst) switch a.Name { case "db", "db-wal", "db-shm": return aside, fmt.Errorf("layout: %s: two databases (%s exists at the destination); refusing to choose", p.Host, a.Dst) case "litestream-shadow": return aside, fmt.Errorf("layout: %s: two replication states (%s exists at the destination); refusing to choose", p.Host, a.Dst) case "env", "sidecar-env", "corrupt": asideDir, err := ensureAsideDir(p.DataDir, p.Host) if err != nil { return aside, fmt.Errorf("layout: %s: %w", p.Host, err) } to := fmt.Sprintf("%s.aside-%d", name, now().Unix()) taken, err := lstatPresent(filepath.Join(asideDir, to)) if err != nil { return aside, err } if taken { return aside, fmt.Errorf("layout: %s: %s and %s both exist; refusing to replace a kept copy", p.Host, a.Dst, filepath.Join(asideDir, to)) } if err := instancefs.RenameBetween(dir, name, asideDir, to); err != nil { return aside, fmt.Errorf("layout: %s: move %s aside: %w", p.Host, a.Dst, err) } aside = append(aside, filepath.Join(asideDir, to)) case "config-env": if err := instancefs.RemoveBeneath(dir, name); err != nil { return aside, fmt.Errorf("layout: %s: clear %s: %w", p.Host, a.Dst, err) } default: return aside, fmt.Errorf("layout: %s: no destination rule for artifact %q", p.Host, a.Name) } } return aside, nil } // asideDir is where the pass keeps what it moved out of a host's way: // under the pass's own layout directory, per host, which no instance // namespace contains. It is created on first use, carlos 0700 like the // journal directory above it, through fds that follow no symlink. func asideDir(dataDir, host string) string { return filepath.Join(journalDir(dataDir), "aside", host) } func ensureAsideDir(dataDir, host string) (string, error) { if _, err := instancefs.EnsureDirBeneath(dataDir, filepath.Base(journalDir(dataDir)), 0o700); err != nil { return "", err } if err := ensureDirDisplacing(dataDir, journalDir(dataDir), "aside"); err != nil { return "", err } if err := ensureDirDisplacing(dataDir, filepath.Join(journalDir(dataDir), "aside"), host); err != nil { return "", err } return asideDir(dataDir, host), nil } // ensureDirDisplacing makes name a real directory under parent. The layout // directory and everything under it are carlos 0700, which every flat // tenant can write, so a tenant can plant a symlink or a file at the aside // directory, or at a sibling's directory under it, ahead of the pass; left // as a refusal that would be a hold on the sibling (or on every host) with // one command. A blocker that is not a real directory is renamed out of // the way to a random, no-replace name under the layout directory itself, // where the operator can see it, and the directory is made once more. // Only a second failure is returned. func ensureDirDisplacing(dataDir, parent, name string) error { _, err := instancefs.EnsureDirBeneath(parent, name, 0o700) if err == nil { return nil } to := fmt.Sprintf(".%s.blocker-%d-%s", name, time.Now().Unix(), randomSuffix()) if rerr := instancefs.RenameBetweenNoReplace(parent, name, journalDir(dataDir), to); rerr != nil { return fmt.Errorf("%w (and moving the blocker aside: %v)", err, rerr) } if _, err := instancefs.EnsureDirBeneath(parent, name, 0o700); err != nil { return err } return nil } // quarantineForeignEntries moves everything out of a migration's instance // data directory that the plan does not name, into the host's aside // directory, and reports where each went. Until every flat tenant on the // box has migrated, the raw directory is writable by every one of them // (same uid, §1.3), and a tenant can make a sibling's future directory // ahead of the pass and hard-link the sibling's database or the box's // bucket credentials into it, which protected_hardlinks allows because // the files are its own uid's; the sibling's namespace would then bind // that directory read-write. So the directory the pass is about to fill // must hold nothing but what the pass itself moves in: an entry at a // planned name is handled by the destination rules (refused, cleared, or // kept aside), and any other entry is renamed out here. // // Renamed, not refused: a refusal would let one tenant hold a sibling // down with one touch(1) until an operator acted. rename(2) moves the // directory entry itself, whatever it names, a hard link or a symlink // included, so the quarantined entry is as harmless in the aside // directory as it was dangerous in the tenant's, and the original file // (if the entry was a second name for one) keeps its own name wherever // that is. The destination name carries a random suffix and is written // with RENAME_NOREPLACE, so nothing pre-planted in the aside directory // can make the quarantine collide. A tenant's own leftovers from an // earlier nested life (a revert leaves what the app wrote beside its // database) are quarantined the same way; the aside copy is kept for // the operator. The run directory is not checked: it is on tmpfs, where // a hard link to anything under /data is impossible, and its mount is // nosymfollow. func quarantineForeignEntries(p *Plan) ([]string, error) { dir := rawInstanceDir(p.DataDir, p.Host) entries, err := os.ReadDir(dir) if errors.Is(err, os.ErrNotExist) { return nil, nil } if err != nil { return nil, fmt.Errorf("layout: %s: list %s: %w", p.Host, dir, err) } planned := map[string]bool{} for _, a := range p.Artifacts { if a.Name != "socket" { planned[filepath.Base(a.Dst)] = true } } var moved []string for _, e := range entries { if planned[e.Name()] { continue } aside, err := ensureAsideDir(p.DataDir, p.Host) if err != nil { return moved, fmt.Errorf("layout: %s: %w", p.Host, err) } var last error for attempt := 0; attempt < 3; attempt++ { to := fmt.Sprintf("%s.q-%d-%s", e.Name(), time.Now().Unix(), randomSuffix()) last = instancefs.RenameBetweenNoReplace(dir, e.Name(), aside, to) if last == nil { moved = append(moved, filepath.Join(aside, to)) break } if !errors.Is(last, os.ErrExist) { break } } if last != nil { return moved, fmt.Errorf("layout: %s: quarantine %s out of %s: %w", p.Host, e.Name(), dir, last) } } return moved, nil } func randomSuffix() string { var b [4]byte if _, err := rand.Read(b[:]); err != nil { return fmt.Sprintf("%d", time.Now().UnixNano()) } return hex.EncodeToString(b[:]) } // LinkCheck refuses a plan whose files have more than one name. It runs // on the sources right before the journal is written, and again on the // destinations after the renames and before the row flips (VerifyMoved): // a regular file with a second hard link somewhere is a second reader and // writer of it, and a tenant that linked a sibling's database into its own // shadow directory before migrating would otherwise carry that link into // its namespace. The shadow directory is walked, and a symlink or anything // that is neither a regular file nor a directory inside it is refused too: // litestream writes only files and directories there. // // This is a check against what was planted before the pass arrived. A // flat tenant that is still running while the pass moves another host can // still plant between the check and the rename, which is why a box is not // isolated until no flat tenant remains on it (Converge's awake bound, and // the mixed-layout warning in carlos status --layout). func LinkCheck(p *Plan, at func(Artifact) string) error { for _, a := range p.Artifacts { if a.Name == "socket" || !a.Present { continue } path := at(a) fi, err := os.Lstat(path) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { return fmt.Errorf("layout: %s: %w", p.Host, err) } if a.Dir { if err := walkLinks(p.Host, p.DataDir, path); err != nil { return err } continue } if a.Extra && fi.IsDir() { // Said before the walk, which stats every file: a blob store // can be large, the host is stopped meanwhile, and the walk // runs twice (sources, then destinations). p.logf("layout: %s: checking every file under %s for hard links and symlinks before the move goes on", p.Host, path) if err := walkExtraLinks(p.Host, p.DataDir, path); err != nil { return err } continue } if err := oneName(p.Host, p.DataDir, path, fi); err != nil { return err } } return nil } // walkExtraLinks is walkLinks for a directory an app made for itself // (extras.go), with one allowance the shadow directory does not need: a // file with several names may be moved when EVERY one of those names is // inside the directory, since the move carries them all and nothing // outside can reach the file afterwards. Apps deduplicate content by // hard-linking inside their own blob store, and refusing that would hold // a host down for its own ordinary files. A name outside the directory // is still a second reader and writer the move does not know about, and // is refused with the inode and the find that lists it. Symlinks and // special files are refused as in walkLinks. func walkExtraLinks(host, dataDir, root string) error { type linked struct { path string nlink uint64 seen uint64 } byInode := map[uint64]*linked{} err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil { return fmt.Errorf("layout: %s: %w", host, err) } if d.IsDir() { return nil } fi, err := d.Info() if err != nil { return fmt.Errorf("layout: %s: %w", host, err) } if !fi.Mode().IsRegular() { return fmt.Errorf("layout: %s: %s is not a regular file (%v); refusing to move it", host, path, fi.Mode().Type()) } if n, ino := linkCount(fi); n > 1 { l := byInode[ino] if l == nil { l = &linked{path: path, nlink: n} byInode[ino] = l } l.seen++ } return nil }) if err != nil { return err } for ino, l := range byInode { if l.seen < l.nlink { return fmt.Errorf("layout: %s: %s has %d names (hard links, inode %d) and only %d of them are inside %s; refusing to move a file something else can still reach. find the others with: find %s -xdev -samefile %s", host, l.path, l.nlink, ino, l.seen, root, dataDir, l.path) } } return nil } func walkLinks(host, dataDir, root string) error { return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil { return fmt.Errorf("layout: %s: %w", host, err) } if d.IsDir() { return nil } fi, err := d.Info() if err != nil { return fmt.Errorf("layout: %s: %w", host, err) } return oneName(host, dataDir, path, fi) }) } // oneName refuses a file that is not a regular file with exactly one // name. The refusal tells the operator how to find the other names: the // inode, and the find invocation that lists every path to it on the data // volume (-xdev, because a hard link cannot cross a filesystem). func oneName(host, dataDir, path string, fi os.FileInfo) error { if !fi.Mode().IsRegular() { return fmt.Errorf("layout: %s: %s is not a regular file (%v); refusing to move it", host, path, fi.Mode().Type()) } if n, ino := linkCount(fi); n > 1 { return fmt.Errorf("layout: %s: %s has %d names (hard links, inode %d); refusing to move a file something else can still reach. find the others with: find %s -xdev -samefile %s", host, path, n, ino, dataDir, path) } return nil } // VerifyMoved is the check after the renames and before the row flips: // every moved file still has one name, and on a migration the instance // directory holds nothing but the plan's files, anything else planted // between the destination check and the renames being quarantined now. // It reports what it quarantined. A link failure keeps the journal at its // files phase, so the host stays fenced and the next recovery meets the // same check with the same answer until an operator has looked. func VerifyMoved(p *Plan) ([]string, error) { if err := LinkCheck(p, func(a Artifact) string { return a.Dst }); err != nil { return nil, err } if p.Mode == ModeNested { return quarantineForeignEntries(p) } return nil, nil } // EnsureDirs makes the raw directories the bind mounts show: the data // directory 0700 (the tenant's own ground) and the run directory 2750 in // the carlos-sockets group, so an enrolled consumer can dial app.sock // (design §3.1). The directories must exist before the row flips, because // Restore and prepare both assume them. // // The pass runs as carlos, which is not a member of carlos-sockets. It // therefore cannot chown anything into that group, and any chmod it makes // on a directory of that group has the kernel clear setgid, which on the // run parent would hand every later instance directory the wrong group. // So nothing here changes a mode or an owner. The run directory gets its // group and setgid by inheritance from /.instances, which tmpfiles // owns, exactly as prepare relies on; both parents are checked and never // repaired, and a new or existing instance directory that is not what // inheritance should have made is refused as it stands. // // Each directory is made and checked through an fd on its parent that // followed no symlink: a same-uid tenant can plant a symlink at // /.instances/ before the pass arrives, and a move into it // would land the database wherever the link points. func EnsureDirs(dataDir, socketDir, host string, socketsGID int) error { data := rawInstanceDir(dataDir, host) run := rawRunDir(socketDir, host) // The run parent is checked before anything is created, so a refusal // leaves nothing behind. runParent, err := checkRunParent(filepath.Dir(run), socketsGID) if err != nil { return fmt.Errorf("layout: %s: %w", host, err) } // The data parent is tmpfiles' too, but carlos:carlos 0700 is exactly // what mkdir as carlos makes, so a missing one is created, never // changed. if _, err := instancefs.EnsureDirBeneath(dataDir, filepath.Base(filepath.Dir(data)), 0o700); err != nil { return fmt.Errorf("layout: %s: %w", host, err) } st, err := instancefs.EnsureDirBeneath(filepath.Dir(data), filepath.Base(data), 0o700) if err != nil { return fmt.Errorf("layout: %s: %w", host, err) } if err := checkInstanceDir(data, st, 0o700, false, -1); err != nil { return fmt.Errorf("layout: %s: %w", host, err) } st, err = instancefs.EnsureDirBeneath(filepath.Dir(run), filepath.Base(run), 0o750) if err != nil { return fmt.Errorf("layout: %s: %w", host, err) } if err := checkInstanceDir(run, st, 0o750, runParent.Mode&os.ModeSetgid != 0, socketsGID); err != nil { return fmt.Errorf("layout: %s: %w", host, err) } return nil } // checkRunParent reads /.instances. With the sockets group known it // must be there already, in that group and setgid, because only tmpfiles, // running as root, can make it so. Without the group (a box that has none, // or a test) a missing parent is simply created. func checkRunParent(parent string, socketsGID int) (instancefs.DirStat, error) { if socketsGID < 0 { return instancefs.EnsureDirBeneath(filepath.Dir(parent), filepath.Base(parent), 0o750) } st, err := instancefs.StatDir(parent) if errors.Is(err, os.ErrNotExist) { return st, fmt.Errorf("%s is missing; tmpfiles should create it, and the pass does not", parent) } if err != nil { return st, err } if st.GID != socketsGID { return st, fmt.Errorf("%s has group %d, not carlos-sockets (%d); tmpfiles should restore it, and the pass does not repair it", parent, st.GID, socketsGID) } if st.Mode&os.ModeSetgid == 0 { return st, fmt.Errorf("%s has lost its setgid bit (mode %v); tmpfiles should restore it, and the pass does not repair it", parent, st.Mode) } return st, nil } // checkInstanceDir refuses an instance directory the pass cannot vouch // for: not its own, the wrong permission bits, no setgid under a setgid // parent, or not the sockets group. Each would need a chmod or chown to // fix, which is exactly what the pass must not do. func checkInstanceDir(path string, st instancefs.DirStat, perm os.FileMode, wantSetgid bool, gid int) error { if st.UID != os.Geteuid() { return fmt.Errorf("%s is owned by uid %d, not the agent's %d; refusing to use it", path, st.UID, os.Geteuid()) } if st.Mode.Perm() != perm { return fmt.Errorf("%s has mode %v, not %v; refusing to use it", path, st.Mode.Perm(), perm) } if wantSetgid && st.Mode&os.ModeSetgid == 0 { return fmt.Errorf("%s has no setgid bit though its parent has; refusing to use it", path) } if gid >= 0 && st.GID != gid { return fmt.Errorf("%s has group %d, not carlos-sockets (%d); refusing to use it", path, st.GID, gid) } return nil } func lstatPresent(path string) (bool, error) { _, err := os.Lstat(path) if errors.Is(err, os.ErrNotExist) { return false, nil } return err == nil, err }