package host import ( "context" "encoding/json" "errors" "fmt" "io" "log" "os" "path/filepath" "strings" "time" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // The moves protocol floor on a box (moves spec §11). Once an operator // publishes control/moves-protocol.json, no box may activate an executable // below its min_release: not by update, not by rollback, not by recovery, // not by picking carlos.prev at boot. The floor is the one Phase 1 element // that is not behind CARLOS_MOVES_PHASE1 (index D13): publishing the object // arms it, and turning the switch off later does not disarm it. A // deployment that never published it has none of the files below, and // every path here then does what main does. // // The files, and who writes them: // // - /.moves/floor.json and the one-line floor.min-release: the // floor as this box last read it. Root's apply writes them from the // store before it stages anything (armFloor); the agent refreshes them // every hour (movesFloorLoop). Both write under .moves/floor.lock, and a // floor only rises: a lower published floor is refused, and lowering // one is an operator's manual step on each box (floorRegression). Never // deleted by this code: a store that no longer answers, or no longer // holds the object, leaves them alone. floor.json is the one every Go // reader trusts; floor.min-release is for the units (writeFloorFiles). // - /carlos.prev.version: carlos.prev's release, written by root // whenever it replaces carlos.prev (decidePrev), or learned by running // `carlos.prev version` when it is missing and a floor is armed. // - /carlos.prev.allowed, the verdict: the min_release carlos.prev // was judged at or above, written only while no registry row is held by // an operation (a .prev is a downgrade by definition). judgePrev is the // one judge, run by root at every apply and every adopt pass. // // The two system units choose carlos.prev for a recovery only when the // floor line is absent, or equal byte for byte to the verdict. The version // comparison happens once, here in Go (moves.ReleaseAtOrAbove, which fails // closed); the units only compare two files, through carlos.prev's floor // gate (movesfloorgate.go), which never follows a symlink or waits on a // FIFO the agent planted. // // What this is not: a defence against a compromised agent. The agent owns // /data and can remove the floor files there. The floor protects the // protocol against old builds and honest mistakes, which is what §11 asks. const movesFloorMaxBytes = 4 << 10 // movesFloorRefreshEvery is the agent's refresh cadence (moves spec §11: // a raised floor reaches every box within the hour). const movesFloorRefreshEvery = time.Hour func prevVersionPath(optDir string) string { return filepath.Join(optDir, moves.PrevVersionFile) } func prevAllowedPath(optDir string) string { return filepath.Join(optDir, moves.PrevAllowedFile) } // validFloorRelease is a min_release the floor can be armed at: a release // moves.ReleaseAtOrAbove can compare, so nothing that would fail closed on // every comparison is ever written to disk. func validFloorRelease(v string) bool { ok, err := moves.ReleaseAtOrAbove(v, v) return ok && err == nil } // readLocalFloor reads /.moves/floor.json without following a // symlink anywhere below dataDir (root reads it, and the agent owns the // tree). (nil, nil) means no floor is armed on this box. A file that is // there but cannot be read, or does not hold a usable floor, is an error: // the callers treat that as armed and unknown, never as absent. func readLocalFloor(dataDir string) (*moves.Floor, error) { f, err := openBeneath(dataDir, filepath.Dir(moves.FloorFile), filepath.Base(moves.FloorFile)) if errors.Is(err, os.ErrNotExist) { return nil, nil } if err != nil { return nil, err } defer f.Close() st, err := f.Stat() if err != nil { return nil, err } if !st.Mode().IsRegular() || st.Size() > movesFloorMaxBytes { return nil, fmt.Errorf("%s is not a regular file under %d bytes", f.Name(), movesFloorMaxBytes) } var fl moves.Floor if err := json.NewDecoder(io.LimitReader(f, movesFloorMaxBytes)).Decode(&fl); err != nil { return nil, fmt.Errorf("%s: %w", f.Name(), err) } if !validFloorRelease(fl.MinRelease) { return nil, fmt.Errorf("%s: min_release %q is not a release", f.Name(), fl.MinRelease) } return &fl, nil } // errFloorLockBusy is floor.lock still held by another process after the // wait: a writer is active, and may be about to publish a floor. Never a // reason to go ahead without the lock (fix round 3): an activation that // meets it is deferred. var errFloorLockBusy = errors.New("another writer holds the floor lock on this box") // floorLockWait bounds the wait for floor.lock (movesDir.Lock); a variable // so a test can make contention time out quickly. var floorLockWait = 2 * time.Minute // movesDirName is the directory both local floor files live in, below // the data dir. var movesDirName = filepath.Dir(moves.FloorFile) // withFloorLock runs fn holding .moves/floor.lock (see floorLockFile). With // create, .moves is made when missing (a writer arming the first floor). // Without it, a missing .moves means no floor is armed and nothing can be // locked: fn runs with a nil dir. An error opening or locking is returned // without running fn. // // Lock order on census boxes: root release exclusion, the apply flock for // system updates, carlos.prev lock, participation transaction, moves floor, then // registry write lock. A held-stage lock is a leaf. Adoption omits the apply // flock. Guarded close and rollback receive the outer release token; they // never acquire release exclusion while holding floor or registry locks. func withFloorLock(dataDir string, create bool, fn func(d *movesDir) error) error { d, err := openMovesDir(dataDir, create) if errors.Is(err, os.ErrNotExist) && !create { return fn(nil) } if err != nil { return err } defer d.Close() if err := d.Lock(); err != nil { return err } return fn(d) } // writeLocalFloor writes both local floor files under the floor lock. func writeLocalFloor(dataDir string, f moves.Floor) error { if !validFloorRelease(f.MinRelease) { return fmt.Errorf("min_release %q is not a release", f.MinRelease) } return withFloorLock(dataDir, true, func(d *movesDir) error { return writeFloorFiles(d, f) }) } // writeFloorFiles writes floor.min-release FIRST and floor.json LAST, under // the floor lock. Every Go reader (activationAllowed, judgePrev, the edges // file) trusts floor.json alone; floor.min-release exists for the units' // shell, which can only compare files. So after a crash between the two, // floor.min-release can only be AHEAD of floor.json (floors only rise, see // floorRegression), and ahead means it matches no verdict judged against // the JSON: the units run the current binary, whose own check then uses the // JSON. The two can never disagree in the direction that would let the // units pick a carlos.prev the Go checks would refuse. func writeFloorFiles(d *movesDir, f moves.Floor) error { body, err := json.Marshal(f) if err != nil { return err } if err := d.WriteDurable(filepath.Base(moves.FloorMinReleaseFile), []byte(f.MinRelease+"\n"), 0o644); err != nil { return err } return d.WriteDurable(filepath.Base(moves.FloorFile), append(body, '\n'), 0o644) } // floorRegression says why remote may not replace local, or nil. A floor // only rises on a box: a lower protocol minimum, or a min_release that is // lower or does not compare, is refused. Lowering a floor is an operator's // manual step, box by box: delete /.moves/floor.json and // floor.min-release (and publish the lower object), and the next apply or // refresh arms what is published. func floorRegression(local, remote *moves.Floor) error { if remote.Min < local.Min { return fmt.Errorf("the published floor's protocol minimum %d is below this box's %d; lowering a floor is done by hand on each box", remote.Min, local.Min) } if ok, _ := moves.ReleaseAtOrAbove(remote.MinRelease, local.MinRelease); !ok { return fmt.Errorf("the published min_release %q is below this box's %s; lowering a floor is done by hand on each box", remote.MinRelease, local.MinRelease) } return nil } // errFloorWrite marks a floor the store holds that this box could not // write down: the one syncLocalFloor failure that leaves the box enforcing // less than what is published. var errFloorWrite = errors.New("could not write it on this box") // floorWriteError is errFloorWrite with what the write said. Its text is // what it always was ("could not write it on this box: "); armFloor // reads cause to say the recorded sentence whole (copy record MV-N92). type floorWriteError struct{ cause error } func (e *floorWriteError) Error() string { return fmt.Sprintf("%v: %v", errFloorWrite, e.cause) } func (e *floorWriteError) Is(target error) bool { return target == errFloorWrite } // syncLocalFloor is the rule root's apply and the agent's refresh share: // read the published floor and make the local files say the same. It // returns the floor on disk afterwards, nil when none is armed. // // - published, usable and not lower: written, under the floor lock, with // the comparison made under the same lock, so an older observation that // finishes after a newer one cannot lower the floor; // - lower than the floor on disk: refused (floorRegression), logged; // - absent from the store: what is on disk stays (once armed, armed); // - the store unreadable, or the object unusable: what is on disk stays, // and the error is returned to be logged. // // A local floor.json that cannot be read is replaced by a usable published // floor: it holds no value to compare, and the published one is the // authority it was copied from. func syncLocalFloor(ctx context.Context, s release.Store, dataDir string) (*moves.Floor, error) { if s == nil { local, lerr := readLocalFloor(dataDir) return local, errors.Join(errors.New("no deployment store to read the floor from"), lerr) } remote, rerr := moves.ReadFloor(ctx, s) switch { case rerr != nil: local, lerr := readLocalFloor(dataDir) return local, errors.Join(rerr, lerr) case remote == nil: return readLocalFloor(dataDir) case !validFloorRelease(remote.MinRelease): local, lerr := readLocalFloor(dataDir) return local, errors.Join(fmt.Errorf("the published min_release %q is not a release; the floor on this box stays as it was", remote.MinRelease), lerr) } var out *moves.Floor ran := false err := withFloorLock(dataDir, true, func(d *movesDir) error { ran = true local, lerr := readLocalFloor(dataDir) if lerr == nil && local != nil { if err := floorRegression(local, remote); err != nil { out = local return err } if *local == *remote { if line, ok := readBeneathLine(dataDir, moves.FloorMinReleaseFile); ok && line == remote.MinRelease { out = local return nil } } } if err := writeFloorFiles(d, *remote); err != nil { out, _ = readLocalFloor(dataDir) return &floorWriteError{cause: err} } out = remote return nil }) if !ran { out, _ = readLocalFloor(dataDir) return out, &floorWriteError{cause: err} } return out, err } // readBeneathLine reads a one-line file under root without following a // symlink below root, trimmed. false means absent, unreadable or empty. func readBeneathLine(root, rel string) (string, bool) { f, err := openBeneath(root, strings.Split(filepath.ToSlash(rel), "/")...) if err != nil { return "", false } defer f.Close() if st, err := f.Stat(); err != nil || !st.Mode().IsRegular() { return "", false } return readLineFrom(f) } // readLine reads a one-line file the reader trusts the path of (root's own // /opt/carlos, or the agent's own /data), trimmed. false means absent, // unreadable or empty. func readLine(path string) (string, bool) { f, err := os.Open(path) if err != nil { return "", false } defer f.Close() return readLineFrom(f) } func readLineFrom(r io.Reader) (string, bool) { b, err := io.ReadAll(io.LimitReader(r, 256)) if err != nil { return "", false } s := strings.TrimSpace(string(b)) return s, s != "" } // writePrevVersion records carlos.prev's release beside it, durably. func writePrevVersion(optDir, version string) error { if !validSystemVersion(version) { return fmt.Errorf("carlos.prev: %q is not a version", version) } return writeFileDurable(prevVersionPath(optDir), []byte(version+"\n"), 0o644) } // removeDurable removes path and, when it was there, fsyncs its directory. // A path that is already gone is removed. func removeDurable(path string) error { err := os.Remove(path) if errors.Is(err, os.ErrNotExist) { return nil } if err != nil { return err } return syncDir(filepath.Dir(path)) } // holdFreezer runs fn with the rows an operation holds, held still for // fn's length (registry.WithHoldsFrozen's shape). type holdFreezer func(fn func(held []registry.Route) error) error // judgePrev is the one judge of carlos.prev (moves spec §11): it writes the // verdict when carlos.prev is at or above the armed floor and no registry // row is held by an operation, and removes it otherwise. With no floor // armed it touches neither the registry nor carlos.prev, and leaves no // verdict (the units do not read one then). func judgePrev(optDir, dataDir string, reg *registry.Registry) error { return judgePrevWith(optDir, dataDir, func(fn func([]registry.Route) error) error { if reg == nil { return errors.New("no registry") } return reg.WithHoldsFrozen(fn) }, nil) } // judgePrevWith is judgePrev with its two inputs as functions: freeze gives // the held rows, held still while the verdict is written, and learn (nil to // skip) answers carlos.prev's version when the sidecar is missing; what it // learns is written as the sidecar. Learning runs carlos.prev, which can // take seconds, so it happens before any lock is taken. // // The judgement itself runs under the floor lock with the holds frozen, so // the verdict it writes is for the floor and the holds as they stand when // it lands. Every refusal removes the verdict before it returns, so an // error here never leaves an old verdict standing. func judgePrevWith(optDir, dataDir string, freeze holdFreezer, learn func() string) error { deny := func(cause error) error { if err := removeDurable(prevAllowedPath(optDir)); err != nil { return errors.Join(cause, fmt.Errorf("remove the verdict on carlos.prev: %w", err)) } return cause } if _, err := os.Stat(systemPrevPath(optDir)); err != nil { return deny(nil) } if _, ok := readLine(prevVersionPath(optDir)); !ok && learn != nil { if f, _ := readLocalFloor(dataDir); f != nil { if v := learn(); validSystemVersion(v) { if err := writePrevVersion(optDir, v); err != nil { return deny(err) } } } } ran := false err := withFloorLock(dataDir, false, func(*movesDir) error { ran = true floor, err := readLocalFloor(dataDir) if err != nil { return deny(fmt.Errorf("moves protocol floor: the floor on this box cannot be read: %v", err)) } if floor == nil { return deny(nil) } version, ok := readLine(prevVersionPath(optDir)) if !ok { return deny(nil) } if at, _ := moves.ReleaseAtOrAbove(version, floor.MinRelease); !at { return deny(nil) } called := false err = freeze(func(held []registry.Route) error { called = true if len(held) > 0 { return deny(nil) } return writeFileDurable(prevAllowedPath(optDir), []byte(floor.MinRelease+"\n"), 0o644) }) if !called { return deny(fmt.Errorf("held rows: %w", err)) } return err }) if !ran { return deny(fmt.Errorf("moves protocol floor: the floor on this box cannot be read: %v", err)) } return err } // prevLockFile serialises the two writers of carlos.prev's sidecar and // verdict: decidePrev, which replaces carlos.prev under the apply lock, and // the root adopt pass, which re-judges it under no apply lock at all. // Without it an adopt pass that read the old sidecar could write its // verdict after decidePrev had replaced carlos.prev and removed it. const prevLockFile = "prev.lock" const ( prevLockWait = 2 * time.Minute prevLockPoll = 100 * time.Millisecond ) // withPrevLock runs fn holding prevLockFile, waiting up to prevLockWait. // Holders are short (a binary copy at most), so a lock that is not free by // then is reported, not waited out. func withPrevLock(optDir string, fn func() error) error { dir := systemRootDir(optDir) if err := os.MkdirAll(dir, 0o755); err != nil { return err } lock, err := os.OpenFile(filepath.Join(dir, prevLockFile), os.O_CREATE|os.O_RDWR, 0o600) if err != nil { return err } defer lock.Close() deadline := time.Now().Add(prevLockWait) for { err := flockExclusiveNonblock(lock) if err == nil { break } if errors.Is(err, errSystemUnsupported) || time.Now().After(deadline) { return fmt.Errorf("carlos.prev lock: %w", err) } time.Sleep(prevLockPoll) } return fn() } // rootJudgePrev is the root adopt pass's judgement (every mode): the moves // half, then the seeds half (seedsfloor.go). Each runs whatever the other // returns. func rootJudgePrev(reg *registry.Registry) error { return rootJudgePrevWithSource(reg, rootMovesFloorIdentity) } func rootJudgePrevWithSource(reg *registry.Registry, source rootFloorIdentitySource) error { if id, required, err := source.read(); err != nil { return err } else if required { return errors.Join(rootJudgeMovesPrevWithSource(reg, source), rootRefreshAndJudgeSeedsPrev(id)) } return errors.Join(rootJudgeMovesPrevWithSource(reg, source), rootJudgeSeedsPrev()) } func rootJudgeMovesPrevWithSource(reg *registry.Registry, source rootFloorIdentitySource) error { if id, required, err := source.read(); err != nil { return err } else if required { return rootRefreshAndJudgeMovesPrev(id) } optDir, dd := release.OptDir(), dataDir() if _, err := os.Stat(filepath.Join(dd, moves.FloorFile)); errors.Is(err, os.ErrNotExist) { // No floor armed: nothing to judge and no verdict to keep. The one // stat is all an unarmed box pays. if _, err := os.Stat(prevAllowedPath(optDir)); errors.Is(err, os.ErrNotExist) { return nil } } return withPrevLock(optDir, func() error { return judgePrevWith(optDir, dd, reg.WithHoldsFrozen, func() string { out, err := productionSysOps().run(context.Background(), systemVersionWait, systemPrevPath(optDir), "version") if err != nil { return "" } return carlosVersionOf(out) }) }) } // movesFloorRefresh is one pass of the agent's refresh: syncLocalFloor on // the agent's own store handle, logged when it fails, then the seeds // floor's (seedsFloorRefresh) on the same handle. func movesFloorRefresh(ctx context.Context, s release.Store, dataDir string, logf func(string, ...any)) { if _, err := syncLocalFloor(ctx, s, dataDir); err != nil { logf("agent: moves protocol floor refresh: %v", err) } seedsFloorRefresh(ctx, s, dataDir, logf) } // movesFloorLoop refreshes the local floor files at start and every hour. // Started by edgesFileLoop, whose file publishes what it writes. func movesFloorLoop(ctx context.Context, s release.Store, dataDir string) { movesFloorRefresh(ctx, s, dataDir, log.Printf) tick := time.NewTicker(movesFloorRefreshEvery) defer tick.Stop() for range tick.C { movesFloorRefresh(ctx, s, dataDir, log.Printf) } } // withMovesFields sets the edges file's four moves fields from this box's // own files, read at each write: the protocol this build speaks, the armed // floor's min_release (absent: not armed), carlos.prev's release ("none": // there is no carlos.prev; absent: unknown) and the verdict (absent: no // verdict, which the door reads as below the floor). func withMovesFields(f edgesFile, dataDir, optDir string) edgesFile { // The public root marker is the agent's mode indication; it cannot inspect // the private census. G2 installs this marker before completing migration. if _, err := os.Lstat(rootFloorDir(optDir)); !errors.Is(err, os.ErrNotExist) { return withRootMovesFields(f, optDir) } f.MovesProtocol = moves.ProtocolVersion // From floor.json, the file every Go reader trusts: floor.min-release // is written first and can be ahead of it (writeFloorFiles). f.MovesFloorMinRelease = "" if fl, err := readLocalFloor(dataDir); err == nil && fl != nil { f.MovesFloorMinRelease = fl.MinRelease } f.PrevRelease = "" if _, err := os.Stat(systemPrevPath(optDir)); errors.Is(err, os.ErrNotExist) { f.PrevRelease = "none" } else if err == nil { f.PrevRelease, _ = readLine(prevVersionPath(optDir)) } f.PrevAllowedFor, _ = readLine(prevAllowedPath(optDir)) return f }