package host import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "io/fs" "os" "path/filepath" "reflect" "sort" "strconv" "strings" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfiles" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/instancefs" "github.com/carlosframework/platform/internal/instancekey" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/regionname" "github.com/carlosframework/platform/internal/release" ) // movesFiles is the C8 files-manifest provider (claim §6.2a P6-D3, moves // spec §7.6, index C8; Q3 ruling). It is two pieces of region HA's files // publisher API, on hostfiles: // // - BuildPrepared, the prepared-manifest build an installation runs in // `local` before any session exists: create-only content objects plus // one create-only sequence-0 manifest under files/g-

/ in the // location, bound to {location, P, op, stamp}, replayed by digest and // key (hostfiles.Prepared). // - Establish, the reader a cold admission establishes its files manifest // with: the newest valid manifest of the admitted predecessor chain, // read by exact version, copied create-only into the admitted // generation with the env file's bytes on disk staged and the adoption // item dropped (hostfiles.NewFromCopy; instance key O1'). // // The session-bound publisher (sequence 1 onwards under a session) stays // region HA's; moves does not need it, the cut archive is the transfer. // // Missing or malformed data is an error, never an empty-files bootstrap // (claim §6.4a). The two empty starts are the claim's own bootstrap // (in.Empty, proved by EmptyAuthority before Establish runs) and a pin // that evidences a host that never published files (region HA stage B's // rule, hostfiles.PinnedFiles.NeverPublished, read through Evidence). // Absence of a manifest, of a pin, or of a reader is never evidence: with // Evidence nil, a chain without a manifest refuses every wake. type movesFiles struct { emptyRecovery claimEmptyRecovery Seed *filesSeedOperation // Files is this box's data bucket as a release.Store: current reads // with the object's etag, create-only and compare-and-swap writes // (the S3 store's Put with If-None-Match and If-Match). Every write // this provider makes goes through it. Files release.Store // Data is the same bucket as a versioned store (the claim's // hostbox.RequestStore in production): the chain's manifests and the // prepared manifest are read at the exact version the listing or the // op journal names, never by key, and BuildPrepared writes through it // create-only. Data lease.VersionedStore // DataDir is the box's data directory, where the op journals live // (moves.ReadJournal): the probe's admission reads the prepared // manifest's binding from the op journal local wrote. DataDir string // Bucket is the data bucket's name, HomeRegion the fleet's signed home // region, Region this box's engine region (park.PlacementRegion of its // placement) and Box its id. Every region comparison goes through // regionname.SameRegion with HomeRegion as the alias. Bucket, HomeRegion, Region, Box string // EnvFile resolves the host's env file on this box for the wake's // database path, as the launch source that minted the instance key into // it resolved it (claimEnvFileFor): the nested instance directory's for // a nested row, the flat one for a flat row, and an error for anything // it cannot place. EnvFile func(host, dbPath string) (claimEnvFile, error) // Proof proves origins against admissions the claim has recorded // (platform-1e's ruling, fix round 3; claim Plan 15): in production // ClaimOriginProof over the deployment store and the verified home // alias, one batch call per establishment. nil refuses: there is no // other route, and retained admissions are never evidence. Proof movesOriginProof // Evidence reads the claim pin's files evidence for a host, when the // claim carries a pin: region HA's ReadPin over the op journal's pin // reference. false means the claim carries no pin. nil means no pin // reader is wired on this box, which reads as no evidence. Evidence func(ctx context.Context, o hostclaim.Observed) (hostfiles.PinnedFiles, bool, error) } var _ ClaimFilesManifest = (*movesFiles)(nil) // preparedManifest is the files_manifest journal step's value: the // binding {location, P, op, stamp} of the prepared manifest an install // wrote, with the digest and the exact version of its bytes. Key is // /files/g-

/manifest.json, which names the location and P. // The probe's admission takes exactly Version and checks it against // SHA256 (fix round 1, finding 2). type preparedManifest struct { Key string `json:"key"` SHA256 string `json:"sha256"` Version string `json:"version"` Op string `json:"op"` Stamp uint64 `json:"stamp"` } // decodePreparedManifest decodes a journalled binding and holds it to the // expected key, op and stamp (fix round 1, finding 3). An empty value, // malformed JSON, a binding for another key, op or stamp, or one without a // digest or a version, is an error: the journal is damaged or another // install's, and the step is neither done nor rebuilt over it. func decodePreparedManifest(raw, key, op string, stamp uint64) (preparedManifest, error) { var pm preparedManifest if raw == "" { return pm, fmt.Errorf("%w: the journal's %s step is empty", errLandingChanged, landStepFilesManifest) } if err := json.Unmarshal([]byte(raw), &pm); err != nil { return pm, fmt.Errorf("%w: the journal's %s step %q: %v", errLandingChanged, landStepFilesManifest, raw, err) } if pm.Key != key || pm.Op != op || pm.Stamp != stamp || !claimHexDigest(pm.SHA256) || pm.Version == "" || pm.Version == "null" { return pm, fmt.Errorf("%w: the journal's %s step %+v is not the binding of %s by %s at stamp %d", errLandingChanged, landStepFilesManifest, pm, key, op, stamp) } return pm, nil } // movesFilesPrefix is a generation's files prefix in its location, // /files/g-/, the rule pinFilesLiveSet and the park // view's files live set share (spec §7.6), and region HA's relocation // with them: moves.GenerationFilesPrefix, the one rule. func movesFilesPrefix(location, generation string) string { return moves.GenerationFilesPrefix(location, generation) } // movesFilesGeneration accepts a generation name as the claim does // (claimFreshGeneration's shape), never the origin or a path. func movesFilesGeneration(g string) bool { return g != "" && g != "." && g != ".." && g != hostclaim.RootGeneration && len(g) <= 64 && !strings.ContainsAny(g, "/\\\x00\r\n") } // createOnlyStore is a release.Store over a versioned store for one // bucket: current reads by the listing's latest version, create-only // writes (PutIfAbsent), never a replace. BuildPrepared writes through it, // so an install needs only the box's versioned store. type createOnlyStore struct { data lease.VersionedStore bucket string } func (s createOnlyStore) Get(ctx context.Context, key string) ([]byte, string, error) { vs, err := s.data.ListVersions(ctx, s.bucket, key) if err != nil { return nil, "", err } for _, v := range vs { if v.Key == key && v.IsLatest { if v.DeleteMarker || v.VersionID == "" || v.VersionID == "null" { return nil, "", fmt.Errorf("host: %s is not a readable version: %w", key, os.ErrNotExist) } body, err := s.data.GetVersion(ctx, s.bucket, key, v.VersionID) return body, v.VersionID, err } } return nil, "", fmt.Errorf("host: %s: %w", key, os.ErrNotExist) } func (s createOnlyStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { if ifMatch != "" { return "", errors.New("host: the prepared manifest build never replaces an object") } id, err := s.data.PutIfAbsent(ctx, s.bucket, key, body) if errors.Is(err, lease.ErrExists) { return "", fmt.Errorf("%w: %s", release.ErrCASConflict, key) } return id, err } func (createOnlyStore) List(context.Context, string) ([]string, error) { return nil, errors.New("host: the prepared manifest build does not list") } func (createOnlyStore) ListDirs(context.Context, string) ([]string, error) { return nil, errors.New("host: the prepared manifest build does not list") } func (createOnlyStore) Delete(context.Context, string) error { return errors.New("host: the prepared manifest build never deletes") } // buildPreparedManifest is BuildPrepared over a versioned store: the // install's own (ir.versioned), which needs no compare-and-swap. func buildPreparedManifest(ctx context.Context, data lease.VersionedStore, bucket, location, generation, op string, stamp uint64, dir, envPath string, wantEnv bool, wake hostfiles.WakeState) (preparedManifest, error) { if data == nil || bucket == "" { return preparedManifest{}, hostwrite.ErrUnavailable } return buildPrepared(ctx, createOnlyStore{data: data, bucket: bucket}, location, generation, op, stamp, dir, envPath, wantEnv, wake) } // BuildPrepared writes the prepared manifest of generation (the // installation's prepared destination generation P, g- or // g-, never holder.generation: R19-1) under location: every // regular file under dir as a create-only object at sequence 0 (a file // over hostfiles.MaxFileBytes is recorded as skipped; anything not a // regular file is left out, as SyncDir leaves it), the env file at envPath // as the EnvItem, the wake state as the WakeItem, then the one create-only // sequence-0 manifest. It returns the binding for the op journal: the key, // the manifest's digest and exact version, op and stamp. A replay reuses // every object by digest and reads the manifest back by key; a different // manifest at the key is hostfiles.ErrManifestExists. // // Missing inputs are never an empty image (fix round 1, finding 4): dir // must exist (the install placed it), and the env file must exist exactly // when wantEnv says the install placed one (the archive's evidence, the // install receipt's Env identity). A vanished tree, a vanished env file, // or an env file nobody placed, refuses before any write. // // The caller runs it as op.exec in a phase that permits installation, // after the install's own claim sync and stamp capture (local_). func (f *movesFiles) BuildPrepared(ctx context.Context, location, generation, op string, stamp uint64, dir, envPath string, wantEnv bool, wake hostfiles.WakeState) (preparedManifest, error) { if f == nil { return preparedManifest{}, hostwrite.ErrUnavailable } return buildPreparedManifest(ctx, f.Data, f.Bucket, location, generation, op, stamp, dir, envPath, wantEnv, wake) } func buildPrepared(ctx context.Context, s release.Store, location, generation, op string, stamp uint64, dir, envPath string, wantEnv bool, wake hostfiles.WakeState) (preparedManifest, error) { if ctx == nil || location == "" || !movesFilesGeneration(generation) || op == "" || stamp == 0 || dir == "" || envPath == "" { return preparedManifest{}, fmt.Errorf("%w: a prepared manifest names its location, generation, op, stamp, files directory and env path", hostwrite.ErrUnavailable) } // The inputs first, before any write: the placed tree and the env file // as the install's evidence says. if st, err := os.Lstat(dir); err != nil || !st.IsDir() { return preparedManifest{}, fmt.Errorf("%w: the files directory %s the install placed is not there (%v)", errLandingChanged, dir, err) } env, err := os.ReadFile(envPath) switch { case err == nil && !wantEnv: return preparedManifest{}, fmt.Errorf("%w: an env file is at %s, which the install did not place", errLandingChanged, envPath) case errors.Is(err, fs.ErrNotExist) && wantEnv: return preparedManifest{}, fmt.Errorf("%w: the env file %s the install placed is not there", errLandingChanged, envPath) case err != nil && !errors.Is(err, fs.ErrNotExist): return preparedManifest{}, err } prefix := movesFilesPrefix(location, generation) p, err := hostfiles.NewPrepared(s, prefix) if err != nil { return preparedManifest{}, err } // The tree is read through an os.Root over dir, every name relative to // it (#236 fix round 1): a path within the files tree's cap (4095 bytes, // which the archive the install extracted holds every path to) is past // PATH_MAX once the data directory is prefixed, so an absolute walk // fails with ENAMETOOLONG after the cut. The root also keeps every read // inside dir. // // Census ruling moves-overcap-cut-236 (platform-1e, 2026-10-15; // binding): no read here falls back to a path-based open. A root that // cannot be opened, or a name it cannot open (readFileBounded), refuses // the build; nothing retries by absolute path. root, err := os.OpenRoot(dir) if err != nil { return preparedManifest{}, fmt.Errorf("%w: the files directory %s the install placed: %v", errLandingChanged, dir, err) } defer root.Close() err = fs.WalkDir(root.FS(), ".", func(rel string, d fs.DirEntry, err error) error { if err != nil { return err } if rel == "." || d.IsDir() || !d.Type().IsRegular() { return nil } info, err := d.Info() if err != nil { return err } // The size before any read, and the read bounded at the limit // plus one byte (fix round 1, finding 5): a file over the limit, by // its stat or by growing while it is read, is recorded as skipped // and never held whole in memory. if info.Size() > hostfiles.MaxFileBytes { return p.StageSkipped(rel, movesFilesTooLarge(info.Size())) } body, tooLarge, err := readFileBounded(root, rel, hostfiles.MaxFileBytes) if err != nil { return err } if tooLarge { return p.StageSkipped(rel, movesFilesTooLarge(int64(len(body)))) } return p.Stage(ctx, rel, body, uint32(info.Mode().Perm())) }) if err != nil { return preparedManifest{}, fmt.Errorf("prepared manifest of %s: %w", dir, err) } if wantEnv { if err := p.StagePlatform(ctx, hostfiles.EnvItem, env); err != nil { return preparedManifest{}, err } } body, err := json.Marshal(wake) if err != nil { return preparedManifest{}, fmt.Errorf("marshal wake state: %w", err) } if err := p.StagePlatform(ctx, hostfiles.WakeItem, body); err != nil { return preparedManifest{}, err } _, raw, version, err := p.Commit(ctx) if err != nil { return preparedManifest{}, err } if version == "" || version == "null" { return preparedManifest{}, fmt.Errorf("%w: the store answered no version for %s", hostwrite.ErrUnavailable, hostfiles.ManifestKey(prefix)) } sum := sha256.Sum256(raw) return preparedManifest{Key: hostfiles.ManifestKey(prefix), SHA256: hex.EncodeToString(sum[:]), Version: version, Op: op, Stamp: stamp}, nil } // movesFilesTooLarge is a skipped entry's reason for a file over the // limit; size is what the stat or the bounded read saw. func movesFilesTooLarge(size int64) string { return fmt.Sprintf("over the %d-byte limit (%d bytes or more)", hostfiles.MaxFileBytes, size) } // readFileBounded reads rel, relative to root, whole when it holds at most // limit bytes. It reads at most limit+1 bytes however large the file is or // becomes while it is read; tooLarge reports that the file exceeded the // limit, and the bytes returned with it are then not the whole file. A // test seam. var readFileBounded = func(root *os.Root, rel string, limit int64) ([]byte, bool, error) { // Through the root only (census ruling moves-overcap-cut-236, binding): // an open that fails is the error, never a retry by path. f, err := root.Open(rel) if err != nil { return nil, false, err } defer f.Close() body, err := io.ReadAll(io.LimitReader(f, limit+1)) if err != nil { return nil, false, err } return body, int64(len(body)) > limit, nil } // movesFilesChain is the admitted predecessor chain whose manifests are in // this region's bucket, newest first: the holder's generation, then each // cold admission's predecessor while it is a named generation in this // region (regionname.SameRegion against the home alias). A predecessor in // another region ends the walk: its objects are not here, and a target's // manifest never references a source-region object (P6-D3). Warm // admissions keep their generation and are passed over. func movesFilesChain(h *hostclaim.Holder, home, region string) []string { if h == nil { return nil } return movesFilesChainFrom(h, h.Generation, home, region) } // recoverySourceGeneration is the generation a recovery's files baseline // starts from (whole-branch fixes, item 3): for a restoring or reverting // op, the source's generation the claim journalled at the flip (src_gen), // which the recovery's database restore is pinned to as well; "" names a // legacy source with no files generation. ok is false for any other op. func recoverySourceGeneration(c *hostclaim.Claim) (string, bool) { if c == nil { return "", false } switch c.Op.Kind { case hostclaim.KindRestoring, hostclaim.KindReverting: return c.Op.Journal[hostclaim.JournalSrcGen], true } return "", false } // restoresSourceBaseline reports whether a recovery admission restores the // recorded source baseline (platform-1e's framing of item 3): a restoring // op that is not a dirty return (a dirty return journals its return seal // and restores the rollback generation or a certified descendant, never // the stopped source), with a journalled src_gen. The files bridge from // src_gen applies only to these, and only while their recovery lineage // holds no manifest (the database selector's "not rooted"); every other // admission follows its recovery lineage. func restoresSourceBaseline(c *hostclaim.Claim) bool { if c == nil || c.Op.Kind != hostclaim.KindRestoring || c.Op.Journal[moves.JournalReturnSeal] != "" { return false } src, ok := recoverySourceGeneration(c) return ok && src != "" } // movesFilesChainFrom is movesFilesChain started at generation start // instead of the holder's: start, then each cold admission's predecessor // in this region, as the admissions record them. func movesFilesChainFrom(h *hostclaim.Holder, start, home, region string) []string { if h == nil || !movesFilesGeneration(start) { return nil } cur := start chain := []string{cur} seen := map[string]bool{cur: true} for i := len(h.Admissions) - 1; i >= 0; i-- { a := h.Admissions[i] if a.Generation != cur || a.Mode != "cold" { continue } next := a.Predecessor if !movesFilesGeneration(next) || seen[next] || !regionname.SameRegion(home, a.PredecessorRegion, region) { break } chain = append(chain, next) seen[next] = true cur = next } return chain } // manifestVersion is the current version of a manifest key as the listing // names it: "" when the key has no current object. A delete marker or an // unversioned current object is an error. func (f *movesFiles) manifestVersion(ctx context.Context, key string) (string, error) { return movesManifestVersion(ctx, f.Data, f.Bucket, key) } // movesManifestVersion is manifestVersion over any versioned reader of the // data bucket: the cut's preflight reads with the moves worker's own // (sourceFilesRefusal), the establishment with the claim's. func movesManifestVersion(ctx context.Context, data lease.VersionedStore, bucket, key string) (string, error) { vs, err := data.ListVersions(ctx, bucket, key) if err != nil { return "", err } for _, v := range vs { if v.Key != key || !v.IsLatest { continue } if v.DeleteMarker || v.VersionID == "" || v.VersionID == "null" { return "", fmt.Errorf("host: the files manifest %s is not a readable version", key) } return v.VersionID, nil } return "", nil } // newestManifest is the newest valid manifest of the chain, read by exact // version, with the origins its entries name (movesFilesOrigins: the // location's rule, not the chain's). A generation without a manifest is // passed over; the first manifest found is validated and is the baseline, // malformed or not (a malformed newest manifest refuses, never an older // one in its place). found is false when no generation of the chain has a // manifest. func (f *movesFiles) newestManifest(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, location string, chain []string) (hostfiles.Manifest, []string, bool, error) { m, key, version, found, err := movesFilesSelect(ctx, f.Data, f.Bucket, location, chain) if err != nil || !found { return hostfiles.Manifest{}, nil, false, err } allowed, err := f.movesFilesAllowed(ctx, o, in, location, in.NewGeneration, m) if err != nil { return hostfiles.Manifest{}, nil, false, fmt.Errorf("host: the files manifest %s@%s: %w", key, version, err) } return m, allowed, true, nil } // movesFilesSelect is newestManifest's selection, apart from its // validation: the first generation of chain whose manifest key has a // current version, read at exactly that version, with the key and the // version. found is false when no generation of chain has a manifest. The // establishment (newestManifest, the src_gen bridge included) and the // cut's preflight (sourceFilesRefusal) both select through it, so the cut // checks the manifest a recovery would restore, not a guess at it (#236). func movesFilesSelect(ctx context.Context, data lease.VersionedStore, bucket, location string, chain []string) (m hostfiles.Manifest, key, version string, found bool, err error) { for _, g := range chain { prefix := movesFilesPrefix(location, g) key := hostfiles.ManifestKey(prefix) version, err := movesManifestVersion(ctx, data, bucket, key) if err != nil { return hostfiles.Manifest{}, "", "", false, err } if version == "" { continue } m, found, err := hostfiles.Load(ctx, versionStore{data: data, bucket: bucket, key: key, version: version}, prefix) if err != nil { return hostfiles.Manifest{}, "", "", false, err } if !found { return hostfiles.Manifest{}, "", "", false, fmt.Errorf("host: the files manifest %s@%s is not readable", key, version) } return m, key, version, true, nil } return hostfiles.Manifest{}, "", "", false, nil } // movesFilesRestorable is the reader's path cap as moves applies it // (hostfiles.RestorableWithinCap, the rule checkManifest applies to every // entry, live, tombstoned or skipped): a manifest that fails it is // hostfiles.ErrTooDeep, and no establishment restores it. The recovery's // establishment (movesFilesAllowed) and the cut's preflight // (sourceFilesRefusal) both call it, so a move whose recovery could not // restore the source's files never starts its cut (#236; Paul's ruling of // 2026-10-15). If region HA relaxes the readers' rule, RestorableWithinCap // follows it, and both sides here with it. func movesFilesRestorable(m hostfiles.Manifest) error { return hostfiles.RestorableWithinCap(m) } // movesOriginProof proves that generations of the holder's incarnation // were admitted, by the claim's own records (platform-1e's ruling, fix // round 3; claim Plan 15): provenance comes from the claim, never from a // manifest, whose writer holds the bucket credentials that could write // its Origins too. The implementation is Plan 15's ClaimOriginProof: a // batch Proven over the whole wanted set, which authenticates the bound // observation against the stored claim (the CAS result the caller holds, // never a bare Admission.NewGeneration), walks the claim's recorded // admissions once from its committed head, stops as soon as every wanted // generation is proven, bounds each node read, and takes home aliases only // through the verified roster binding. The chain starts empty: nothing is // seeded from retained admissions, which are never evidence, and a claim // with no recorded head proves nothing, so every existing host refuses // until the #250 files backfill has run. That is intended. A false answer // is ErrOrigin; an error is the error. type movesOriginProof interface { Proven(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, generations []string) (map[string]bool, error) } // proveOrigins is the one route to origin proof (fix round 4; Plan 15): // one Proven call over the deduplicated set of generations, and a // generation's files prefix under location enters the allowed set only // when that call proves it. A generation the proof does not prove is // ErrOrigin, naming the first; an error is the error; no proof wired // refuses. No per-origin calls (Plan 15's review: one walk per // establishment). func (f *movesFiles) proveOrigins(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, location string, generations []string) ([]string, error) { if f.Proof == nil { return nil, fmt.Errorf("%w: no origin proof is wired", hostwrite.ErrUnavailable) } set := make([]string, 0, len(generations)) seen := map[string]bool{} for _, gen := range generations { if !movesFilesGeneration(gen) { return nil, fmt.Errorf("%w: %q is not a generation name", hostfiles.ErrOrigin, gen) } if !seen[gen] { seen[gen] = true set = append(set, gen) } } proven, err := f.Proof.Proven(ctx, o, in, set) if err != nil { return nil, fmt.Errorf("prove the origins of %s: %w", location, err) } origins := make([]string, 0, len(set)) for _, gen := range set { if !proven[gen] { return nil, fmt.Errorf("%w: %s is not proven by the claim's recorded admissions", hostfiles.ErrOrigin, movesFilesPrefix(location, gen)) } origins = append(origins, movesFilesPrefix(location, gen)) } // The prefixes are sorted, not the generation names: the two do not // order alike ("mv1" before "mv1-old", "g-mv1-old/" before "g-mv1/"), // and this is the carried set SetOrigins stores and establishedOver // compares (Astra's finding on the Plan 15 switch). sort.Strings(origins) return origins, nil } // movesFilesAllowed is the origin set an admission allows for m, the // manifest it establishes from (Task 6b, the moves side of claim Plan 15's // review): the origins m's entries actually reference, each proven against // the claim's records through proveOrigin after the location rule // (movesFilesOrigins), plus the one generation this admission opens // (admitted: proven through the same route, round 4's binding, one call). // Nothing else is proven and nothing else is carried: the retained chain's // other generations are not candidates, and a carried hint // (Manifest.Origins) that no entry references is neither proven nor // carried forward, so the proof's cost follows the live references, not // the history. A referenced origin the proof cannot prove is ErrOrigin. // Then ValidateManifest over the proven set, which is the new manifest's // carried set (SetOrigins). Nothing refused before is accepted now: every // referenced origin and the admitted generation go through the proof as // before; only unreferenced hints and unreferenced chain generations, which // no entry depends on, are no longer asked about. func (f *movesFiles) movesFilesAllowed(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, location, admitted string, m hostfiles.Manifest) ([]string, error) { // The path cap first, by the rule the cut's preflight applies // (movesFilesRestorable): ValidateManifest below refuses the same // manifest with the same error, so this only refuses it before any // origin proof, and keeps the two sides on one function. if err := movesFilesRestorable(m); err != nil { return nil, err } named, err := movesFilesOrigins(location, "", m) if err != nil { return nil, err } base := moves.FilesGenerationBase(location) generations := []string{admitted} for _, origin := range named { rest, under := strings.CutPrefix(origin, base) gen, slash := strings.CutSuffix(rest, "/") if !under || !slash || !movesFilesGeneration(gen) { return nil, fmt.Errorf("%w: %s is not a generation's files prefix of %s", hostfiles.ErrOrigin, origin, location) } generations = append(generations, gen) } // One Proven call over the deduplicated referenced origins plus the // admitted generation (Plan 15). allowed, err := f.proveOrigins(ctx, o, in, location, generations) if err != nil { return nil, err } if err := hostfiles.ValidateManifest(m, allowed); err != nil { return nil, err } return allowed, nil } // movesFilesOrigins is the set of origins a manifest's objects name, each // held to the claim's provenance rule (P6-D3): an immutable object within // the authorized incarnation location in this region's bucket, under a // generation's files prefix, /files/g-/. A manifest // naming nothing has its own prefix as its one origin. The admitted chain // selects which manifest is newest; it does not bound the origins that // manifest may name, because AdmitWriter keeps eight admissions while a // copied manifest carries its oldest objects for as long as they are live // (fix round 1, finding 6). The same location rule is the files live // set's (pinFilesLiveSet), so every object a live manifest names stays // pinned under it. func movesFilesOrigins(location, prefix string, m hostfiles.Manifest) ([]string, error) { base := moves.FilesGenerationBase(location) seen := map[string]bool{} var out []string for _, entries := range []map[string]hostfiles.Entry{m.Entries, m.Platform} { for name, e := range entries { if e.Object == "" || e.Deleted || e.Skipped != "" { continue } origin, _, ok := strings.Cut(e.Object, "objects/") rest, under := strings.CutPrefix(origin, base) gen, slash := strings.CutSuffix(rest, "/") if !ok || !under || !slash || !movesFilesGeneration(gen) { return nil, fmt.Errorf("%w: %s names %s, not an object under a generation's files prefix of %s", hostfiles.ErrOrigin, name, e.Object, location) } if !seen[origin] { seen[origin] = true out = append(out, origin) } } } if len(out) == 0 && prefix != "" { out = []string{prefix} } return out, nil } // openPrepared is the probe's baseline (in.Prepared, or a relocating // op's opening of P, preparedOpening): P's prepared manifest at exactly // the version its installation's binding names (fix round 1, finding 2). // The binding is read by preparedBinding, which chooses its reader by the // claim's op kind, and is held to P's key, the claim's op and the // installation's stamp; the bytes are read at that version, // checked against the bound digest, and validated as a prepared // manifest (sequence 0, objects within the location by the claim's rule, // movesFilesOrigins: a cross-region landing's names destination-local // objects by its build, a same-region relocation's may retain source // generation objects). The key must // currently hold either those bytes (the first establishment, which the // returned publisher replaces by compare-and-swap) or this box's own // establishment of them (a retry after a crash past the commit: the same // entries, the env bytes on disk, no adoption item, which the publisher // then leaves as it is). Anything else refuses before any write. func (f *movesFiles) openPrepared(ctx context.Context, files release.Store, o hostclaim.Observed, in hostclaim.Admission, gen, prefix string, env []byte) (*hostfiles.Publisher, error) { if f.DataDir == "" { return nil, hostwrite.ErrUnavailable } h := o.Claim.Holder key := hostfiles.ManifestKey(prefix) pm, err := f.preparedBinding(o, gen, key) if err != nil { return nil, err } exact, err := f.Data.GetVersion(ctx, f.Bucket, key, pm.Version) if err != nil { return nil, fmt.Errorf("host: the prepared files manifest %s@%s: %w", key, pm.Version, err) } if sum := sha256.Sum256(exact); hex.EncodeToString(sum[:]) != pm.SHA256 { return nil, fmt.Errorf("%w: the prepared files manifest %s@%s is not the journalled bytes", hostbox.ErrChanged, key, pm.Version) } prepared, found, err := hostfiles.Load(ctx, exactBytesStore{key: key, body: exact, version: pm.Version}, prefix) if err != nil || !found { return nil, fmt.Errorf("host: the prepared files manifest %s@%s: %v", key, pm.Version, err) } // The prepared manifest's allowed set: the origins its entries // reference, proven, plus P itself (its carried set, seeded by the // build with its own prefix, is a hint only; an old one carries none). if len(prepared.Origins) == 0 { prepared.Origins = []string{prefix} } allowed, err := f.movesFilesAllowed(ctx, o, in, h.Location, gen, prepared) if err != nil { return nil, fmt.Errorf("host: the prepared files manifest %s@%s: %w", key, pm.Version, err) } if prepared.Seq != 0 { return nil, fmt.Errorf("%w: the prepared files manifest %s@%s is at sequence %d, not 0", hostbox.ErrChanged, key, pm.Version, prepared.Seq) } // One read of the key: the bytes and the token the publisher is built // from are the ones checked here, never a second read (fix round 2, // finding 2). A replacement after this read fails the commit's // compare-and-swap. cur, token, err := files.Get(ctx, key) if err != nil { return nil, err } if !bytes.Equal(cur, exact) { current, found, err := hostfiles.Load(ctx, exactBytesStore{key: key, body: cur, version: token}, prefix) if err != nil || !found { return nil, fmt.Errorf("host: the files manifest %s: %v", key, err) } if err := establishedOver(ctx, files, h.Location, prefix, allowed, current, prepared, env); err != nil { return nil, fmt.Errorf("%w: the files manifest %s is neither the prepared manifest %s nor this box's establishment of it: %v", hostbox.ErrChanged, key, pm.Version, err) } } p, err := hostfiles.OpenAt(files, prefix, cur, token) if err != nil { return nil, err } p.SetOrigins(allowed) return p, nil } // preparedBinding is the binding of P's prepared manifest, read by the // reader the claim's op kind names, never by which file happens to exist: // a move's install journals it in the moves op journal // (.moves//

/journal.json, the files_manifest step); a // relocation's install records it in its own install journal // (relocate/ops//install.json), which the claim's local_install // receipt binds by digest, and the claim's prepared_authority binds its // bytes. Either way the binding names key, op, stamp, digest and version. // The absence of one reader's file is never permission to use the other's, // and any other kind opens no prepared generation. func (f *movesFiles) preparedBinding(o hostclaim.Observed, gen, key string) (preparedManifest, error) { switch kind := o.Claim.Op.Kind; { case kind == hostclaim.KindRelocating: return relocationPreparedManifest(f.DataDir, f.Box, o, gen, key) case claimMovesKind(kind): return movesPreparedManifest(f.DataDir, o, gen, key) default: return preparedManifest{}, fmt.Errorf("%w: a %s claim opens no prepared generation", hostbox.ErrChanged, kind) } } // movesPreparedManifest is a move's binding: the op journal's // files_manifest step, held to P's key, the claim's op and the // installation's stamp (installationStamp). func movesPreparedManifest(dataDir string, o hostclaim.Observed, gen, key string) (preparedManifest, error) { j, err := moves.ReadJournal(dataDir, o.Host, gen) if err != nil { return preparedManifest{}, err } raw, ok := j.Steps[landStepFilesManifest] if !ok { return preparedManifest{}, fmt.Errorf("%w: the op journal of %s/%s has no prepared files manifest", errLandingChanged, o.Host, gen) } stamp, err := installationStamp(o.Claim) if err != nil { return preparedManifest{}, err } return decodePreparedManifest(raw, key, o.Claim.Op.ID, stamp) } // relocationPreparedManifest is a relocation's binding (region HA §5.8). // It holds two stamps apart, each exactly. P's build stamp is the one the // claim's prepared_authority binds (install_stamp when ToInstalled ran): // the authority names P, that stamp and the digest of P's prepared files // manifest (sequence 0), and the install journal's files_manifest step // binds P's key, the op, that build stamp, the digest and the exact // version. This box's install stamp is the claim's current install_stamp, // which a replacement target issues anew while keeping P and its // authority (hostclaim ReplaceTarget at installed, or in probing before // P's first admission): the holder's stamp (the probe's admission keeps // it), the claim's local_install receipt (this box, that stamp, the // digest of its install journal) and the journal itself (read whole, held // to the receipt's digest, naming the host, the op, this box, that stamp // and P) all carry it. The build precedes every install, so the build // stamp is never past the install stamp. The step's digest must be the // authority's. Anything missing or different refuses; neither stamp // stands in for the other. func relocationPreparedManifest(dataDir, box string, o hostclaim.Observed, gen, key string) (preparedManifest, error) { c := o.Claim auth, err := preparedAuthority(c) if err != nil { return preparedManifest{}, err } stamp, err := relocationInstallStamp(c) if err != nil { return preparedManifest{}, err } if auth.Generation != gen || auth.FilesSeq != 0 || !claimHexDigest(auth.FilesManifestSHA256) || auth.InstallStamp == 0 || auth.InstallStamp > stamp || c.Holder.Stamp != stamp { return preparedManifest{}, fmt.Errorf("%w: the prepared authority %+v is not a build of P %s at or before the install stamp %d", hostbox.ErrChanged, auth, gen, stamp) } var receipt hostclaim.LocalInstall if json.Unmarshal([]byte(c.Op.Journal["local_install"]), &receipt) != nil || receipt.Box != box || receipt.InstallStamp != stamp || !claimHexDigest(receipt.JournalSHA256) { return preparedManifest{}, fmt.Errorf("%w: the claim journals no local install receipt of this box at the install stamp", hostbox.ErrChanged) } path, err := relocationInstallPath(dataDir, c.Op.ID) if err != nil { return preparedManifest{}, err } raw, err := os.ReadFile(path) if err != nil { return preparedManifest{}, fmt.Errorf("host: the relocation's install journal: %w", err) } if hostfilesSum(raw) != receipt.JournalSHA256 { return preparedManifest{}, fmt.Errorf("%w: %s is not the install journal the claim's receipt names", hostbox.ErrChanged, path) } var j relocationInstallJournal if err := json.Unmarshal(raw, &j); err != nil { return preparedManifest{}, fmt.Errorf("%w: %s: %v", hostbox.ErrChanged, path, err) } if j.V != 1 || j.Host != o.Host || j.Op != c.Op.ID || j.Box != box || j.InstallStamp != stamp || j.Prepared != gen { return preparedManifest{}, fmt.Errorf("%w: %s is not this box's install of %s at the install stamp", hostbox.ErrChanged, path, gen) } step, ok := j.Steps[relocationStepFilesManifest] if !ok { return preparedManifest{}, fmt.Errorf("%w: the install journal of %s has no prepared files manifest", errLandingChanged, c.Op.ID) } pm, err := decodePreparedManifest(step, key, c.Op.ID, auth.InstallStamp) if err != nil { return preparedManifest{}, err } if pm.SHA256 != auth.FilesManifestSHA256 { return preparedManifest{}, fmt.Errorf("%w: the install bound %s at %s, not the prepared authority's digest", hostbox.ErrChanged, key, pm.SHA256) } return pm, nil } // preparedOpening reports whether this admission opens a prepared // generation whose files manifest an installation built. For a move it is // in.Prepared. For a relocation it is the probe's admission that opens the // journalled P, whether or not the database is empty: relocation always // builds P's files manifest (§5.8), so database emptiness (in.Empty) and // files preparation are independent, and an empty database still starts // from P's files. A relocating admission that says prepared or empty for // any other generation is refused. func preparedOpening(o hostclaim.Observed, in ClaimLaunchInputs) (bool, error) { c := o.Claim if c.Op.Kind != hostclaim.KindRelocating { return in.Prepared, nil } p := c.Op.Journal["prepared"] opens := p != "" && c.Op.Phase == hostclaim.PhaseRelocatingProbing && in.Admission.NewGeneration == p if (in.Prepared || in.Empty) && !opens { return false, fmt.Errorf("%w: a relocating admission is prepared or empty only when it opens P", hostbox.ErrChanged) } return opens, nil } // installationStamp is the stamp the installation that wrote P's prepared // manifest captured (ir.capture at local), validated apart from the // admitted session's authority (whole-branch fixes, item 2). A return's // install (restoring, or a revert's) ran at the source installation's // stamp S, which the claim journals at the flip (source_stamp) and the // recovery's cold admission then replaces with a new one; a landing's // install ran at the flip's stamp, which the first probe's admission // keeps (KeepStamp, since no writer was admitted between the installing // CAS and the probe), so it is the holder's. func installationStamp(c *hostclaim.Claim) (uint64, error) { switch c.Op.Kind { case hostclaim.KindRestoring, hostclaim.KindReverting: stamp, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalSourceStamp], 10, 64) if err != nil || stamp == 0 { return 0, fmt.Errorf("%w: the claim journals no source installation stamp for %s", errLandingChanged, c.Op.ID) } return stamp, nil } if c.Holder == nil || c.Holder.Stamp == 0 { return 0, fmt.Errorf("%w: the holder has no installation stamp", errLandingChanged) } return c.Holder.Stamp, nil } // establishedOver reports whether cur is this box's establishment of // prepared, in full (fix round 2 finding 6, fix round 3): exactly what // this establishment would have written, derived from the prepared // baseline and the establishment's own operations by the publisher's // rules, never from the replay's own numbers. From prepared's sequence, // the env file's bytes take a new object at the next sequence when they // differ from prepared's env item; the adoption item's deletion takes the // sequence after when prepared carried one; the manifest's sequence is // the last handed out; the entries and other platform items are // prepared's; the carried set is the admitted one. cur must equal that // manifest field for field, and the env object must be at its key with // the env's digest. A replay whose env entry names an object elsewhere, a // different object under P, or a fabricated sequence whose object does // not exist, is not this box's establishment however its digest reads. func establishedOver(ctx context.Context, files release.Store, location, prefix string, allowed []string, cur, prepared hostfiles.Manifest, env []byte) error { want := hostfiles.Manifest{V: hostfiles.ManifestVersion, Entries: map[string]hostfiles.Entry{}, Platform: map[string]hostfiles.Entry{}, Origins: allowed} for k, v := range prepared.Entries { want.Entries[k] = v } for k, v := range prepared.Platform { want.Platform[k] = v } next := prepared.Seq envSHA := hostfilesSum(env) if e, ok := prepared.Platform[hostfiles.EnvItem]; !ok || e.SHA256 != envSHA { next++ want.Platform[hostfiles.EnvItem] = hostfiles.Entry{Object: fmt.Sprintf("%sobjects/%016d-%s", prefix, next, envSHA), SHA256: envSHA, Size: int64(len(env)), Mode: 0o600, Seq: next} } if _, has := prepared.Platform[instancekey.AdoptionItem]; has { next++ delete(want.Platform, instancekey.AdoptionItem) } want.Seq = next if cur.Entries == nil { cur.Entries = map[string]hostfiles.Entry{} } if cur.Platform == nil { cur.Platform = map[string]hostfiles.Entry{} } if !reflect.DeepEqual(cur, want) { return fmt.Errorf("the manifest %+v is not what this establishment writes over the prepared one (%+v)", cur, want) } if _, err := movesFilesOrigins(location, prefix, cur); err != nil { return err } if err := hostfiles.ValidateManifest(cur, allowed); err != nil { return err } e := want.Platform[hostfiles.EnvItem] body, _, err := files.Get(ctx, e.Object) if err != nil { return fmt.Errorf("the env object %s: %w", e.Object, err) } if hostfilesSum(body) != e.SHA256 || int64(len(body)) != e.Size { return fmt.Errorf("the env object %s is not the env file's bytes", e.Object) } return nil } func hostfilesSum(body []byte) string { sum := sha256.Sum256(body) return hex.EncodeToString(sum[:]) } // exactBytesStore is a read-only release.Store holding one object's exact // bytes at a version already read: hostfiles.Load's reader for a manifest // the caller verified. type exactBytesStore struct { key, version string body []byte } func (s exactBytesStore) Get(_ context.Context, key string) ([]byte, string, error) { if key != s.key { return nil, "", os.ErrNotExist } return s.body, s.version, nil } func (exactBytesStore) Put(context.Context, string, []byte, string) (string, error) { return "", errors.New("host: the prepared manifest's exact bytes are read only") } func (exactBytesStore) List(context.Context, string) ([]string, error) { return nil, errors.New("host: the prepared manifest's exact bytes are read only") } func (exactBytesStore) ListDirs(context.Context, string) ([]string, error) { return nil, errors.New("host: the prepared manifest's exact bytes are read only") } func (exactBytesStore) Delete(context.Context, string) error { return errors.New("host: the prepared manifest's exact bytes are read only") } // selectionStore is the data bucket as Establish writes it (fix round 1, // finding 7): every write is fenced by the start's captured selection // immediately before the request (work.With with no effect, the // writerFence idiom: the lock is taken for the check and released before // the request), and every request is bounded by hostbox.RequestLimit. No // request is made under the lock, so a stalled request never delays // BeginFence; a selection that no longer holds refuses the write. type selectionStore struct { release.Store fence func(context.Context) error } func (s selectionStore) bound(ctx context.Context) (context.Context, context.CancelFunc) { return context.WithTimeout(ctx, hostbox.RequestLimit) } func (s selectionStore) Get(ctx context.Context, key string) ([]byte, string, error) { ctx, cancel := s.bound(ctx) defer cancel() return s.Store.Get(ctx, key) } func (s selectionStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { if err := s.fence(ctx); err != nil { return "", err } ctx, cancel := s.bound(ctx) defer cancel() return s.Store.Put(ctx, key, body, ifMatch) } // emptyManifest is the base of an empty start. func emptyManifest() hostfiles.Manifest { return hostfiles.Manifest{V: hostfiles.ManifestVersion, Entries: map[string]hostfiles.Entry{}, Platform: map[string]hostfiles.Entry{}} } // Establish is ClaimFilesManifest.Establish. It runs once per start under // the start's captured selection, after the launch source has minted the // instance key into .env and before the app executes. The env file // is the one the launch source minted into, found through the host's row // (EnvFile; a nested row's is in its instance directory). // // The baseline: // - a prepared opening (preparedOpening: in.Prepared for a move, the // probe's admission opening P for a relocation, whatever its // database): P's own prepared manifest at the version its // installation bound (preparedBinding, by the op's kind), read current // with its etag so the env revision replaces exactly it (a new // version; the sequence-0 version stays), validated against the // origins its entries name; // - in.Empty (the claim's empty bootstrap, EmptyAuthority already run): // no manifest; // - otherwise the newest valid manifest of the chain, read by exact // version; with none, the pin's never-published evidence or an error // (hostfiles.ErrNoEvidence), never an empty bootstrap. // // Then the env file's exact bytes on disk are staged as the EnvItem, the // adoption item is dropped and the revision is committed (create-only for // a new generation; compare-and-swap on P), each bucket write fenced by // the start's selection immediately before it and bounded by the request // limit, never under the lock the work holds (selectionStore; fix round // 1, finding 7). Then, under work, the local adoption record is removed: // the one local effect. Establish never writes the env file. An error, a // bucket that cannot be reached included, fails the wake. func (f *movesFiles) Establish(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error { if f == nil || f.Files == nil || f.Data == nil || f.Bucket == "" || f.Region == "" || f.Box == "" || f.EnvFile == nil || ctx == nil || work == nil { return hostwrite.ErrUnavailable } if o.Absent || o.Claim == nil || o.Claim.Holder == nil || o.Host == "" || in.Identity.Config == "" || in.Identity.Release == "" { return hostbox.ErrChanged } h := o.Claim.Holder gen := in.Admission.NewGeneration if in.Admission.Box != f.Box || !regionname.SameRegion(f.HomeRegion, in.Admission.Region, f.Region) || !movesFilesGeneration(gen) || h.Location == "" { return hostbox.ErrChanged } envFile, err := f.EnvFile(o.Host, in.DBPath) if err != nil { return err } env, err := readClaimEnvFile(envFile) if err != nil { return fmt.Errorf("host: the env file the files manifest must carry: %w", err) } prepared, err := preparedOpening(o, in) if err != nil { return err } prefix := movesFilesPrefix(h.Location, gen) var storage release.Store = f.Files if f.Seed != nil { if err := f.Seed.check(ctx, o, in); err != nil { return err } storage = filesSeedCheckingStore{Store: f.Files, data: f.Data, bucket: f.Bucket, prefix: prefix} } files := selectionStore{Store: storage, fence: writerFence(work, in.Identity)} var p *hostfiles.Publisher switch { case prepared: p, err = f.openPrepared(ctx, files, o, in.Admission, gen, prefix, env) if err != nil { return err } case in.Empty: own, err := f.proveOrigins(ctx, o, in.Admission, h.Location, []string{gen}) if err != nil { return err } p, err = hostfiles.NewFromCopy(files, prefix, emptyManifest(), nil) if err != nil { return err } p.SetOrigins(own) default: // The chain is the holder's: the newest authorized manifest through // it wins, a recovery's own included (the prepared rollback // generation's, or a later recovery publication's). Only when that // chain holds no manifest, the first recovery admission, whose // predecessor is the target's generation in the other region, does // src_gen bridge the missing source baseline (item 3 as re-ruled; // the database selector's distinction, movesrecoveryowners.go // baseline: the recorded source restore only while the recovery has // no root of its own). base, origins, found, err := f.newestManifest(ctx, o, in.Admission, h.Location, movesFilesChain(h, f.HomeRegion, f.Region)) if err != nil { return err } if !found && restoresSourceBaseline(o.Claim) { src, _ := recoverySourceGeneration(o.Claim) base, origins, found, err = f.newestManifest(ctx, o, in.Admission, h.Location, movesFilesChainFrom(h, src, f.HomeRegion, f.Region)) if err != nil { return err } } if !found { never := false if f.Evidence != nil && f.Seed == nil { pin, ok, err := f.Evidence(ctx, o) if err != nil { return fmt.Errorf("host: the claim pin's files evidence: %w", err) } never = ok && pin.NeverPublished() } if !never && f.Seed == nil { return fmt.Errorf("%w: no files manifest in the admitted predecessor chain of %s (%s)", hostfiles.ErrNoEvidence, o.Host, h.Generation) } own, err := f.proveOrigins(ctx, o, in.Admission, h.Location, []string{gen}) if err != nil { return err } base, origins = emptyManifest(), own } p, err = hostfiles.NewFromCopy(files, prefix, base, origins) if err != nil { return err } p.SetOrigins(origins) } if f.Seed != nil { if err := f.Seed.stage(ctx, o, in, work, p, envFile); err != nil { return err } } if err := p.StagePlatform(ctx, hostfiles.EnvItem, env); err != nil { return err } if err := p.StagePlatformDelete(instancekey.AdoptionItem); err != nil { return err } if f.Seed != nil { if err := f.Seed.check(ctx, o, in); err != nil { return err } } if _, err := p.Commit(ctx); err != nil { return err } return work.With(ctx, in.Identity, func() error { return removeClaimAdoption(envFile, o.Host+instancekey.AdoptionSuffix) }) } // readClaimEnvFile reads the env file's exact bytes: in a nested instance // directory, which the tenant can write, only beneath it (readInstanceFile: // no link at any component, a regular file, at most the env read limit), // and a missing file is an error either way. func readClaimEnvFile(e claimEnvFile) ([]byte, error) { if !e.Nested { return os.ReadFile(e.Path()) } env, absent, err := readInstanceFile(e.Dir, e.Name, instancekey.EnvReadLimit) if err == nil && absent { err = fmt.Errorf("%s: %w", e.Path(), fs.ErrNotExist) } return env, err } // removeClaimAdoption removes the adoption record beside the env file, a // nested one beneath the instance directory (instancefs.RemoveBeneath). A // record that is not there is already removed. func removeClaimAdoption(e claimEnvFile, name string) error { var err error if e.Nested { err = instancefs.RemoveBeneath(e.Dir, name) } else { err = os.Remove(filepath.Join(e.Dir, name)) } if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } return nil }