// Package hostfiles publishes an instance's files directory, and the small // platform-managed items that travel with it, to object storage in an order // that survives late and duplicate uploads (region HA spec 1 ยง5.6). // // Nothing is overwritten: each change is a new immutable object, and one // manifest, replaced by compare-and-swap, says which object is current. package hostfiles import ( "context" "encoding/json" "errors" "fmt" "os" "path" "strings" "github.com/carlosframework/platform/internal/release" ) // MaxFileBytes is the largest file published as one object. release.Store // takes a whole body in memory, so a larger file is recorded as skipped // rather than read. Plan 1b lifts this with a streaming put. const MaxFileBytes = 64 << 20 // Entry is one path's state. Object is origin-qualified: the object's full // key from the bucket root, "objects/-", where origin // is the files prefix of the generation that wrote it. A manifest copied // into a later generation keeps every Object as it was, so the copy names // the same immutable objects and nothing is uploaded again (claim L13). // Readers resolve Object exactly as written, never against their own // prefix, and only within the origins their caller admits. type Entry struct { Object string `json:"object,omitempty"` SHA256 string `json:"sha256,omitempty"` Size int64 `json:"size,omitempty"` Mode uint32 `json:"mode,omitempty"` Seq int64 `json:"seq"` Deleted bool `json:"deleted,omitempty"` Torn bool `json:"torn,omitempty"` Skipped string `json:"skipped,omitempty"` } // ManifestVersion is the manifest format this binary reads and writes. // // Version 2 made Entry.Object origin-qualified (a full key) where version // 1 held a key relative to the manifest's prefix. That changes what an // existing field means; it adds nothing a reader could ignore. A version 1 // reader would join a qualified key to its own prefix and look in the // wrong place, and a version 1 manifest copied to another generation // would resolve against the copy's prefix. So the version is bumped and // every other version is refused. No version 1 manifest was written in // production (nothing called the publisher), so there is nothing to // migrate. const ManifestVersion = 2 type Manifest struct { V int `json:"v"` Seq int64 `json:"seq"` Entries map[string]Entry `json:"entries"` Platform map[string]Entry `json:"platform"` // Origins is the admitted origin set the manifest carries forward // (moves Task 6 fix round 2, platform-1e's ruling): the generation // files prefixes whose objects its entries may name, sorted. A // prepared manifest seeds it with its own prefix; each admission's // establishment sets the new manifest's to the previous manifest's set // plus the generations admitted in the then-current chain plus its // own prefix, so a pruned ancestor stays proven by it and a generation // never admitted cannot enter it. A manifest without the field (the // format before this) carries none, and a reader falls back to the // admitted chain alone. Additive: every other field is as before. Origins []string `json:"origins,omitempty"` } func ManifestKey(prefix string) string { return prefix + "manifest.json" } // objectKey is an object's key relative to the origin that wrote it. func objectKey(seq int64, sha string) string { return fmt.Sprintf("objects/%016d-%s", seq, sha) } // ErrOrigin marks an entry whose object lies outside the origins its // reader admits, or an origin that is not a well-formed prefix. var ErrOrigin = errors.New("hostfiles: object origin not admitted") // checkPrefix accepts a generation prefix: a clean, relative bucket key // ending in "/". func checkPrefix(prefix string) error { dir, ok := strings.CutSuffix(prefix, "/") if !ok { return fmt.Errorf("hostfiles: prefix %q must end in /", prefix) } if _, err := CleanPath(dir); err != nil { return fmt.Errorf("hostfiles: prefix %q is not a clean bucket key", prefix) } return nil } // admitted resolves a caller's origins: nil or empty means the manifest's // own prefix alone. Every origin must be a well-formed prefix. func admitted(prefix string, origins []string) ([]string, error) { if len(origins) == 0 { origins = []string{prefix} } for _, o := range origins { if err := checkPrefix(o); err != nil { return nil, fmt.Errorf("%w: %v", ErrOrigin, err) } } return origins, nil } // CleanPath accepts only a relative, already-clean, slash-separated path // that stays inside its root. It never repairs a path: a manifest is // written by this package, so anything else is damage or an attack. func CleanPath(p string) (string, error) { if p == "" || strings.ContainsRune(p, 0) || strings.HasPrefix(p, "/") { return "", fmt.Errorf("hostfiles: path %q is not a relative path", p) } c := path.Clean(p) if c != p || c == "." || c == ".." || strings.HasPrefix(c, "../") { return "", fmt.Errorf("hostfiles: path %q is not clean or leaves its root", p) } return c, nil } // Load reads the manifest. The bool is false when there is none yet. func Load(ctx context.Context, s release.Store, prefix string) (Manifest, bool, error) { m, _, found, err := load(ctx, s, prefix) return m, found, err } func load(ctx context.Context, s release.Store, prefix string) (Manifest, string, bool, error) { empty := Manifest{V: ManifestVersion, Entries: map[string]Entry{}, Platform: map[string]Entry{}} raw, etag, err := s.Get(ctx, ManifestKey(prefix)) if errors.Is(err, os.ErrNotExist) { return empty, "", false, nil } if err != nil { return Manifest{}, "", false, err } m, err := decode(prefix, raw) if err != nil { return Manifest{}, "", false, err } return m, etag, true, nil } // decode is a manifest from its exact bytes. func decode(prefix string, raw []byte) (Manifest, error) { var m Manifest if err := json.Unmarshal(raw, &m); err != nil { return Manifest{}, fmt.Errorf("hostfiles: manifest %s: %w", prefix, err) } if m.V != ManifestVersion { return Manifest{}, fmt.Errorf("hostfiles: manifest %s is version %d; this binary reads %d", prefix, m.V, ManifestVersion) } if m.Entries == nil { m.Entries = map[string]Entry{} } if m.Platform == nil { m.Platform = map[string]Entry{} } return m, nil }