package hostrelocate import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "os" "sort" "strings" "time" "github.com/carlosframework/platform/internal/genselect" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfiles" "github.com/carlosframework/platform/internal/instancekey" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/release" ) // PinObject is the immutable pinned recovery set (§5.8): the database // plan genselect selected, with every file's exact version, and the files // pin, the newest manifest in the admitted chain, by digest (spec §18: the // store has no version-id read, so content is identified by SHA-256). // ClaimGen is the claim generation the set was selected against; the // object's key names it (hostclaim.RecoveryPinKey), so a pin whose journal // CAS lost is never reused. DB is nil for an evidenced empty bootstrap. type PinObject struct { V int `json:"v"` Host string `json:"host"` Op string `json:"op"` ClaimGen uint64 `json:"claim_gen"` DB *genselect.Plan `json:"db,omitempty"` Empty string `json:"empty,omitempty"` // "release" | "session-0": the evidence Files FilesPin `json:"files"` PinnedAt time.Time `json:"pinned_at"` // Release is the app release version the relocation installs, read // at pinning from StorePinner.Release (R9); a reuse carries it. Release string `json:"release"` // InstanceKey is the key a nonempty database is sealed with when the // pinned manifest evidences it by an adoption record rather than by // instance.key (O2', instance-key plan rev 3): its digest and source. // The install delivers exactly this key or refuses (O3'). InstanceKey *KeyPin `json:"instance_key,omitempty"` } // KeyPin is a pinned instance key: the SHA-256 of the key and the file // that supplies it at launch (instancekey.SourcePlatform or SourceConfig). type KeyPin struct { Source string `json:"source"` SHA256 string `json:"sha256"` } // FilesPin names one generation's files manifest by prefix, sequence and // the digest of its exact bytes, and the origins its entries may name: // every generation prefix in the admitted chain as observed at the pin, // the manifest's own included (contracts note §1). type FilesPin struct { Prefix string `json:"prefix"` Seq int64 `json:"seq"` SHA256 string `json:"sha256"` Origins []string `json:"origins"` // Retained is the pinner's immutable copy of the exact manifest bytes // (RetainedManifestKey), written create-only before the pin object. A // late revision of the source manifest changes nothing the build // reads: P is written from this copy, never from the source again. // Empty for a host that never published files. Retained string `json:"retained,omitempty"` } // RetainedManifestKey is where the pinner keeps the pinned manifest's // bytes: control/relocations///manifest-.json. func RetainedManifestKey(host, op, sha string) string { return "control/relocations/" + host + "/" + op + "/manifest-" + sha + ".json" } // PinKey is hostclaim.RecoveryPinKey. func PinKey(host, op string, gen uint64) string { return hostclaim.RecoveryPinKey(host, op, gen) } // StorePinner selects the pinned recovery set against the claim // observation it is given (§5.8 "The pinned recovery set") and writes it // create-only at the key for that observation's claim generation. It // checks every bound JournalPin will check before it writes anything, so // a doomed pin is never published; a retry at the same generation reuses // its own object unchanged. The caller holds the location lease (claim §4: // "New pins must be established under the required leases"). type StorePinner struct { Claims release.Store // control bucket: pin objects Data release.Store // the region's data bucket: files manifests SelectDB func(context.Context, genselect.Request) (*genselect.Plan, error) Bucket string // the region's data bucket name, for genselect Region string // HomeRegion is the box's verified home binding, or "": genselect sees // the holder's admissions through it (park.HolderRegionView), as main's // serving baseline does, so an admission recorded under the home's // other name is this region's. HomeRegion string // Release is the app release version the host's relocation installs, // read from the region controller's authoritative registry route for // the host (registry.Route.Version, the route's deployed version) at // pinning (R9 on amadan #236). It is never read from the box, the // request or the box's install journal, and nothing else in the pin // supplies it: the installing CAS copies it to install_release, which // moves' held-stage relay requires the request's version to equal. // Stage B wires it to the controller's registry. Nil is ErrUnsupported; // an empty version refuses the pin, never a guess. Release func(ctx context.Context, host string) (string, error) } func (p StorePinner) Pin(ctx context.Context, obs hostclaim.Observed, now time.Time) (hostclaim.RecoveryPin, error) { if p.Claims == nil || p.Data == nil || p.SelectDB == nil || p.Release == nil || p.Bucket == "" || p.Region == "" || obs.Claim == nil || obs.Claim.Holder == nil { return hostclaim.RecoveryPin{}, ErrUnsupported } c := obs.Claim key := PinKey(c.Host, c.Op.ID, c.Gen) if raw, _, err := p.Claims.Get(ctx, key); err == nil { return p.reuse(c, key, raw) } else if !errors.Is(err, os.ErrNotExist) { return hostclaim.RecoveryPin{}, err } var rel hostclaim.Release released := json.Unmarshal([]byte(c.Op.Journal["release"]), &rel) == nil && rel.ReleasedBy != "" for k, at := range c.Op.Journal { if len(k) > len(".observed_at") && k[len(k)-len(".observed_at"):] == ".observed_at" { t, err := time.Parse(time.RFC3339Nano, at) if err != nil { return hostclaim.RecoveryPin{}, err } if now.Before(t.Add(hostclaim.FailoverPinDelay)) { return hostclaim.RecoveryPin{}, ErrNotYet } } } version, err := p.Release(ctx, c.Host) if err != nil { return hostclaim.RecoveryPin{}, err } if version == "" { return hostclaim.RecoveryPin{}, fmt.Errorf("%w: the registry route for %s names no release", ErrRefused, c.Host) } obj := PinObject{V: 1, Host: c.Host, Op: c.Op.ID, ClaimGen: c.Gen, PinnedAt: now.UTC(), Release: version} plan, err := p.SelectDB(ctx, genselect.Request{Bucket: p.Bucket, Region: p.Region, Host: c.Host, Holder: park.HolderRegionView(*c.Holder, p.HomeRegion, p.Region)}) switch { case err == nil: obj.DB = plan case errors.Is(err, genselect.ErrEmpty) && released && rel.FinalTXID == 0: obj.Empty = "release" // the stopped writer flushed and reports position 0 case errors.Is(err, genselect.ErrEmpty) && !released && c.Holder.Session == 0: obj.Empty = "session-0" // no writer was ever admitted, so nothing was written default: // A listing that finds nothing after a writer ran is not evidence // of an empty database (§5.8, claim §6.4a): refuse, never bootstrap. return hostclaim.RecoveryPin{}, err } if released && obj.DB != nil && obj.DB.MaxTXID < rel.FinalTXID { return hostclaim.RecoveryPin{}, fmt.Errorf("%w: the listing is at %d, the release at %d", ErrNotYet, obj.DB.MaxTXID, rel.FinalTXID) } // A nonempty database written by an admitted writer is sealed with // that writer's instance.key: the pinned manifest must carry it (rev // 18, round 17 finding 1). needKey := c.Holder.Session != 0 && obj.DB != nil && obj.DB.MaxTXID > 0 // The writer whose data is pinned: the session admitted as the pinned // database's generation (O2', instance-key plan ruling 20). Asked only // when instance.key is missing. writer := func() (instancekey.Binding, error) { return pinnedWriter(c, obj.DB.Generation) } files, keyPin, retained, err := p.filesPin(ctx, writer, c.Holder, released && rel.ManifestSeq == 0, needKey) if err != nil { return hostclaim.RecoveryPin{}, err } obj.Files, obj.InstanceKey = files, keyPin if released && obj.Files.Seq < rel.ManifestSeq { return hostclaim.RecoveryPin{}, fmt.Errorf("%w: the files manifest is at %d, the release at %d", ErrNotYet, obj.Files.Seq, rel.ManifestSeq) } if retained != nil { // The exact bytes, kept where nothing rewrites them, before the // pin that names them exists. obj.Files.Retained = RetainedManifestKey(c.Host, c.Op.ID, files.SHA256) if err := release.CreateExact(ctx, p.Claims, obj.Files.Retained, retained); err != nil && !errors.Is(err, release.ErrCASConflict) { return hostclaim.RecoveryPin{}, err } } raw, err := json.Marshal(obj) if err != nil { return hostclaim.RecoveryPin{}, err } if err := release.CreateExact(ctx, p.Claims, key, raw); err != nil { if !errors.Is(err, release.ErrCASConflict) { return hostclaim.RecoveryPin{}, err } got, _, gerr := p.Claims.Get(ctx, key) // another worker pinned this generation first if gerr != nil { return hostclaim.RecoveryPin{}, gerr } return p.reuse(c, key, got) } return pinRef(key, raw, obj), nil } func (p StorePinner) reuse(c *hostclaim.Claim, key string, raw []byte) (hostclaim.RecoveryPin, error) { var obj PinObject if err := json.Unmarshal(raw, &obj); err != nil || obj.V != 1 || obj.Host != c.Host || obj.Op != c.Op.ID || obj.ClaimGen != c.Gen || (obj.DB == nil) == (obj.Empty == "") || obj.Release == "" { return hostclaim.RecoveryPin{}, fmt.Errorf("%w: pin object %s is not this operation's at this generation", ErrRefused, key) } return pinRef(key, raw, obj), nil } func pinRef(key string, raw []byte, obj PinObject) hostclaim.RecoveryPin { sum := sha256.Sum256(raw) ref := hostclaim.RecoveryPin{Key: key, SHA256: hex.EncodeToString(sum[:]), FilesManifest: hostfiles.ManifestKey(obj.Files.Prefix), FilesSeq: obj.Files.Seq, PinnedAt: obj.PinnedAt, Empty: obj.DB == nil, Release: obj.Release} if obj.DB != nil { ref.Generation, ref.MaxTXID = obj.DB.Generation, obj.DB.MaxTXID } return ref } // InstanceKeyPath is H6's always-continuous instance.key, at the root of // the host's files directory. const InstanceKeyPath = "instance.key" func liveEntry(m hostfiles.Manifest, path string) bool { e, ok := m.Entries[path] return ok && !e.Deleted && e.Skipped == "" && e.Object != "" } // ErrFilesUnevidenced is a pin refused because its files set cannot be // shown to hold the host's platform files: a writer was admitted and no // files manifest exists in the admitted chain (rev 17), or the database // is nonempty and the pinned manifest has no instance.key (rev 18). The // worker marks the operation stuck. var ErrFilesUnevidenced = errors.New("hostrelocate: the files evidence does not cover the host's platform files") // filesPin is §5.6's files pin: the manifest of holder.generation as // observed or, if that generation has none, of the nearest predecessor in // holder.admissions that has one. It is chosen independently of the // database baseline (B2-4). A read error refuses; no manifest anywhere in // the chain is a host that never published files, pinned at sequence 0. // Origins is the retained chain plus every generation of this incarnation // whose objects the pinned manifest references (rev 12, round 11): the // claim's §6.2a files live set is the objects the live manifests // reference, independent of holder.admissions, which AdmitWriter trims to // eight. An unchanged file published at g1 and carried by every later copy // stays restorable at g10. Every referenced object must lie under this // holder's location (its incarnation) in this region's data bucket, at // files/g-/objects/ (moves.GenerationFilesPrefix, the files // prefix rule the C8 provider and the park view's files live set share); // any other reference refuses the pin. func (p StorePinner) filesPin(ctx context.Context, writer func() (instancekey.Binding, error), h *hostclaim.Holder, releasedEmpty, needKey bool) (FilesPin, *KeyPin, []byte, error) { chain := []string{h.Generation} for i := len(h.Admissions) - 1; i >= 0; i-- { chain = append(chain, h.Admissions[i].Generation, h.Admissions[i].Predecessor) } seen := map[string]bool{} var origins []string for _, gen := range chain { if gen == "" || gen == hostclaim.RootGeneration || seen[gen] { continue } seen[gen] = true origins = append(origins, moves.GenerationFilesPrefix(h.Location, gen)) } for _, prefix := range origins { raw, _, err := p.Data.Get(ctx, hostfiles.ManifestKey(prefix)) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { return FilesPin{}, nil, nil, err } var m hostfiles.Manifest // decoded from the bytes the digest names if err := json.Unmarshal(raw, &m); err != nil || m.V != hostfiles.ManifestVersion { return FilesPin{}, nil, nil, errors.Join(fmt.Errorf("%w: files manifest %s", ErrRefused, prefix), err) } if origins, err = referencedOrigins(h.Location, origins, m); err != nil { return FilesPin{}, nil, nil, err } // A pinned manifest carries the env file and the wake state // (Task 8c, ruling 68 amended; the install's own rule, // relocationPlatformItem): a host that published files has no // evidence that it has neither. Platform items have no tombstone // (StagePlatformDelete drops the item), so a missing one reads the // same as one never published, and a manifest no release flush // wrote (a failover's) carries no wake state however many timers // the host had. Refused here, before any build, rather than at the // target's install. for _, name := range []string{hostfiles.EnvItem, hostfiles.WakeItem} { if e, ok := m.Platform[name]; !ok || e.Deleted { return FilesPin{}, nil, nil, fmt.Errorf("%w: %s's manifest has no %s", ErrFilesUnevidenced, prefix, name) } } var keyPin *KeyPin if needKey && !liveEntry(m, InstanceKeyPath) { // (a) failed: no instance.key. (b'): an adoption record bound // to the pinned writer, for the key the target will deliver // (O2', instance-key plan rev 3). Without it the manifest may // have been copied before the app minted a key of its own // (§5.6's continuous-sync window), and installing this set // would mint a new key over data sealed with the old one. w, err := writer() if err != nil { return FilesPin{}, nil, nil, err } if keyPin, err = p.adoptedKey(ctx, w, prefix, m, origins); err != nil { return FilesPin{}, nil, nil, err } } sum := sha256.Sum256(raw) return FilesPin{Prefix: prefix, Seq: m.Seq, SHA256: hex.EncodeToString(sum[:]), Origins: origins}, keyPin, raw, nil } // No manifest anywhere in the chain. An empty files pin needs the // same kind of evidence an empty database pin does (ruling 15): no // writer was ever admitted on this holder (session 0), so no app ever // ran to mint instance.key, report.token or an env (H6); or the // stopped writer's release flush, which publishes every file whatever // the setting, reported nothing (manifest_seq 0). Any other admitted // writer without a manifest (an adopted host whose files never // published, a first manifest that never landed, a trimmed chain) // refuses: the worker marks the operation stuck for an operator (rev // 17, finding F2), never an install that mints a fresh key over data // sealed with the old one. if h.Session != 0 && !releasedEmpty { return FilesPin{}, nil, nil, fmt.Errorf("%w: %s has admitted a writer but no files manifest exists in its chain", ErrFilesUnevidenced, h.Location) } gen := h.Generation if gen == "" { gen = hostclaim.RootGeneration } prefix := moves.GenerationFilesPrefix(h.Location, gen) return FilesPin{Prefix: prefix, Origins: []string{prefix}}, nil, nil, nil } // referencedOrigins adds to origins the files prefix of every generation // whose objects m references, each of which must be an object this // incarnation's location published: /files/g-/objects/ // (moves.GenerationFilesPrefix). func referencedOrigins(location string, origins []string, m hostfiles.Manifest) ([]string, error) { base := moves.FilesGenerationBase(location) have := make(map[string]bool, len(origins)) for _, o := range origins { have[o] = true } var extra []string check := func(name string, e hostfiles.Entry) error { if e.Object == "" { return nil // a tombstone or a skipped entry names no object } gen, rest, ok := strings.Cut(strings.TrimPrefix(e.Object, base), "/") if !strings.HasPrefix(e.Object, base) || !ok || gen == "" || gen == "." || gen == ".." || !strings.HasPrefix(rest, "objects/") { return fmt.Errorf("%w: %s references %s, outside %s: %w", ErrRefused, name, e.Object, base, hostfiles.ErrOrigin) } if o := moves.GenerationFilesPrefix(location, gen); !have[o] { have[o] = true extra = append(extra, o) } return nil } for name, e := range m.Entries { if err := check(name, e); err != nil { return nil, err } } for name, e := range m.Platform { if err := check(name, e); err != nil { return nil, err } } sort.Strings(extra) return append(origins, extra...), nil } // ReadPin reads and verifies the pin object a journaled RecoveryPin names: // the exact bytes hash to the journaled digest (ruling 5), and the // journaled release is the object's. Every reader of a pin goes through it. func ReadPin(ctx context.Context, claims release.Store, ref hostclaim.RecoveryPin) (PinObject, error) { raw, _, err := claims.Get(ctx, ref.Key) if err != nil { return PinObject{}, err } if sum := sha256.Sum256(raw); hex.EncodeToString(sum[:]) != ref.SHA256 { return PinObject{}, fmt.Errorf("%w: pin object %s does not hash to its journaled digest", ErrRefused, ref.Key) } var obj PinObject if err := json.Unmarshal(raw, &obj); err != nil || obj.V != 1 { return PinObject{}, fmt.Errorf("%w: pin object %s", ErrRefused, ref.Key) } // The journal's copy of the release is what install_release copies // (R9): it must be the object's own. if obj.Release == "" || obj.Release != ref.Release { return PinObject{}, fmt.Errorf("%w: pin object %s names release %q, the journal %q", ErrRefused, ref.Key, obj.Release, ref.Release) } // A pin without a database is an empty bootstrap only by its named // evidence, and the journal's copy must agree: a missing plan is never // read as an empty database (absence is never evidence). if (obj.DB == nil) == (obj.Empty == "") || ref.Empty != (obj.DB == nil) { return PinObject{}, fmt.Errorf("%w: pin object %s does not say whether its database is empty", ErrRefused, ref.Key) } // Likewise a files pin is a host that published no files only when it // names neither a digest nor a retained copy: one without the other is // never read as an empty manifest. if (obj.Files.SHA256 == "") != (obj.Files.Retained == "") { return PinObject{}, fmt.Errorf("%w: pin object %s names a files digest or a retained copy without the other", ErrRefused, ref.Key) } return obj, nil } // CopyPinnedManifest writes the pinned files manifest into toPrefix // (§5.8 "write P's files manifest, create-only, as the pinned manifest's // entries"), from the pinner's retained copy of its exact bytes (claims: // the control bucket), never from the source prefix again: a late // revision of the source manifest cannot move the pin (B2-4). The bytes // must hash to the pin's digest and sit at the pinned sequence; every // entry is validated against the pinned origins (L13); the write is // create-only, and a retry finding the same bytes at toPrefix succeeds. // A host that never published files gets an empty manifest. func CopyPinnedManifest(ctx context.Context, claims, data release.Store, pin FilesPin, toPrefix string) (hostfiles.Manifest, error) { if pin.Prefix == "" || toPrefix == "" || len(pin.Origins) == 0 || (pin.SHA256 == "") != (pin.Retained == "") { return hostfiles.Manifest{}, ErrUnsupported } var pinned hostfiles.Manifest if pin.Retained == "" { // A host that never published files: P starts an empty manifest. pinned = hostfiles.Manifest{V: hostfiles.ManifestVersion, Entries: map[string]hostfiles.Entry{}, Platform: map[string]hostfiles.Entry{}} } else { raw, _, err := claims.Get(ctx, pin.Retained) if err != nil { return hostfiles.Manifest{}, err } if sum := sha256.Sum256(raw); hex.EncodeToString(sum[:]) != pin.SHA256 { return hostfiles.Manifest{}, fmt.Errorf("%w: retained manifest %s is not the pinned bytes", ErrRefused, pin.Retained) } if err := json.Unmarshal(raw, &pinned); err != nil || pinned.Seq != pin.Seq { return hostfiles.Manifest{}, fmt.Errorf("%w: retained manifest %s", ErrRefused, pin.Retained) } if err := hostfiles.ValidateManifest(pinned, pin.Origins); err != nil { return hostfiles.Manifest{}, err } } // P's prepared manifest (claim rev 6.30 R18-1, moves rev 7.22 §7.6): // the pinned entries, by their exact content-addressed objects, at // sequence 0. The source's sequence is provenance (the pin keeps it), // never P's: the session publisher starts at 1 after the first // admission. The binding {location, P, op, install stamp} is the // prefix (location, P) and the claim's prepared_authority (op, stamp, // this manifest's digest). pinned.Seq = 0 body, err := json.Marshal(pinned) if err != nil { return hostfiles.Manifest{}, err } if err := release.CreateExact(ctx, data, hostfiles.ManifestKey(toPrefix), body); err != nil { if errors.Is(err, release.ErrCASConflict) { return hostfiles.Manifest{}, hostfiles.ErrManifestExists } return hostfiles.Manifest{}, err } return pinned, nil } // pinnedWriter is the writer whose data a pin of generation gen holds: the // holder's own session when gen is the holder's generation, else the // admitted session recorded for gen. A generation no admission names has // no writer to bind evidence to, and refuses. func pinnedWriter(c *hostclaim.Claim, gen string) (instancekey.Binding, error) { h := c.Holder w := instancekey.Binding{Host: c.Host, Incarnation: h.Incarnation, Generation: gen} if gen == h.Generation { w.Session = h.Session } for i := len(h.Admissions) - 1; w.Session == 0 && i >= 0; i-- { if h.Admissions[i].Generation == gen { w.Session = h.Admissions[i].Session } } if w.Session == 0 || w.Incarnation == "" || gen == "" { return instancekey.Binding{}, fmt.Errorf("%w: no admitted writer for generation %q", ErrFilesUnevidenced, gen) } return w, nil } // adoptedKey is O2' source (b') (instance-key plan rev 3, ruling 20): the // pinned manifest's adoption record must bind exactly the pinned writer, // and name the key the target will deliver. A platform-sourced record must // match the env item's own key, which the install restores byte for byte. // A config-sourced record needs an env item with no own key (one would // override the configuration at launch); its digest is pinned, and the // install verifies the configuration it renders still delivers it (O3'). // The env item is parsed with instancekey.ParseEnv, the activator's own // rule. A named item that cannot be read is an error, never "no evidence". func (p StorePinner) adoptedKey(ctx context.Context, writer instancekey.Binding, prefix string, m hostfiles.Manifest, origins []string) (*KeyPin, error) { read := func(name string) ([]byte, bool, error) { if e, ok := m.Platform[name]; !ok || e.Deleted { return nil, false, nil } raw, err := hostfiles.ReadPlatform(ctx, p.Data, prefix, m, name, origins) return raw, err == nil, err } rec, ok, err := read(instancekey.AdoptionItem) if err != nil { return nil, err } refuse := func(why string) (*KeyPin, error) { return nil, fmt.Errorf("%w: %s's pinned manifest has no %s and %s, for a nonempty database", ErrFilesUnevidenced, prefix, InstanceKeyPath, why) } if !ok { return refuse("no adoption record") } a, err := instancekey.ParseAdoption(rec) if err != nil { return refuse("its adoption record is malformed") } if !a.Binds(writer) { return refuse("its adoption record is not the pinned writer's") } env, haveEnv, err := read(hostfiles.EnvItem) if err != nil { return nil, err } own, _ := instancekey.Effective(nil, instancekey.ParseEnv(env)) // A configured key needs the env item, present and readable, with no // key of its own: a manifest without host.env is not evidence that // nothing overrides the configuration (absence is never evidence). // Residual: a source with no host.env item and a configured key // refuses, because platform items have no tombstone (StagePlatformDelete // drops the item), so a deleted env reads as one never published. That // is rare only for a source that released normally: its release flush // stages .env, which holds CARLOS_ADMIN_TOKEN on every exec // instance. After a box loss no release flush ran, and the item is // there only if O1' (the C8 provider's Establish) published .env // at the writer's admission. switch { case a.Source == instancekey.SourcePlatform && own != "" && instancekey.Digest(own) == a.SHA256: case a.Source == instancekey.SourceConfig && haveEnv && own == "": default: return refuse("its adoption record does not name the key the env delivers") } return &KeyPin{Source: a.Source, SHA256: a.SHA256}, nil }