package host import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "os" "path/filepath" "slices" "strconv" "github.com/carlosframework/platform/internal/appconfig" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfiles" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/hostrelocate" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/instancefs" "github.com/carlosframework/platform/internal/instancekey" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // The install authority's actions (ruling 46): the target's guard // activation, its build writes and its local install; its held config; // its held release. The claim's Policy admits none of them for relocating // (R1); relocationInstallPolicy does, on main's WorkSource, until claim // ask C9 moves them into the claim. const ( relocationActionInstall = "install" relocationActionHeldConfig = "held-config" relocationActionHeldRelease = "held-release" ) func relocationInstallStamp(c *hostclaim.Claim) (uint64, error) { stamp, err := strconv.ParseUint(c.Op.Journal["install_stamp"], 10, 64) if err != nil || stamp == 0 { return 0, fmt.Errorf("%w: no install stamp", hostrelocate.ErrRefused) } return stamp, nil } // relocationInstallAuthority is the install's claim check (rulings 43, 46, // 52): this box executes this relocating op at one of phases, P unopened, // not stuck, not lost, and the holder's stamp is the journaled install // stamp. func relocationInstallAuthority(c *hostclaim.Claim, op, box string, phases ...string) error { if c == nil || c.Holder == nil || c.Op.Kind != hostclaim.KindRelocating || c.Op.ID != op || c.Op.Exec != box || c.Op.Journal["stuck"] != "" || slices.Contains(c.Lost, box) || !slices.Contains(phases, c.Op.Phase) || hostclaim.PreparedOpened(c) { return fmt.Errorf("%w: the install's authority changed", hostrelocate.ErrRefused) } stamp, err := relocationInstallStamp(c) if err != nil { return err } if c.Holder.Stamp != stamp { return fmt.Errorf("%w: the holder's stamp is not the install stamp", hostrelocate.ErrRefused) } return nil } // relocationInstallLocal is main's Local check (identity, quarantine, the // participation slot's binding, deadline and loss latch) for an install // action on c at captured: the one Selection both the guard activation // and every build write ask with. func relocationInstallLocal(ctx context.Context, src hostbox.WorkSource, c *hostclaim.Claim, captured hostguard.Captured) error { sel := hostbox.Selection{HomeRegion: src.HomeRegion, FleetHomeRegion: src.FleetHomeRegion, Host: c.Host, Action: relocationActionInstall, Box: src.Box, Region: src.Region, Captured: captured, Kind: c.Op.Kind, Op: c.Op.ID, Phase: c.Op.Phase, Generation: c.Holder.Generation, Session: c.Holder.Session} return src.Local(ctx, c, sel) } // relocationActivationReader validates every fresh read the guard makes // while it activates the target's installation (ruling 43), as reconcile's // activationReader does for its own. type relocationActivationReader struct { claims release.Store check func(context.Context, *hostclaim.Claim) error } func (r relocationActivationReader) Read(ctx context.Context, host string) (hostclaim.Observed, error) { obs, err := hostclaim.Read(ctx, r.claims, host) if err != nil { return obs, err } if obs.Absent || obs.Claim == nil { return hostclaim.Observed{}, hostbox.ErrChanged } if err := r.check(ctx, obs.Claim); err != nil { return hostclaim.Observed{}, err } return obs, nil } // relocationActivate is ruling 43: the first act of installing, before the // remote build reads hostguard.Capture. It activates this box's guard // record at install_stamp (Activate takes holder.stamp, which the // installing CAS set to install_stamp), creating the record on an empty // target, under a reader that refuses any read that is not this box's // installing op at that stamp or that main's Local check refuses. // Idempotent on the same stamp; a retired stamp is ErrRetiredStamp, never // reactivated (ruling 52). func (a ClaimAssembly) relocationActivate(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed) (hostguard.Captured, error) { return a.relocationActivateAt(cfg, hostclaim.PhaseInstalling) } // relocationActivateInstalled is the same activation for a target that // resumes at installed without having built P (Astra round 8): a // replacement named at installed, or in probing before P's first // admission, which hostclaim.ReplaceTarget leaves at installed under a new // install stamp, with P and its prepared authority kept and no build to // run. The local install activates its unopened installation at that // stamp before it captures it, under the build's reader and checks with // the phase widened to exactly installed and probing, P not opened // (relocationInstallAuthority refuses an opened P), this box the executor // and the named current target, and the holder's stamp the install stamp. // For the original target, activated by its build at the same stamp, it // is the guard's idempotent no-op. A retired or older stamp is never // reactivated (ErrRetiredStamp, ErrOldStamp). func (a ClaimAssembly) relocationActivateInstalled(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed) (hostguard.Captured, error) { return a.relocationActivateAt(cfg, hostclaim.PhaseInstalled, hostclaim.PhaseRelocatingProbing) } // relocationActivateAt is the one activation: this box's guard record at // install_stamp, every fresh read the guard makes held to this box's op at // one of phases (relocationInstallAuthority: the executor, not stuck, not // lost, P not opened, the holder's stamp the install stamp), this box the // named current target, and main's Local check at that installation. func (a ClaimAssembly) relocationActivateAt(cfg RelocationAssemblyConfig, phases ...string) func(context.Context, hostclaim.Observed) (hostguard.Captured, error) { return func(ctx context.Context, o hostclaim.Observed) (hostguard.Captured, error) { src := cfg.Source if src.Local == nil || src.Box == "" || o.Claim == nil || o.Claim.Holder == nil { return hostguard.Captured{}, ErrClaimStopUnavailable } op := o.Claim.Op.ID authority := func(c *hostclaim.Claim) error { if err := relocationInstallAuthority(c, op, src.Box, phases...); err != nil { return err } if c.Op.Intent["target_box"] != src.Box { return fmt.Errorf("%w: %s is not the named target", hostrelocate.ErrRefused, src.Box) } return nil } guard := a.Guard guard.Claims = relocationActivationReader{claims: a.Claims, check: func(ctx context.Context, c *hostclaim.Claim) error { if err := authority(c); err != nil { return err } return relocationInstallLocal(ctx, src, c, hostguard.Captured{Incarnation: c.Holder.Incarnation, Stamp: c.Holder.Stamp}) }} if a.Reconcile != nil && a.Reconcile.Inventory.Remote() { if err := authority(o.Claim); err != nil { return hostguard.Captured{}, err } rec, err := relocationRecord(ctx, cfg.DataDir, a.Claims, o) if err != nil { return hostguard.Captured{}, err } _, etag, err := a.Claims.Get(ctx, rec.Key) if err != nil { return hostguard.Captured{}, err } selected := hostbox.RecordSelection{Host: o.Host, Key: rec.Key, ETag: etag, Incarnation: o.Claim.Holder.Incarnation, Family: rec.Family, Layout: "nested"} if err = a.Reconcile.PrepareInstallation(ctx, selected, hostguard.Captured{Incarnation: o.Claim.Holder.Incarnation, Stamp: o.Claim.Holder.Stamp}); err != nil { return hostguard.Captured{}, err } } if err := guard.Activate(ctx, o.Host, src.Box, nil); err != nil { return hostguard.Captured{}, err } captured, err := hostguard.Capture(a.Guard.DataDir, o.Host) if err != nil { return hostguard.Captured{}, err } stamp, err := relocationInstallStamp(o.Claim) if err != nil { return hostguard.Captured{}, err } if captured.Stamp != stamp || captured.Incarnation != o.Claim.Holder.Incarnation { return hostguard.Captured{}, hostguard.ErrStampMismatch } return captured, nil } } // relocationBuildAuthority is the remote build's check before every // durable write (ruling 59): the claim still runs this box's installing op // at the install stamp with the same P and pin, the guard still captures // that installation, and main's Local still admits it. Local is the // execution lease's own check, so an execution lease lost while the // replica lease, claim and guard are unchanged stops the next write. func (a ClaimAssembly) relocationBuildAuthority(cfg RelocationAssemblyConfig, o hostclaim.Observed, captured hostguard.Captured) func(context.Context) error { return func(ctx context.Context) error { src := cfg.Source if src.Local == nil || src.Box == "" || o.Claim == nil { return ErrClaimStopUnavailable } fresh, err := hostclaim.Read(ctx, a.Claims, o.Host) if err != nil { return err } c := fresh.Claim if err := relocationInstallAuthority(c, o.Claim.Op.ID, src.Box, hostclaim.PhaseInstalling); err != nil { return err } if c.Op.Journal["prepared"] != o.Claim.Op.Journal["prepared"] || c.Op.Journal["pin"] != o.Claim.Op.Journal["pin"] { return fmt.Errorf("%w: the build's P or pin changed", hostrelocate.ErrRefused) } now, err := hostguard.Capture(a.Guard.DataDir, o.Host) if err != nil { return err } if now != captured || captured.Stamp != c.Holder.Stamp { return fmt.Errorf("%w: this box's installation is not the one the installing CAS issued (%w)", hostrelocate.ErrRefused, hostguard.ErrStampMismatch) } return relocationInstallLocal(ctx, src, c, captured) } } // relocationInstallJournal is the box's durable record of its install for // one operation; its digest is the receipt's journal_sha256. type relocationInstallJournal struct { V int `json:"v"` Host string `json:"host"` Op string `json:"op"` Box string `json:"box"` Enrollment string `json:"enrollment"` InstallStamp uint64 `json:"install_stamp"` Prepared string `json:"prepared"` Steps map[string]string `json:"steps"` // row, files_manifest, files, env, wake, render, key, materialize } // relocationStepFilesManifest is the install journal's binding of P's // prepared files manifest (JSON preparedManifest: key, sha256, version, // op, stamp): what the probe's C8 establishment consumes as its baseline, // read by exactly that version (relocationPreparedManifest), as a move's // op journal binds its own (landStepFilesManifest). Its stamp is P's build // stamp, the one the claim's prepared_authority binds (install_stamp when // ToInstalled ran), not this box's install stamp: a replacement target // keeps P and its authority under a new install stamp (hostclaim // ReplaceTarget at installed, or in probing before P's first admission), // and its own install journal records its own stamp beside this binding. const relocationStepFilesManifest = "files_manifest" // relocationPreparedBinding binds P's prepared files manifest for the // probe's establishment: the current version of P's manifest key as the // versioned listing names it, read by that exact version, which must be a // prepared manifest (sequence 0) with exactly the entries the install // judged (loaded) and the digest the claim's prepared authority (auth) // verified, bound to the op and P's build stamp (auth.InstallStamp). A // missing, deleted or unversioned manifest, one that is not what the // install judged, or an authority that is not P's, refuses. func relocationPreparedBinding(ctx context.Context, data lease.VersionedStore, bucket, prefix, op string, auth hostclaim.PreparedAuthority, loaded hostfiles.Manifest) (preparedManifest, error) { if data == nil || bucket == "" { return preparedManifest{}, ErrClaimStopUnavailable } if auth.InstallStamp == 0 || auth.FilesSeq != 0 || !claimHexDigest(auth.FilesManifestSHA256) { return preparedManifest{}, fmt.Errorf("%w: the prepared authority %+v binds no build of P's files manifest", hostrelocate.ErrRefused, auth) } key := hostfiles.ManifestKey(prefix) vs, err := data.ListVersions(ctx, bucket, key) if err != nil { return preparedManifest{}, err } version := "" for _, v := range vs { if v.Key != key || !v.IsLatest { continue } if v.DeleteMarker || v.VersionID == "" || v.VersionID == "null" { return preparedManifest{}, fmt.Errorf("%w: P's files manifest %s is not a readable version", hostrelocate.ErrRefused, key) } version = v.VersionID } if version == "" { return preparedManifest{}, fmt.Errorf("%w: P's files manifest %s has no current version", hostrelocate.ErrRefused, key) } raw, err := data.GetVersion(ctx, bucket, key, version) if err != nil { return preparedManifest{}, err } var m hostfiles.Manifest if err := json.Unmarshal(raw, &m); err != nil || m.V != hostfiles.ManifestVersion || m.Seq != 0 || loaded.Seq != 0 || !sameEntries(m, loaded) { return preparedManifest{}, errors.Join(fmt.Errorf("%w: P's files manifest %s@%s is not the prepared manifest the install judged", hostrelocate.ErrRefused, key, version), err) } sum := hostfilesSum(raw) if sum != auth.FilesManifestSHA256 { return preparedManifest{}, fmt.Errorf("%w: P's files manifest %s@%s is not the prepared authority's", hostrelocate.ErrRefused, key, version) } return preparedManifest{Key: key, SHA256: sum, Version: version, Op: op, Stamp: auth.InstallStamp}, nil } // relocationOpDir is the operation's local journal directory (ruling 44): // keyed by the op id, never by the host, so no region HA file under a // managed root names a host and the empty-installation census never sees // one (N9 makes relocate/ opaque). func relocationOpDir(dataDir, op string) (string, error) { if dataDir == "" || !validOpID(op) { return "", ErrClaimStopUnavailable } return filepath.Join(dataDir, "relocate", "ops", op), nil } func relocationInstallPath(dataDir, op string) (string, error) { dir, err := relocationOpDir(dataDir, op) if err != nil { return "", err } return filepath.Join(dir, "install.json"), nil } // relocationRecord reads the host's authoritative record and the route // this box mints for it (ruling 45): always the nested layout, which the // isolated launch requires, so only a hibernating, path-deriving instance // record relocates. The route carries the app's update policy as the // ordinary mint's does (settings spec §3.2, provision.Reconcile): an // absent manifest is no policy, and a manifest that cannot be read // refuses, so a relocated instance never lands as auto when its app's // default is manual. func relocationRecord(ctx context.Context, dataDir string, claims release.Store, o hostclaim.Observed) (provision.ClaimRecord, error) { if o.Claim == nil || o.Claim.Holder == nil { return provision.ClaimRecord{}, hostbox.ErrChanged } rec, err := hostops.ReadRecord(ctx, claims, o.Claim.Holder.Key) if err != nil { return provision.ClaimRecord{}, err } var record provision.Record if err := json.Unmarshal(rec.Body, &record); err != nil { return provision.ClaimRecord{}, err } if record.Host != o.Host || !record.DerivesPaths() || !record.HibernateOrDefault() || record.Unit != "" { return provision.ClaimRecord{}, fmt.Errorf("%w: %s is not a relocatable hibernating instance record", hostrelocate.ErrRefused, o.Host) } policy, err := relocationUpdatePolicy(ctx, claims, record) if err != nil { return provision.ClaimRecord{}, err } rt := provision.RouteFor(record, dataDir, "", true, policy) if rt.DBPath != provision.NestedDBPath(dataDir, o.Host) || rt.Socket != provision.NestedSocket(o.Host) { return provision.ClaimRecord{}, fmt.Errorf("%w: %s does not mint nested", hostrelocate.ErrRefused, o.Host) } return provision.ClaimRecord{Host: o.Host, Key: provision.RecordKey(record.Account, record.App, record.Host), Family: "instance", Account: record.Account, App: record.App, CreatedAt: record.CreatedAt, Value: record, Route: rt}, nil } // relocationUpdatePolicy is the app's update policy for record, read the // way provision.Reconcile's policyFor reads it: an absent manifest is // known to have none; any other read failure is an error, never "none". func relocationUpdatePolicy(ctx context.Context, store release.Store, record provision.Record) (*provision.UpdatePolicy, error) { m, err := provision.GetManifest(ctx, store, record.Account, record.App) switch { case errors.Is(err, os.ErrNotExist): return nil, nil case err != nil: return nil, fmt.Errorf("the update policy of %s/%s could not be read: %w", record.Account, record.App, err) } return m.Updates, nil } // relocationPin reads the pin the claim's op journals, through ReadPin. func relocationPin(ctx context.Context, claims release.Store, c *hostclaim.Claim) (hostrelocate.PinObject, error) { var ref hostclaim.RecoveryPin if err := json.Unmarshal([]byte(c.Op.Journal["pin"]), &ref); err != nil { return hostrelocate.PinObject{}, fmt.Errorf("%w: the op journals no readable pin: %w", hostrelocate.ErrRefused, err) } return hostrelocate.ReadPin(ctx, claims, ref) } // relocationPlatformItem reports whether P's manifest m carries the // platform item name (the env file, the wake state) for the install to // restore, or the host has none (ruling 68, amended in Task 8c: absence is // never evidence). A live item is the host's. A manifest without it is // "none" only when the pin evidences a host that never published files: // a files pin with neither a digest nor a retained copy, which the pinner // admits only for a holder no writer was ever admitted to or a release // flush that published nothing, and which ReadPin refuses half-made (H2). // No app ran there, so there is no env file and no timer. Any other // manifest without the item refuses: platform items have no tombstone // (StagePlatformDelete drops the item), so a source whose env file was // deleted reads the same as one whose env file was never published, and a // manifest no release flush wrote (a failover's) carries no wake state // however many timers the host had. A tombstone, which no code writes, // refuses too. func relocationPlatformItem(pin hostrelocate.PinObject, m hostfiles.Manifest, name string) (bool, error) { e, ok := m.Platform[name] switch { case ok && !e.Deleted: return true, nil case ok: return false, fmt.Errorf("%w: P's manifest holds a tombstone for %s, which no publisher writes", hostrelocate.ErrRefused, name) case pin.Files.SHA256 == "" && pin.Files.Retained == "": return false, nil default: return false, fmt.Errorf("%w: P's manifest has no %s, and the pin does not evidence a host that never published files", hostrelocate.ErrRefused, name) } } // relocationPolicy is region HA's relocating owner: it runs after the // claim's own gate and only refuses. The captured installation must be the // holder's, this box a participant and not lost. func relocationPolicy(box string) func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return func(_ context.Context, c *hostclaim.Claim, s hostbox.Selection, _ hostwrite.Identity) error { if c == nil || c.Holder == nil || s.Box != box || !slices.Contains(c.Op.Participants, box) || slices.Contains(c.Lost, box) { return hostbox.ErrChanged } if s.Captured.Incarnation != c.Holder.Incarnation || s.Captured.Stamp != c.Holder.Stamp { return hostbox.ErrChanged } return nil } } // relocationInstallPolicy is the typed install authority's policy (ruling // 46): only the three install actions, only under // relocationInstallAuthority at installed or probing, and a registry hold // absent or this op's; then region HA's relocating owner, which checks the // captured installation and can only refuse. func relocationInstallPolicy(held func(context.Context, string) (string, error), owner func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error) func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return func(ctx context.Context, c *hostclaim.Claim, s hostbox.Selection, id hostwrite.Identity) error { switch s.Action { case relocationActionInstall, relocationActionHeldConfig, relocationActionHeldRelease: default: return hostbox.ErrChanged } if held == nil || owner == nil { return hostbox.ErrUnsupported } if err := relocationInstallAuthority(c, s.Op, s.Box, hostclaim.PhaseInstalled, hostclaim.PhaseRelocatingProbing); err != nil { return errors.Join(hostbox.ErrChanged, err) } hold, err := held(ctx, s.Host) if err != nil { return err } if hold != "" && hold != c.Op.ID { return hostbox.ErrChanged } return owner(ctx, c, s, id) } } // relocationInstallSource is main's WorkSource with only its Policy // replaced (ruling 46): Guard, Box, Region, Now, Prepare (main's // participation) and Local (identity and quarantine, the lease's deadline // and loss latch, failed cleanup) are the assembly's own, and every With // re-checks them. Continue is nil: a phase change ends the selection. func relocationInstallSource(src hostbox.WorkSource, reg *registry.Registry) hostbox.WorkSource { src.Policy = relocationInstallPolicy(hostbox.RegistryHoldPolicy(reg), relocationPolicy(src.Box)) src.Continue = nil return src } // relocationInstall is §5.8's local install (B1-2) in ruling 44's order, // every write through the typed install authority (ruling 46): // // 1. the captured installation must be the active install stamp // (activated at installing by the build, or here for a replacement // that resumes at installed, relocationActivateInstalled; a retired // stamp is never reactivated, ruling 52); // 2. the claim's typed inventory says nested (park.Local, once); // 3. the build's scratch is removed (ruling 44); // 4. in the install work: MintHeld first (the census initialises the // installation from an empty box; the route entry; the row held by // the op with status relocating); then the instance directory through // main's PrepareState (ruling 57); then the files and env from P's // manifest into the nested paths, and the wake state; // 5. the held config (held-config selection), and the pinned key checked // against the env the target will launch with (O3'); // 6. the pinned release (held-release selection; a blocked stage leaves // root's relay request, M3); // 7. the journal, then the receipt. // // Idempotent: a pass that ends in ErrNotYet is repeated whole. func (a ClaimAssembly) relocationInstall(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed) (hostclaim.LocalInstall, error) { activate := a.relocationActivateInstalled(cfg) render, materialize, verifyLive := cfg.render, cfg.materialize, cfg.verifyLive if verifyLive == nil { verifyLive = release.VerifyHeldLiveUnder } if render == nil { render = appconfig.RenderHeldUnder } if materialize == nil { materialize = func(ctx context.Context, s release.Store, optDir string, rt registry.Route, version string, proof *release.ProofGate, work hostwrite.Work, guard func(func() error) error) (*release.Manifest, error) { if hostwrite.Enabled(os.Getenv) { return nil, errHeldReleaseUnavailable } return release.MaterializeHeldUnder(ctx, s, cfg.ReleasePub, cfg.Ratchet, optDir, rt, version, proof, work, guard) } } return func(ctx context.Context, o hostclaim.Observed) (hostclaim.LocalInstall, error) { // Every precondition before anything is written (a misconfigured // assembly leaves no row, file or claim write). if cfg.Data == nil || cfg.Registry == nil || cfg.OptDir == "" || cfg.DataDir == "" || a.Reconcile == nil || a.Wake == nil || a.Wake.Roles == nil || cfg.Source.Box == "" || cfg.Source.Local == nil || o.Claim == nil || o.Claim.Holder == nil { return hostclaim.LocalInstall{}, ErrClaimStopUnavailable } c := o.Claim local, err := a.localStopIdentity(ctx) if err != nil { return hostclaim.LocalInstall{}, err } box := local.Actor.Box check := func(fresh *hostclaim.Claim) error { return relocationInstallAuthority(fresh, c.Op.ID, box, hostclaim.PhaseInstalled, hostclaim.PhaseRelocatingProbing) } if err := check(c); err != nil { return hostclaim.LocalInstall{}, err } stamp, _ := relocationInstallStamp(c) pin, err := relocationPin(ctx, a.Claims, c) if err != nil { return hostclaim.LocalInstall{}, err } p := c.Op.Journal["prepared"] prefix := moves.GenerationFilesPrefix(c.Holder.Location, p) origins := append(append([]string(nil), pin.Files.Origins...), prefix) // What P's manifest says about the env file and the wake state is // judged before anything is written: a manifest missing either, // under a pin that does not evidence its absence, refuses here with // no row, file or claim write (absence is never evidence). pm, found, err := hostfiles.Load(ctx, cfg.Data, prefix) if err != nil { return hostclaim.LocalInstall{}, err } if !found { return hostclaim.LocalInstall{}, fmt.Errorf("%w: P's files manifest %s is missing", hostrelocate.ErrRefused, prefix) } for _, item := range []string{hostfiles.EnvItem, hostfiles.WakeItem} { if _, err := relocationPlatformItem(pin, pm, item); err != nil { return hostclaim.LocalInstall{}, err } } // The binding the probe's C8 establishment reads P's prepared // manifest by: its exact version, journalled below. auth, err := preparedAuthority(c) if err != nil { return hostclaim.LocalInstall{}, err } binding, err := relocationPreparedBinding(ctx, cfg.Versions, cfg.Bucket, prefix, c.Op.ID, auth, pm) if err != nil { return hostclaim.LocalInstall{}, err } bindingRaw, err := json.Marshal(binding) if err != nil { return hostclaim.LocalInstall{}, err } rec, err := relocationRecord(ctx, cfg.DataDir, a.Claims, o) if err != nil { return hostclaim.LocalInstall{}, err } rt := rec.Route paths := relocationPathsFor(cfg.DataDir, rt) // The installation at the install stamp, activated here when this // box resumes at installed without a build of its own (a // replacement; the original target's build activated it already): // a stamp the guard retired, or one older than its history, is // never reactivated, and refuses. captured, err := activate(ctx, o) if errors.Is(err, hostguard.ErrRetiredStamp) || errors.Is(err, hostguard.ErrOldStamp) || errors.Is(err, hostguard.ErrStampMismatch) || errors.Is(err, hostguard.ErrNotParticipant) { err = errors.Join(hostrelocate.ErrRefused, err) } if err != nil { return hostclaim.LocalInstall{}, err } if captured.Stamp != stamp || captured.Incarnation != c.Holder.Incarnation { return hostclaim.LocalInstall{}, fmt.Errorf("%w: the install stamp %d is not active here (%w)", hostrelocate.ErrRefused, stamp, hostguard.ErrStampMismatch) } src := relocationInstallSource(cfg.Source, cfg.Registry) work, err := src.SelectAt(ctx, o.Host, relocationActionInstall, captured) if err != nil { return hostclaim.LocalInstall{}, err } // The claim's typed inventory for this box, once: RecordInventory // writes the claim on every call, so a repeated install (a retry, a // repairing EnsureLocal) does not rewrite it once it says nested // (Astra round 2 finding 2; the participant re-reads after). if entry, ok := c.Inventory[box]; !ok || entry.Layout != "nested" { if err := a.Reconcile.Reconciler.Local.RecordInventory(ctx, o.Host, captured, "nested", nil, check); err != nil { return hostclaim.LocalInstall{}, err } } // The build's scratch names the host: gone before the census runs // (ruling 44), even after a crash between the build and here. if err := removeRelocationScratch(cfg, c.Op.ID); err != nil { return hostclaim.LocalInstall{}, err } journal := relocationInstallJournal{V: 1, Host: o.Host, Op: c.Op.ID, Box: box, Enrollment: local.Actor.Enrollment, InstallStamp: stamp, Prepared: p, Steps: map[string]string{relocationStepFilesManifest: string(bindingRaw)}} if err := a.Reconcile.MintHeldUnder(ctx, rec, captured, "relocating", c.Op.ID, work); err != nil { return hostclaim.LocalInstall{}, err } journal.Steps["row"] = c.Op.ID // The instance directory through main's typed protocol (ruling 57): // ClaimRoleAssembly.PrepareState records the path entry intended, // makes the directory, then records its device and inode // materialized, under the install work; the probe's own // PrepareState then finds it as its own. Never MkdirAll. installed, err := hostclaim.Read(ctx, a.Claims, o.Host) if err != nil { return hostclaim.LocalInstall{}, err } if err := check(installed.Claim); err != nil { return hostclaim.LocalInstall{}, err } if err := a.Wake.Roles.PrepareState(ctx, installed, work, ClaimLaunchInputs{DBPath: rt.DBPath}); err != nil { return hostclaim.LocalInstall{}, err } var m hostfiles.Manifest if err := work.With(ctx, hostwrite.Identity{}, func() error { // Every write below lands in the instance directory, which the // tenant can write (an install repeated after the app ran there // meets what it left). So nothing here resolves a path the // tenant could have linked: the files directory is made with // mkdirat beneath the instance directory (EnsureDirBeneath, which // refuses a link or a FIFO at the name), opened once with no // symlink at any component (OpenRootBeneath) and restored from // that root (RestoreRoot); the env file is written and removed // beneath the directory (WriteAtomic, RemoveBeneath). instDir, filesName := filepath.Dir(paths.FilesDir), filepath.Base(paths.FilesDir) envDir, envName := filepath.Dir(paths.EnvPath), filepath.Base(paths.EnvPath) if _, err := instancefs.EnsureDirBeneath(instDir, filesName, 0o700); err != nil { return err } tenantRootSeam("checked") root, err := instancefs.OpenRootBeneath(instDir, filesName) if err != nil { return err } defer root.Close() tenantRootSeam("opened") if m, _, err = hostfiles.RestoreRoot(ctx, cfg.Data, prefix, root, origins); err != nil { // The root is named by its /proc/self/fd link, closed by // the time anyone reads this: name the files directory. return fmt.Errorf("hostfiles: restore into %s: %w", paths.FilesDir, err) } journal.Steps["files"] = strconv.FormatInt(m.Seq, 10) // O3: the env file is P's EnvItem byte for byte; nothing is // minted. No env file only by the pin's evidence (ruling 68 as // amended, relocationPlatformItem): then any env file here is // not the host's, so it goes. A manifest without the item and // without that evidence refuses. wantEnv, err := relocationPlatformItem(pin, m, hostfiles.EnvItem) if err != nil { return err } if wantEnv { env, err := hostfiles.ReadPlatform(ctx, cfg.Data, prefix, m, hostfiles.EnvItem, origins) if err != nil { return err } // WriteAtomic fsyncs the file before its rename; the // directory sync makes the rename durable. if err := instancefs.WriteAtomic(envDir, envName, env, 0o600); err != nil { return err } if err := syncTenantDir(envDir); err != nil { return err } journal.Steps["env"] = "written" } else { if err := instancefs.RemoveBeneath(envDir, envName); err != nil && !errors.Is(err, os.ErrNotExist) { return err } if err := syncTenantDir(envDir); err != nil { return err } journal.Steps["env"] = "none" } // The wake state likewise: no timers only by the pin's evidence. // A manifest no release flush wrote (a failover's) has none, // and that is not a host without timers. wantWake, err := relocationPlatformItem(pin, m, hostfiles.WakeItem) if err != nil { return err } journal.Steps["wake"] = "none" if wantWake { ws, _, err := hostfiles.LoadWake(ctx, cfg.Data, prefix, m, origins) if err != nil { return err } if err := hostfiles.ImportWake(cfg.Registry, o.Host, ws); err != nil { return err } journal.Steps["wake"] = "imported" } return nil }); err != nil { return hostclaim.LocalInstall{}, err } held := rt held.Status, held.HoldOp = "relocating", c.Op.ID rendered, err := render(ctx, a.Claims, cfg.SecretsKey, cfg.DataDir, held, c.Op.ID, box, func(pre appconfig.HeldRender) (hostwrite.Work, error) { w, err := src.SelectAt(ctx, o.Host, relocationActionHeldConfig, captured) if err != nil { return nil, err } return heldIdentityWork{Work: w, ID: HeldConfigIdentity(pre.BundleETags, pre.OverlayETag)}, nil }) if err != nil { return hostclaim.LocalInstall{}, err } journal.Steps["render"] = rendered.SHA256 // O3' (instance-key plan rev 3; ruling 55): when the pin carries the // key by an adoption record, the env files this install will launch // with must deliver exactly that key; a configuration changed since // the pin refuses here, before anything launches. The adoption item // itself is never restored: the target's sessions are new writers. if pin.InstanceKey != nil { // Both files are in the instance directory, which the tenant can // write (an install repeated after the app has run there reads // what it left), so both are read through main's one safe read // for that ground (readInstanceFile), never os.ReadFile. The render // always writes its file: a missing one is an error, as any // other failure to read it is. configEnv, absent, err := readInstanceFile(filepath.Dir(rendered.Path), filepath.Base(rendered.Path), instancekey.EnvReadLimit) if err == nil && absent { err = fmt.Errorf("%s: %w", rendered.Path, os.ErrNotExist) } if err != nil { return hostclaim.LocalInstall{}, err } // The env file the install wrote must still be there: a missing // one is not an env with no key of its own. Only an install // that evidenced no env file reads its absence as none, and // only when the directory itself opened (positive evidence). ownEnv, absent, err := readInstanceFile(filepath.Dir(paths.EnvPath), filepath.Base(paths.EnvPath), instancekey.EnvReadLimit) if err == nil && absent && journal.Steps["env"] != "none" { err = fmt.Errorf("%s: %w", paths.EnvPath, os.ErrNotExist) } if err != nil { return hostclaim.LocalInstall{}, errors.Join(hostrelocate.ErrRefused, err) } if err := instancekey.VerifyDelivered(configEnv, ownEnv, pin.InstanceKey.SHA256); err != nil { return hostclaim.LocalInstall{}, errors.Join(hostrelocate.ErrRefused, err) } journal.Steps["key"] = pin.InstanceKey.SHA256 } // The pinned release (R9): install_release, never the route's // version, through the claim-aware held stage (never MaterializeHeld, // which refuses with CARLOS_HOST_CLAIM on). The edge unit cannot // write the version store: ErrStageBlocked leaves root's relay // request (M3) and the receipt waits. // // As moves' T7 (heldReleaseReady): the publication holds the host's // held-stage lock (withHeldStageLock) across its swap, re-hash and // rollback, as root's relay does, and the receipt waits for the // reading under that same lock (release.VerifyHeldLiveUnder: the // link names the version and D3 accepts the bytes it names). A link // alone is never evidence: a publication may be about to roll it // back, and the bytes may have changed since. version := c.Op.Journal["install_release"] if version == "" { return hostclaim.LocalInstall{}, fmt.Errorf("%w: no install_release journaled", hostrelocate.ErrRefused) } released, err := src.SelectAt(ctx, o.Host, relocationActionHeldRelease, captured) if err != nil { return hostclaim.LocalInstall{}, err } proof := release.NewRootProofGate(release.ProofPolicyPath, cfg.DataDir) lock := func(fn func() error) error { return withHeldStageLock(cfg.DataDir, o.Host, fn) } err = verifyLive(ctx, a.Claims, cfg.OptDir, held, version, proof, lock) if errors.Is(err, release.ErrHeldNotLive) { _, err = materialize(ctx, a.Claims, cfg.OptDir, held, version, proof, heldIdentityWork{Work: released, ID: hostwrite.Identity{Release: version}}, lock) switch { case errors.Is(err, release.ErrStageBlocked): req := release.HeldStageRequest{Account: rec.Account, App: rec.App, Host: o.Host, Version: version, Op: c.Op.ID, Box: box} if err := writeHeldStageRequest(cfg.DataDir, req); err != nil { return hostclaim.LocalInstall{}, err } return hostclaim.LocalInstall{}, fmt.Errorf("%w: the pinned release %s waits for root's held stage", hostrelocate.ErrNotYet, version) case err != nil: return hostclaim.LocalInstall{}, err } // Published by this call: the receipt waits for the same reading. err = verifyLive(ctx, a.Claims, cfg.OptDir, held, version, proof, lock) } if errors.Is(err, release.ErrHeldNotLive) { return hostclaim.LocalInstall{}, errors.Join(fmt.Errorf("%w: the pinned release %s is not live", hostrelocate.ErrNotYet, version), err) } if err != nil { return hostclaim.LocalInstall{}, err } journal.Steps["materialize"] = version raw, err := json.Marshal(journal) if err != nil { return hostclaim.LocalInstall{}, err } path, err := relocationInstallPath(cfg.DataDir, c.Op.ID) if err != nil { return hostclaim.LocalInstall{}, err } if err := work.With(ctx, hostwrite.Identity{}, func() error { return writeRelocationJournal(path, raw) }); err != nil { return hostclaim.LocalInstall{}, err } sum := sha256.Sum256(raw) return hostclaim.LocalInstall{Box: box, Enrollment: local.Actor.Enrollment, InstallStamp: stamp, FilesManifestVersion: m.Seq, JournalSHA256: hex.EncodeToString(sum[:])}, nil } } // relocationVerifyLocal reports whether the receipt's install is whole on // this box now: the journal at the receipt's digest, the guard record // active at its stamp, the row held by the operation, and every manifest // entry and the env file as P's manifest says (hostfiles.Present), no env // file only by the pin's evidence (relocationPlatformItem). func (a ClaimAssembly) relocationVerifyLocal(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed, hostclaim.LocalInstall) (bool, error) { receipt := a.relocationReceipt(cfg) return func(ctx context.Context, o hostclaim.Observed, l hostclaim.LocalInstall) (bool, error) { if cfg.Registry == nil || cfg.Data == nil || o.Claim == nil || o.Claim.Holder == nil { return false, ErrClaimStopUnavailable } got, ok, err := receipt(ctx, o) if err != nil || !ok || got != l { return false, err } captured, err := hostguard.Capture(a.Guard.DataDir, o.Host) if errors.Is(err, hostguard.ErrNoRecord) { return false, nil } if err != nil { return false, err } if captured.Stamp != l.InstallStamp { return false, nil } row, err := cfg.Registry.Get(o.Host) if err != nil { return false, err } if row == nil || row.HoldOp != o.Claim.Op.ID || !registry.IsHold(row.Status) { return false, nil } p := o.Claim.Op.Journal["prepared"] prefix := moves.GenerationFilesPrefix(o.Claim.Holder.Location, p) m, found, err := hostfiles.Load(ctx, cfg.Data, prefix) if err != nil { return false, err } if !found { // Not "the install is incomplete": the install itself refuses // without it, so a repair could not mend this (absence is // never evidence). return false, fmt.Errorf("%w: P's files manifest %s is missing", hostrelocate.ErrRefused, prefix) } // No env file only by the pin's evidence, as the install judged it. pin, err := relocationPin(ctx, a.Claims, o.Claim) if err != nil { return false, err } wantEnv, err := relocationPlatformItem(pin, m, hostfiles.EnvItem) if err != nil { return false, err } paths := relocationPathsFor(cfg.DataDir, *row) return hostfiles.Present(paths.FilesDir, paths.EnvPath, m, !wantEnv) } } // relocationReceipt reads this box's durable install journal for the // claim's operation; false, with no error, only when there is none. A // journal that does not decode, names another host or operation, or // records no materialized release is an error, never "no receipt" // (final review minor 1, as readRelocationProbeFence): it is never // reinstalled over, so an operator keeps the evidence. func (a ClaimAssembly) relocationReceipt(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed) (hostclaim.LocalInstall, bool, error) { return func(_ context.Context, o hostclaim.Observed) (hostclaim.LocalInstall, bool, error) { if o.Claim == nil { return hostclaim.LocalInstall{}, false, ErrClaimStopUnavailable } path, err := relocationInstallPath(cfg.DataDir, o.Claim.Op.ID) if err != nil { return hostclaim.LocalInstall{}, false, err } raw, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { return hostclaim.LocalInstall{}, false, nil } if err != nil { return hostclaim.LocalInstall{}, false, err } var j relocationInstallJournal if err := json.Unmarshal(raw, &j); err != nil { return hostclaim.LocalInstall{}, false, fmt.Errorf("%w: %s cannot be read: %v", hostrelocate.ErrRefused, path, err) } if j.V != 1 || j.Host != o.Host || j.Op != o.Claim.Op.ID || j.Steps["materialize"] == "" { return hostclaim.LocalInstall{}, false, fmt.Errorf("%w: %s is not this operation's completed install journal", hostrelocate.ErrRefused, path) } seq, _ := strconv.ParseInt(j.Steps["files"], 10, 64) sum := sha256.Sum256(raw) return hostclaim.LocalInstall{Box: j.Box, Enrollment: j.Enrollment, InstallStamp: j.InstallStamp, FilesManifestVersion: seq, JournalSHA256: hex.EncodeToString(sum[:])}, true, nil } }