package host import ( "context" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "log" "os" "path/filepath" "slices" "sync" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostrelocate" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/release" ) // RelocationSource is region HA's ClaimProductionRelocationSource (claim // Plan 11 R1, R8), registered once at agent startup (Task 10) and opened // by the production provider only with CARLOS_REGION_HA on. Open returns // the complete owner set and two refusals (rulings 37, 39), or, while // moves' C8 provider is not bound (dependency C8, ruling 53), the zero // owner set: relocating stays refused exactly as before Plan 11. type RelocationSource struct { DataDir string Directory hostactors.Directory // FilesStore opens the data bucket as a release.Store (ruling 50); // production is release.NewS3Store. FilesStore func(ctx context.Context, bucket string) (release.Store, error) mu sync.Mutex serving ClaimServingSources } var _ ClaimProductionRelocationSource = (*RelocationSource)(nil) // SetServingSources hands the source the serving launch, markers and the // C8 provider before Open (claimproduction.go, inside the region HA gate). func (s *RelocationSource) SetServingSources(src ClaimServingSources) { s.mu.Lock() defer s.mu.Unlock() s.serving = src } func (s *RelocationSource) Open(ctx context.Context, store release.Store, data lease.VersionedStore, actor hostbox.Actor) (ClaimRelocationOwners, []hostbox.PolicyRefusal, error) { if s == nil { return ClaimRelocationOwners{}, nil, hostwrite.ErrUnavailable } s.mu.Lock() src := s.serving s.mu.Unlock() if store == nil || data == nil || actor.Box == "" || actor.Bucket == "" || actor.Region == "" || s.DataDir == "" || s.FilesStore == nil { return ClaimRelocationOwners{}, nil, hostwrite.ErrUnavailable } if src.Files == nil { // C8 unbound (ruling 53): no owners, no refusals. The dispatcher // keeps relocating refused and Task 10's pass stays off. log.Printf("region HA: relocation owners not opened: the moves C8 files-manifest provider is not bound") return ClaimRelocationOwners{}, nil, nil } if src.Launch == nil || src.Markers == nil || src.Identity == nil { return ClaimRelocationOwners{}, nil, hostwrite.ErrUnavailable } files, err := s.FilesStore(ctx, actor.Bucket) if err != nil { return ClaimRelocationOwners{}, nil, err } releases := StopReceiptReleases{Receipts: RelocationReleaseReceipts{Store: store, Directory: s.Directory}} inputs := &RelocationWakeInputs{Launch: src.Launch, FilesManifest: src.Files, Markers: src.Markers, Claims: store, FilesStore: files, Data: hostbox.RequestStore{Store: data, Check: src.Identity}, Bucket: actor.Bucket, Region: park.PlacementRegion(actor.Region), HomeRegion: actor.HomeRegion, Box: actor.Box, NewID: newClaimID, Now: time.Now, Releases: releases} attempts := &relocationProbeAttempts{Claims: store, DataDir: s.DataDir, Box: actor.Box, NewID: newClaimID, Now: time.Now} owners := ClaimRelocationOwners{ BeforeAdmission: relocationBeforeAdmission(actor.Box, releases), AfterAdmission: relocationAfterAdmission(s.DataDir, actor.Box), ClaimKindOwners: ClaimKindOwners{Inputs: inputs, AdmissionEvidence: inputs.AdmissionEvidence, PublicationEvidence: inputs.PublicationEvidence}, Operations: attempts, Policy: relocationPolicy(actor.Box), } return owners, []hostbox.PolicyRefusal{refuseStaleStamp, refuseLostBox}, nil } func (s *RelocationSource) Close(context.Context) error { return nil } // newClaimID is the 16-byte hex id claimconstruction.go's newID makes. func newClaimID() string { var b [16]byte if _, err := rand.Read(b[:]); err != nil { return "" } return hex.EncodeToString(b[:]) } func refuseStaleStamp(_ context.Context, c *hostclaim.Claim, s hostbox.Selection, _ hostwrite.Identity) error { if c != nil && c.Op.Kind == hostclaim.KindRelocating && (c.Holder == nil || s.Captured.Incarnation != c.Holder.Incarnation || s.Captured.Stamp != c.Holder.Stamp) { return hostbox.ErrChanged } return nil } func refuseLostBox(_ context.Context, c *hostclaim.Claim, s hostbox.Selection, _ hostwrite.Identity) error { if c != nil && slices.Contains(c.Lost, s.Box) { return hostbox.ErrChanged } return nil } // relocationBeforeAdmission verifies the source's release (a drain's // journaled release at its receipt) or its fence (a failover) before the // target's first admission; a later attempt's predecessor is this box's // own session, which main's Stop ended. func relocationBeforeAdmission(box string, releases hostclaim.ReleaseAuthority) func(context.Context, hostclaim.Observed) error { return func(ctx context.Context, prior hostclaim.Observed) error { c := prior.Claim if c == nil || c.Holder == nil || c.Op.Kind != hostclaim.KindRelocating || c.Op.Exec != box { return hostbox.ErrChanged } if hostclaim.PreparedOpened(c) { return nil } certified, err := hostrelocate.CertifiedFor(c) if err != nil { return err // present and unreadable, or mismatched, is never a failover } if certified { var rel hostclaim.Release if err := json.Unmarshal([]byte(c.Op.Journal["release"]), &rel); err != nil { return err } return releases.VerifyRelease(ctx, *c, rel) } src := c.Op.Intent["source_box"] if slices.Contains(c.Lost, src) && c.Op.Journal["fence."+src] != "" { return nil } return fmt.Errorf("%w: no release and no fence for %s", hostrelocate.ErrRefused, src) } } // relocationAfterAdmission verifies the target's installation after the // admission: the local install receipt is this box's, as journaled, and // P's opening keeps install_stamp. func relocationAfterAdmission(dataDir, box string) func(context.Context, hostclaim.Observed, hostclaim.Observed) error { return func(_ context.Context, prior, admitted hostclaim.Observed) error { c, n := prior.Claim, admitted.Claim if c == nil || n == nil || n.Holder == nil || n.Op.ID != c.Op.ID || n.Op.Exec != box || n.Holder.WriterBox != box { return hostbox.ErrChanged } var l hostclaim.LocalInstall if json.Unmarshal([]byte(n.Op.Journal["local_install"]), &l) != nil || l.Box != box || l.InstallStamp == 0 { return hostbox.ErrChanged } path, err := relocationInstallPath(dataDir, n.Op.ID) if err != nil { return err } raw, err := os.ReadFile(path) if err != nil { return err } if sum := sha256.Sum256(raw); hex.EncodeToString(sum[:]) != l.JournalSHA256 { return hostbox.ErrChanged } if n.Holder.Generation == n.Op.Journal["prepared"] && n.Holder.Stamp != l.InstallStamp { return hostbox.ErrChanged } return nil } } // relocationProbe is what one probe attempt over the claim needs and what // Preflight reads before the attempt is counted (ruling 48; review M1): // the journaled release's verdict, the prepared authority, the pin's // agreement with it, and when probing began. Each is a deterministic // reading of the claim, so a defect never heals by itself: it is found // before an attempt is spent on it, and found again at the attempt. type relocationProbe struct { certified bool auth hostclaim.PreparedAuthority empty bool since time.Time } func relocationProbeClaim(c *hostclaim.Claim) (relocationProbe, error) { if c == nil || c.Holder == nil || c.Op.Kind != hostclaim.KindRelocating { return relocationProbe{}, hostbox.ErrChanged } certified, err := hostrelocate.CertifiedFor(c) if err != nil { return relocationProbe{}, err } auth, err := preparedAuthority(c) if err != nil { return relocationProbe{}, err } empty, err := emptyPrepared(c, auth) if err != nil { return relocationProbe{}, err } since, err := time.Parse(time.RFC3339Nano, c.Op.Journal["probing_since"]) if err != nil { return relocationProbe{}, fmt.Errorf("%w: probing_since: %v", hostbox.ErrChanged, err) } return relocationProbe{certified: certified, auth: auth, empty: empty, since: since}, nil } // relocationAdmit is one probe attempt: the activator's ProbeWake with the // probe deadline and, on P's prepared opening, P's install manifest. The // claim it reads was checked by Preflight before the attempt was counted // (relocationProbeClaim); it is read again here, on the counted // observation. The install manifest read and a ProbeWake error that is // not a probe result (an activator or launch failure) run after the count, // so an attempt spent on transient I/O is an attempt all the same (final // review minor 11; the same shape as moves' probe): until claim ask C11 // one attempt is one launch, and its follow-up (plan "After this plan", // C11) is where three attempts become three probes by moving these reads // before the count. func (a ClaimAssembly) relocationAdmit(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed, bool) (uint64, bool, error) { return func(ctx context.Context, o hostclaim.Observed, certified bool) (uint64, bool, error) { if cfg.Activator == nil || cfg.Versions == nil || o.Claim == nil || o.Claim.Holder == nil { return 0, false, ErrClaimStopUnavailable } c := o.Claim probe, err := relocationProbeClaim(c) if err != nil { return 0, false, err } if certified != probe.certified { return 0, false, fmt.Errorf("%w: the participant and the owners disagree on certification", hostrelocate.ErrRefused) } pc := activator.ProbeConfig{Host: o.Host, Op: c.Op.ID, HoldOp: c.Op.ID, Deadline: probe.since.Add(hostrelocate.ProbeDeadline)} if !hostclaim.PreparedOpened(c) && !probe.empty { im, _, err := readInstallManifest(ctx, cfg.Versions, cfg.Bucket, moves.GenerationPrefix(c.Holder.Location, probe.auth.Generation)) if err != nil { return 0, false, err } pc.Prepared = &im } res, err := cfg.Activator.ProbeWake(ctx, pc) switch { case err == nil: return res.Session, true, nil case errors.Is(err, activator.ErrProbeFailed), errors.Is(err, activator.ErrAttemptOver): return 0, false, nil default: return 0, false, err } } } // relocationStopProbe is ruling 49 steps 1 to 3, while the op is still // relocating/probing so the captured session still matches: the // activator's local fence at the session and its stop of the probe entry; // main's writer-session stop through root (deterministic, so a repeat // meets the same fence and receipt), which also releases the attempt's // lease; then root's receipt for exactly that session. // // The fence is an owned one (ruling 75, Astra round 6 findings 1 and 2). // Before anything is asked of the activator, the owner this probe stop // will install under is persisted with the session in the op's journal // directory (probe-fence.json); a failed write installs nothing, and the // next pass writes it again. FenceAndStopAs then installs only on an // unfenced host, or meets again exactly the fence it installed under // that owner at that session, deciding under the activator's own lock. // Any other fence (another operation's, of any generation, including one // installed after an agent restart emptied the activator's fences while // the marker survived) is ErrFenceHeld, with nothing stopped and nothing // asked of root, and is never replaced, so ReleaseHold's lift can never // reopen admission another operation closed. func (a ClaimAssembly) relocationStopProbe(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed) (hostclaim.ProbeStop, error) { return func(ctx context.Context, o hostclaim.Observed) (hostclaim.ProbeStop, error) { if cfg.Activator == nil || a.Wake == nil || a.Wake.Stop == nil || o.Claim == nil || o.Claim.Holder == nil { return hostclaim.ProbeStop{}, ErrClaimStopUnavailable } local, err := a.localStopIdentity(ctx) if err != nil { return hostclaim.ProbeStop{}, err } c, h, box := o.Claim, o.Claim.Holder, local.Actor.Box if err := passedProbeSession(c, box); err != nil { return hostclaim.ProbeStop{}, err } marker, err := relocationProbeFencePath(cfg.DataDir, c.Op.ID) if err != nil { return hostclaim.ProbeStop{}, err } owner, err := relocationProbeFenceOwner(marker, o.Host, h.Session) if err != nil { return hostclaim.ProbeStop{}, err // nothing installed: the next pass persists it again } fenced, cancel := context.WithTimeout(ctx, hostbox.RequestLimit) _, err = cfg.Activator.FenceAndStopAs(fenced, o.Host, h.Session, owner) cancel() if err != nil && !errors.Is(err, activator.ErrStopUnproven) { return hostclaim.ProbeStop{}, err // not this probe's fence: nothing stopped, nothing asked } if err := a.Wake.Stop(ctx, o); err != nil { return hostclaim.ProbeStop{}, err } r, err := writerSessionReceipt(ctx, a.Guard, o.Host, hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp}, c.Op.ID, box, h.Session) if err != nil { return hostclaim.ProbeStop{}, err } sum := sha256.Sum256(r.Bytes) return hostclaim.ProbeStop{Box: box, Session: h.Session, Receipt: r.Target.FenceID, ReceiptSHA256: hex.EncodeToString(sum[:])}, nil } } // passedProbeSession checks c is this box's passed probe at // relocating/probing, its session the holder's own: the one session the // probe stop may flush (ruling 87) and stop (ruling 49). Anything else is // hostbox.ErrChanged, with nothing stopped. func passedProbeSession(c *hostclaim.Claim, box string) error { var p hostclaim.Probe if c == nil || c.Holder == nil || box == "" || json.Unmarshal([]byte(c.Op.Journal["probe"]), &p) != nil || p.Box != box || p.Session != c.Holder.Session || c.Op.Exec != box || c.Op.Phase != hostclaim.PhaseRelocatingProbing || c.Holder.WriterBox != box { return hostbox.ErrChanged } return nil } // relocationProbeFencePath is where this box's probe stop persists the // owner of the activator fence it installs, with the session and host: // /relocate/ops//probe-fence.json. The journal sweep keeps // the op's directory until the completed relocation is marked discharged // (ruling 76), so the lift always finds it. func relocationProbeFencePath(dataDir, op string) (string, error) { dir, err := relocationOpDir(dataDir, op) if err != nil { return "", err } return filepath.Join(dir, "probe-fence.json"), nil } // relocationProbeFence is the persisted owner of one probe stop's fence. type relocationProbeFence struct { Host string `json:"host"` Session uint64 `json:"session"` Owner string `json:"owner"` } // relocationProbeFenceOwner returns the owner persisted for host's probe // session, or persists a new, random one first. A marker that is present // is the only record of the owner a fence may already be installed under, // so it is never replaced: one that cannot be read, or that names another // host or session, refuses (absence is never evidence), nothing is // installed, and the marker stays for an operator. The op's probe session // is fixed once its probe passed, so such a marker is a defect, never a // retry's. func relocationProbeFenceOwner(path, host string, session uint64) (string, error) { rec, found, err := readRelocationProbeFence(path) if err != nil { return "", err } if found { if rec.Host != host || rec.Session != session { return "", fmt.Errorf("%w: %s names %s session %d, not %s session %d", hostrelocate.ErrRefused, path, rec.Host, rec.Session, host, session) } return rec.Owner, nil } var b [16]byte if _, err := rand.Read(b[:]); err != nil { return "", err } rec = relocationProbeFence{Host: host, Session: session, Owner: hex.EncodeToString(b[:])} raw, err := json.Marshal(rec) if err != nil { return "", err } if err := writeRelocationJournal(path, raw); err != nil { return "", err } return rec.Owner, nil } // readRelocationProbeFence reads the persisted owner; false, with no // error, only when there is no marker at all. A marker that does not // decode, or names no host, session or owner, is an error, never "none". func readRelocationProbeFence(path string) (relocationProbeFence, bool, error) { raw, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { return relocationProbeFence{}, false, nil } if err != nil { return relocationProbeFence{}, false, err } var rec relocationProbeFence if err := json.Unmarshal(raw, &rec); err != nil { return relocationProbeFence{}, false, fmt.Errorf("%w: %s cannot be read: %v", hostrelocate.ErrRefused, path, err) } if rec.Host == "" || rec.Session == 0 || rec.Owner == "" { return relocationProbeFence{}, false, fmt.Errorf("%w: %s names no host, session or owner", hostrelocate.ErrRefused, path) } return rec, true, nil } // relocationReleaseHold finishes a completed relocation on the box it // completed on (ruling 49 step 5, rulings 76 and 80), in this order: while // the row is held for the op, the holder must still be the stopped probe // session, main's participation is discharged (C10) and the hold is // released (ruling 66: nothing is released before the discharge); then // the activator fence the probe stop installed under its persisted owner // is lifted; last, the serving op's completed relocation is marked // discharged, which is what lets any successor start (the completion // barrier, ruling 77). The pass calls it for every serving claim whose // completion record names this box and is not yet marked, never from the // row (ruling 80), and every step is idempotent, so a failure or a crash // at any point leaves the record unmarked and the next pass resumes where // it stopped. Once the row's hold is released, a replay discharges again // rather than inferring that the discharge ran (absence is never // evidence): main's discharge then finds no slot and no .hostexec record, // which is positive evidence, and does nothing more; only then are the // fence lifted and the record marked. A row held by any other op is // refused with nothing done: nothing has proved this op's participation // discharged. A missing row is refused: absence is never evidence of the // discharge. So is a missing, unreadable or mismatched probe-fence owner, // read before anything is released: the probe stop persisted it before it // fenced, and without it nothing can say the fence is lifted, so the mark // is never written over its absence. A claim that is absent, or does not // serve on this box, is ErrChanged: the pass asks only about completions // recorded on this box (ruling 80), so a caller that asks about anything // else is mis-wired, never finished. Anything else leaves everything: an // excluded box's row is its park-local's. func (a ClaimAssembly) relocationReleaseHold(cfg RelocationAssemblyConfig) func(context.Context, string, string) error { return func(ctx context.Context, host, op string) error { if cfg.Registry == nil || cfg.Activator == nil || cfg.Discharge == nil { return ErrClaimStopUnavailable } local, err := a.localStopIdentity(ctx) if err != nil { return err } obs, err := hostclaim.Read(ctx, a.Claims, host) if err != nil { return err } if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.Exec != local.Actor.Box || obs.Claim.Holder.WriterBox != local.Actor.Box { return fmt.Errorf("%w: %s does not serve on this box", hostbox.ErrChanged, host) } done, ok, err := hostclaim.ReadCompletedRelocation(obs.Claim) if err != nil { return err // unreadable is not "no completion": nothing is released over it } if !ok || done.Op != op || done.Stop.Box != local.Actor.Box { return fmt.Errorf("%w: %s's serving op does not record %s as completed on this box", hostrelocate.ErrRefused, host, op) } if done.Discharged { return nil } row, err := cfg.Registry.Get(host) if err != nil { return err } if row == nil { return fmt.Errorf("%w: %s has no row on this box", hostrelocate.ErrRefused, host) } if row.HoldOp != "" && row.HoldOp != op { return fmt.Errorf("%w: %s's row is held by %s, not %s", hostrelocate.ErrRefused, host, row.HoldOp, op) } marker, err := relocationProbeFencePath(cfg.DataDir, op) if err != nil { return err } fence, found, err := readRelocationProbeFence(marker) if err != nil { return err // the owner is unreadable: nothing is released, the fence stays, the claim is not marked } if !found || fence.Host != host || fence.Session != done.Stop.Session { return fmt.Errorf("%w: %s has no probe-fence owner for %s's stopped session %d", hostrelocate.ErrRefused, host, op, done.Stop.Session) } if row.HoldOp == op { if done.Stop.Session != obs.Claim.Holder.Session { return fmt.Errorf("%w: %s's serving holder is not %s's stopped probe session", hostrelocate.ErrRefused, host, op) } if err := cfg.Discharge(ctx, obs); err != nil { return err } if _, err := cfg.Registry.ReleaseHold(host, op); err != nil { return err } } else if err := cfg.Discharge(ctx, obs); err != nil { // The hold is already released. The holder may have moved on // since, so there is no holder check here: main's serving Check // let no wake through while a slot or record remained, and the // discharge's postcondition (no slot, no record) is the // evidence, not the released hold. return err } cfg.Activator.LiftFenceAs(host, fence.Session, fence.Owner) _, err = hostclaim.Update(ctx, a.Claims, obs, hostclaim.MarkCompletedDischarged(op)) if errors.Is(err, hostclaim.ErrAlreadyRecorded) { return nil } return err } }