package host import ( "context" "encoding/json" "errors" "fmt" "io" "os" "os/exec" "path/filepath" "strconv" "sync" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostrelocate" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/instancefs" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/release" ) // relocationAttemptJournal is one operation's probe journal, at // /relocate/ops//attempts.json (ruling 41, keyed by op per // ruling 44), written whole and renamed. The claim's probe_attempts. // is the record attempt k; the journal binds each admitted session to one // k, keeps each k's first app start, and each k's current start. Every // field has its own JSON name (finding 14). type relocationAttemptJournal struct { V int `json:"v"` Host string `json:"host"` Op string `json:"op"` Opened map[string]time.Time `json:"opened"` // "k" -> the attempt's first app start Sessions map[string]int `json:"sessions"` // "session" -> k Starts map[string]relocationStart `json:"starts"` // "k" -> the attempt's current start } // relocationStart is one start's reservation, bound to the admitted // session and generation root's reader validates (Plan 11: the exact // operation, phase, generation, stamp and session). type relocationStart struct { Session uint64 `json:"session"` Generation string `json:"generation"` Stamp uint64 `json:"stamp"` N int `json:"n"` Attempt hostclaim.WakeRetryAttempt `json:"attempt"` State string `json:"state"` // reserved, failed, healthy } func relocationAttemptsPath(dataDir, op string) (string, error) { dir, err := relocationOpDir(dataDir, op) if err != nil { return "", err } return filepath.Join(dir, "attempts.json"), nil } // readRelocationAttempts is the agent's read of its own journal: a // missing file is the journal's first state (OpenProbe creates it). func readRelocationAttempts(dataDir, host, op string) (relocationAttemptJournal, error) { path, err := relocationAttemptsPath(dataDir, op) if err != nil { return relocationAttemptJournal{}, err } raw, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { return relocationAttemptJournal{V: 1, Host: host, Op: op, Opened: map[string]time.Time{}, Sessions: map[string]int{}, Starts: map[string]relocationStart{}}, nil } if err != nil { return relocationAttemptJournal{}, err } return decodeRelocationAttempts(raw, host, op) } func decodeRelocationAttempts(raw []byte, host, op string) (relocationAttemptJournal, error) { var j relocationAttemptJournal if err := json.Unmarshal(raw, &j); err != nil || j.V != 1 || j.Host != host || j.Op != op || j.Opened == nil || j.Sessions == nil || j.Starts == nil { return relocationAttemptJournal{}, fmt.Errorf("%w: relocation attempt journal %s", hostbox.ErrChanged, op) } return j, nil } // relocationAttemptsLimit bounds root's read of the journal: three record // attempts, each with one start and one session binding, is well under a // kibibyte. const relocationAttemptsLimit = 64 << 10 // readRelocationAttemptsBeneath is root's read of the agent-written // journal (fix round 1, I1). The file and its directories belong to the // agent's uid, which may replace them at will, so root opens it the way // it opens every agent-writable path on main: beneath the data root with // openat2 (RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS, instancefs), O_NONBLOCK // so a FIFO cannot hold root under the host guard, a regular file only, // the size capped before and during the read, and the context honoured. // A missing journal is a refusal, never a fresh one: root takes nothing // the journal does not hold. func readRelocationAttemptsBeneath(ctx context.Context, dataDir, host, op string) (relocationAttemptJournal, error) { if err := ctx.Err(); err != nil { return relocationAttemptJournal{}, err } if dataDir == "" || !validOpID(op) { return relocationAttemptJournal{}, ErrClaimStopUnavailable } f, err := instancefs.OpenBeneath(dataDir, filepath.Join("relocate", "ops", op, "attempts.json"), os.O_RDONLY, 0) if err != nil { return relocationAttemptJournal{}, fmt.Errorf("%w: relocation attempt journal %s: %w", hostbox.ErrChanged, op, err) } defer f.Close() st, err := f.Stat() if err != nil { return relocationAttemptJournal{}, err } if !st.Mode().IsRegular() || st.Size() > relocationAttemptsLimit { return relocationAttemptJournal{}, fmt.Errorf("%w: relocation attempt journal %s is not a regular file within %d bytes", hostbox.ErrChanged, op, relocationAttemptsLimit) } raw, err := io.ReadAll(io.LimitReader(f, relocationAttemptsLimit+1)) if err != nil { return relocationAttemptJournal{}, err } if len(raw) > relocationAttemptsLimit { return relocationAttemptJournal{}, fmt.Errorf("%w: relocation attempt journal %s grew past %d bytes", hostbox.ErrChanged, op, relocationAttemptsLimit) } if err := ctx.Err(); err != nil { return relocationAttemptJournal{}, err } return decodeRelocationAttempts(raw, host, op) } func (j relocationAttemptJournal) write(dataDir string) error { path, err := relocationAttemptsPath(dataDir, j.Op) if err != nil { return err } raw, err := json.Marshal(j) if err != nil { return err } return writeRelocationJournal(path, raw) } // relocationProbeAttempts is region HA's OperationAttempts for a probe // admitted under relocating/probing (claim Plan 11 R1, D2), modelled on // movesProbeAttempts without the record. Every method refuses // (hostbox.ErrChanged) once the claim no longer names the admitted writer // at probing, unstuck, with k = probe_attempts. in 1..MaxProbeAttempts. type relocationProbeAttempts struct { Claims release.Store DataDir string Box string NewID func() string Now func() time.Time Settle, Ready time.Duration mu sync.Mutex current map[string]boundAttempt // host\x00session } var _ OperationAttempts = (*relocationProbeAttempts)(nil) func (p *relocationProbeAttempts) bound(ctx context.Context, s OperationStart) (int, error) { if p == nil || p.Claims == nil || p.DataDir == "" || p.Box == "" || p.NewID == nil || p.Now == nil { return 0, hostwrite.ErrUnavailable } a := s.Admitted if a.Claim == nil || a.Claim.Holder == nil || a.Claim.Op.Kind != hostclaim.KindRelocating || !s.Policy.Probe { return 0, hostbox.ErrUnsupported } h := a.Claim.Holder if s.Owner.Host != a.Host || s.Owner.Session != h.Session || s.Owner.Stamp != h.Stamp || s.Owner.Incarnation != h.Incarnation || s.Owner.Box != p.Box { return 0, hostbox.ErrChanged } fresh, err := hostclaim.Read(ctx, p.Claims, a.Host) if err != nil { return 0, err } if !claimSameWriter(a, fresh, hostclaim.SessionContinues) || fresh.Claim.Op.Phase != hostclaim.PhaseRelocatingProbing || fresh.Claim.Op.Journal["stuck"] != "" || fresh.Claim.Op.Exec != p.Box { return 0, hostbox.ErrChanged } k, err := strconv.Atoi(fresh.Claim.Op.Journal["probe_attempts."+p.Box]) if err != nil || k < 1 || k > hostclaim.MaxProbeAttempts { return 0, hostbox.ErrChanged } return k, nil } func relocationSessionKey(s OperationStart) string { return s.Admitted.Host + "\x00" + strconv.FormatUint(s.Owner.Session, 10) } // bind checks, and with write records, the session's record attempt. func (j relocationAttemptJournal) bind(session uint64, k int, write bool) error { key := strconv.FormatUint(session, 10) got, ok := j.Sessions[key] switch { case ok && got == k: return nil case ok: return fmt.Errorf("%w: session %d is bound to attempt %d, not %d", hostbox.ErrChanged, session, got, k) case !write: return fmt.Errorf("%w: session %d is bound to no attempt", hostbox.ErrChanged, session) } j.Sessions[key] = k return nil } func (p *relocationProbeAttempts) OpenProbe(ctx context.Context, s OperationStart, proposed time.Time) (time.Time, error) { k, err := p.bound(ctx, s) if err != nil { return time.Time{}, err } if proposed.IsZero() { return time.Time{}, hostwrite.ErrUnavailable } p.mu.Lock() defer p.mu.Unlock() j, err := readRelocationAttempts(p.DataDir, s.Admitted.Host, s.Admitted.Claim.Op.ID) if err != nil { return time.Time{}, err } if err := j.bind(s.Owner.Session, k, true); err != nil { return time.Time{}, err } key := strconv.Itoa(k) opened, ok := j.Opened[key] if !ok { opened = proposed.UTC() j.Opened[key] = opened } return opened, j.write(p.DataDir) } func (p *relocationProbeAttempts) Reserve(ctx context.Context, s OperationStart, n int) error { k, err := p.bound(ctx, s) if err != nil { return err } digest, err := ClaimLaunchDigest(s.Inputs) if err != nil { return err } p.mu.Lock() defer p.mu.Unlock() j, err := readRelocationAttempts(p.DataDir, s.Admitted.Host, s.Admitted.Claim.Op.ID) if err != nil { return err } if err := j.bind(s.Owner.Session, k, n == 0); err != nil { return err } key := strconv.Itoa(k) st, ok := j.Starts[key] same := ok && st.Session == s.Owner.Session h := s.Admitted.Claim.Holder switch { case n < 0: return hostbox.ErrChanged case n == 0 && same: if st.N != 0 || st.State != probeReserved || !sameStartOwner(st.Attempt, s, digest) || st.Generation != h.Generation || st.Stamp != h.Stamp { return hostbox.ErrChanged } case n == 0: st = relocationStart{Session: s.Owner.Session, Generation: h.Generation, Stamp: h.Stamp, State: probeReserved} default: if !same || st.N != n-1 || st.State != probeFailed || st.Attempt.LaunchInputs != digest { return hostbox.ErrChanged } st = relocationStart{Session: s.Owner.Session, Generation: h.Generation, Stamp: h.Stamp, N: n, State: probeReserved} } if st.Attempt.ID == "" { o := s.Owner st.Attempt = hostclaim.WakeRetryAttempt{ID: p.NewID(), Box: o.Box, Enrollment: o.Enrollment, BootID: o.BootID, ControllerID: o.ControllerID, Session: o.Session, InvocationID: p.NewID(), LaunchInputs: digest} if st.Attempt.ID == "" || st.Attempt.InvocationID == "" || st.Attempt.ID == st.Attempt.InvocationID { return hostwrite.ErrUnavailable } j.Starts[key] = st } if err := j.write(p.DataDir); err != nil { return err } if p.current == nil { p.current = map[string]boundAttempt{} } p.current[relocationSessionKey(s)] = boundAttempt{k: k, attempt: st.Attempt} return nil } func (p *relocationProbeAttempts) CurrentAttempt(s OperationStart) hostclaim.WakeRetryAttempt { if p == nil { return hostclaim.WakeRetryAttempt{} } p.mu.Lock() cur, ok := p.current[relocationSessionKey(s)] p.mu.Unlock() if !ok { return hostclaim.WakeRetryAttempt{} } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if k, err := p.bound(ctx, s); err != nil || k != cur.k { return hostclaim.WakeRetryAttempt{} } return cur.attempt } func (p *relocationProbeAttempts) Failed(ctx context.Context, s OperationStart, n int) error { recovery, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() return p.settle(recovery, s, n, probeFailed) } func (p *relocationProbeAttempts) settle(ctx context.Context, s OperationStart, n int, state string) error { k, err := p.bound(ctx, s) if err != nil { return err } p.mu.Lock() defer p.mu.Unlock() j, err := readRelocationAttempts(p.DataDir, s.Admitted.Host, s.Admitted.Claim.Op.ID) if err != nil { return err } if err := j.bind(s.Owner.Session, k, false); err != nil { return err } key := strconv.Itoa(k) st, ok := j.Starts[key] cur := p.current[relocationSessionKey(s)] if !ok || st.Session != s.Owner.Session || st.N != n || st.State != probeReserved || cur.k != k || st.Attempt != cur.attempt { return hostbox.ErrChanged } st.State = state j.Starts[key] = st return j.write(p.DataDir) } // Healthy is movesProbeAttempts.Healthy's settle and second request // (moves §7.7), which the relocation probe shares (spec §5.8). func (p *relocationProbeAttempts) Healthy(ctx context.Context, s OperationStart, n int, cmd *exec.Cmd, again func(context.Context) error) error { if _, err := p.bound(ctx, s); err != nil { return err } if again == nil || !probeAlive(cmd) { return hostbox.ErrChanged } settle, ready := p.Settle, p.Ready if settle <= 0 { settle = time.Duration(moves.SettleSeconds * float64(time.Second)) } if ready <= 0 { ready = time.Duration(moves.ProbeSeconds * float64(time.Second)) } timer := time.NewTimer(settle) select { case <-ctx.Done(): timer.Stop() return ctx.Err() case <-timer.C: } if !probeAlive(cmd) { return fmt.Errorf("%w: the app exited during the settle", hostbox.ErrChanged) } asked, cancel := context.WithTimeout(ctx, ready) err := again(asked) cancel() if err != nil { return err } if !probeAlive(cmd) { return fmt.Errorf("%w: the app exited after the second request", hostbox.ErrChanged) } return p.settle(ctx, s, n, probeHealthy) } // relocationPassedProbe is hostrelocate.Effects.PassedProbe (final review // A2): the attempt journal's healthy start for the claim's holder session, // the state Healthy settles once the probe passed, read as the agent // reads its own journal. The start must be the executor's record attempt // k, bound to the holder's session, generation and stamp, so a start of // another attempt or session is never taken for this one. After P's // opening the journal exists (OpenProbe wrote it before the app first // started), so a missing journal is a refusal like an unreadable one, // never "no probe passed": the worker's deadlines decide such an // operation, not an absence. func relocationPassedProbe(dataDir string) func(context.Context, hostclaim.Observed) (uint64, bool, error) { return func(_ context.Context, o hostclaim.Observed) (uint64, bool, error) { c := o.Claim if dataDir == "" || c == nil || c.Holder == nil || c.Op.Kind != hostclaim.KindRelocating || c.Op.Phase != hostclaim.PhaseRelocatingProbing || !hostclaim.PreparedOpened(c) { return 0, false, ErrClaimStopUnavailable } h := c.Holder if h.WriterBox != c.Op.Exec || h.Session == 0 { return 0, false, hostbox.ErrChanged } k, err := strconv.Atoi(c.Op.Journal["probe_attempts."+c.Op.Exec]) if err != nil || k < 1 || k > hostclaim.MaxProbeAttempts { return 0, false, hostbox.ErrChanged } path, err := relocationAttemptsPath(dataDir, c.Op.ID) if err != nil { return 0, false, err } raw, err := os.ReadFile(path) if err != nil { return 0, false, fmt.Errorf("%w: the attempt journal of %s after P's opening: %w", hostrelocate.ErrRefused, c.Op.ID, err) } j, err := decodeRelocationAttempts(raw, o.Host, c.Op.ID) if err != nil { return 0, false, err } st, ok := j.Starts[strconv.Itoa(k)] if !ok || j.Sessions[strconv.FormatUint(h.Session, 10)] != k || st.Session != h.Session || st.Generation != h.Generation || st.Stamp != h.Stamp || st.Attempt.Box != c.Op.Exec || st.State != probeHealthy { return 0, false, nil } return h.Session, true, nil } } // relocationAttemptReader is root's authoritative attempt reader for kind // relocating (ruling 48; claim Plan 11 "install and test root-owned // authoritative attempt readers"): it reads the operation's durable // journal and returns the reserved start only for exactly the observed // op at probing, its executor's attempt k, the holder's session, // generation and stamp (the journaled install stamp). Root's authorize // then binds the reservation, the launch-input digest and the peer. // // The journal is the agent's file (the same shape as moves' probe journal // on main; claim ask C14 confirms it as root authority). Every field root // uses that the claim also has is cross-checked against root's own claim // read: the op (the journal's header), the host, the record attempt k // (probe_attempts.), the holder's session, generation and stamp, and // the executor as the attempt's box. The read itself is readRelocation- // AttemptsBeneath's (I1). type relocationAttemptReader struct{ DataDir string } func (r relocationAttemptReader) ReadAttempt(ctx context.Context, o hostclaim.Observed) (hostclaim.WakeRetryAttempt, error) { c := o.Claim if r.DataDir == "" || c == nil || c.Holder == nil || c.Op.Kind != hostclaim.KindRelocating || c.Op.Phase != hostclaim.PhaseRelocatingProbing || c.Op.Journal["stuck"] != "" { return hostclaim.WakeRetryAttempt{}, hostbox.ErrChanged } h := c.Holder // P's opening keeps the install stamp; a later cold attempt runs at // its own admission's stamp (generation.go: !KeepStamp sets stamp to // the session), which the journaled start must name (Astra round 2 // finding 4). Main's claimLaunchAuthority still pins every relocating // launch to install_stamp until claim ask C11. if h.Generation == c.Op.Journal["prepared"] && c.Op.Journal["install_stamp"] != strconv.FormatUint(h.Stamp, 10) { return hostclaim.WakeRetryAttempt{}, hostbox.ErrChanged } k, err := strconv.Atoi(c.Op.Journal["probe_attempts."+c.Op.Exec]) if err != nil || k < 1 || k > hostclaim.MaxProbeAttempts { return hostclaim.WakeRetryAttempt{}, hostbox.ErrChanged } j, err := readRelocationAttemptsBeneath(ctx, r.DataDir, o.Host, c.Op.ID) if err != nil { return hostclaim.WakeRetryAttempt{}, err } if j.Sessions[strconv.FormatUint(h.Session, 10)] != k { return hostclaim.WakeRetryAttempt{}, hostbox.ErrChanged } st, ok := j.Starts[strconv.Itoa(k)] if !ok || st.State != probeReserved || st.Session != h.Session || st.Generation != h.Generation || st.Stamp != h.Stamp || st.Attempt.Session != h.Session || st.Attempt.Box != c.Op.Exec { return hostclaim.WakeRetryAttempt{}, hostbox.ErrChanged } return st.Attempt, nil }