package hostbox import ( "context" "crypto/rand" "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" "errors" "os" "path/filepath" "sync" "time" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/rootstate" ) var ErrLegacyJournalProof = errors.New("legacy journal requires root-signed stop evidence") // ErrJournalUncertain says nothing about whether a request committed. var ErrJournalUncertain = errors.New("hostbox: journal request outcome uncertain") // A mutation executes only within JournalRequestWindow of the caller's issue // time, so a receipt older than JournalReceiptRetention can never be needed: // its request either replayed already or can only ever answer uncertain. const ( JournalRequestWindow = 15 * time.Minute JournalReceiptRetention = 24 * time.Hour journalPruneBatch = 256 ) // journalPrepare fixes a request's identity once, before its first attempt. // Reads get a fresh random ID. Every mutation is v2: a per-call nonce and issue time // are hashed in, so two distinct calls with identical payloads never share a // receipt. Retries resend the same prepared request. func journalPrepare(req JournalRequest, now time.Time) (JournalRequest, error) { req.Nonce, req.IssuedAt = "", 0 switch { case journalRead(req.Op): var id [16]byte if _, err := rand.Read(id[:]); err != nil { return req, err } req.V = 1 req.ID = hex.EncodeToString(id[:]) default: var nonce [16]byte if _, err := rand.Read(nonce[:]); err != nil { return req, err } req.V = 2 if req.RecordKey != "" || req.RecordETag != "" { req.V = 3 } if req.ServingPark != nil { req.V = 4 } req.Nonce = hex.EncodeToString(nonce[:]) req.IssuedAt = now.UnixNano() req.ID = journalRequestID(req) } return req, nil } // JournalRequest carries values, never SQL or caller-chosen filesystem paths. // ID is retained by the transport across retries. Journal-only effects and // their replies share a FULL-synchronous transaction. Parking records a pending // receipt before filesystem work and a completed reply afterwards. type JournalRequest struct { Timer *TimerMutation `json:"timer,omitempty"` ServingPark *ServingParkRequest `json:"serving_park,omitempty"` RecordKey string `json:"record_key,omitempty"` RecordETag string `json:"record_etag,omitempty"` Revision uint64 `json:"inventory_revision"` V int `json:"v"` ID string `json:"request_id"` Op string `json:"operation"` Target hostguard.FenceTarget `json:"target"` Launch LaunchInventory `json:"launch"` Reservation LaunchReservation `json:"reservation"` File LaunchFile `json:"file"` Producer string `json:"producer"` Previous ProducerCompletion `json:"previous"` PID int `json:"pid"` ProcessStart string `json:"process_start"` Inventory StopInventory `json:"inventory"` Nonce string `json:"nonce,omitempty"` IssuedAt int64 `json:"issued_at,omitempty"` } type JournalResponse struct { ServingPark *ServingParkResult `json:"serving_park,omitempty"` FenceView *hostguard.FenceView `json:"fence_view,omitempty"` Revision uint64 `json:"inventory_revision"` LastRequest string `json:"last_request"` V int `json:"v"` ID string `json:"request_id"` Launch LaunchInventory `json:"launch"` Launches []LaunchInventory `json:"launches"` Inventory StopInventory `json:"inventory"` Completion ProducerCompletion `json:"completion"` Found bool `json:"found"` Error string `json:"error"` } type JournalCall func(context.Context, JournalRequest) (JournalResponse, error) // JournalUncertainError retains the exact request for explicit same-ID replay. // Read-back can inform recovery; it never proves that a pending write failed. type JournalUncertainError struct { Request JournalRequest Cause error } func (e *JournalUncertainError) Error() string { return "journal request outcome uncertain" } func (e *JournalUncertainError) Unwrap() error { return e.Cause } func RemoteJournals(call JournalCall) (*LaunchJournal, *StopInventoryDB) { return &LaunchJournal{remote: call}, &StopInventoryDB{remote: call} } func journalCall(ctx context.Context, call JournalCall, req JournalRequest) (JournalResponse, error) { req, err := journalPrepare(req, time.Now()) if err != nil { return JournalResponse{}, err } return journalSend(ctx, call, req, false) } // ReplayUncertain resends the exact request an uncertain call retained. The // earlier attempt may have committed, so the replay stays uncertain on every // refusal, including after the receipt was pruned. Only a reply that names // this exact request with no error confirms it. func (j *LaunchJournal) ReplayUncertain(ctx context.Context, uncertain *JournalUncertainError) (JournalResponse, error) { if j == nil || j.remote == nil || uncertain == nil || uncertain.Request.ID == "" { return JournalResponse{}, ErrUnsupported } return journalSend(ctx, j.remote, uncertain.Request, true) } func journalSend(ctx context.Context, call JournalCall, req JournalRequest, replay bool) (JournalResponse, error) { out, err := call(ctx, req) if err != nil { return out, &JournalUncertainError{Request: req, Cause: err} } if out.V != 1 || out.ID != req.ID { return out, &JournalUncertainError{Request: req, Cause: ErrJournalUncertain} } switch out.Error { case "": return out, nil } if replay { return out, &JournalUncertainError{Request: req, Cause: ErrJournalUncertain} } switch out.Error { case "missing": return out, sql.ErrNoRows case "changed": return out, ErrChanged case "unsupported": return out, ErrUnsupported case "uncertain": return out, &JournalUncertainError{Request: req, Cause: ErrJournalUncertain} default: return out, ErrTypedInventory } } // JournalReadOrSettle identifies read-only operations and positive completion // of already issued work. It never includes creating a new producer or role. func JournalReadOrSettle(op string) bool { return journalRead(op) || op == "close-writers" || op == "started" || op == "settled" || op == "complete" } func journalRead(op string) bool { return op == "fence-view" || op == "read" || op == "session" || op == "has-host" || op == "has-effects" || op == "inventory-read" || op == "inventory-exact" || op == "inventory-version" } // journalRequestV1 freezes every field, including nested inventories, in an // ordered v1 tuple. Do not extend these tuples when the public structs grow: // a new hashed field requires a new protocol version and a separate encoder. func journalRequestV1(q JournalRequest) []byte { target := func(t hostguard.FenceTarget) any { return []any{t.Region, t.ClaimGen, t.ObligationID, t.FenceID, t.Host, t.Incarnation, t.Stamp, t.Session, t.Op, t.Phase, t.Box, t.Enrollment, t.BootID, t.ControllerID, t.ClaimETag, t.DirectoryRevision} } file := func(f LaunchFile) any { return []any{f.Device, f.Inode} } group := func(g LaunchGroup) any { return []any{g.RelativePath, g.Device, g.Inode, g.InvocationID} } reservation := func(r LaunchReservation) any { return []any{r.ID, r.Role, r.InvocationID, r.Inputs, r.State, r.ProcessStart, target(r.Target), group(r.Group), group(r.Parent), file(r.Executable), r.PID} } groups := make([]any, len(q.Launch.Groups)) for i, g := range q.Launch.Groups { groups[i] = group(g) } files := make([]any, len(q.Launch.Files)) for i, f := range q.Launch.Files { files[i] = file(f) } reservations := make([]any, len(q.Launch.Reservations)) for i, r := range q.Launch.Reservations { reservations[i] = reservation(r) } launch := []any{q.Launch.Completed.Count, q.Launch.Completed.Digest, q.Launch.ControllerPID, q.Launch.ProcessStart, target(q.Launch.Target), groups, files, append([]string{}, q.Launch.ProducerIDs...), q.Launch.NoState, reservations} entries := make([]any, len(q.Inventory.Entries)) for i, e := range q.Inventory.Entries { entries[i] = []any{e.Device, e.Inode, e.Type, e.Kind, e.ArtifactID, e.ManagedRoot, e.RelativePath, e.RowIdentity, e.Ownership, e.Materialization} } inventory := []any{q.Inventory.Complete, q.Inventory.Host, q.Inventory.Incarnation, q.Inventory.InstallationStamp, q.Inventory.Box, q.Inventory.Enrollment, q.Inventory.DirectoryRevision, entries} raw, _ := json.Marshal([]any{"carlos-journal-request-v1", q.Revision, q.Op, target(q.Target), launch, reservation(q.Reservation), file(q.File), q.Producer, q.Previous.Count, q.Previous.Digest, q.PID, q.ProcessStart, inventory}) return raw } // journalRequestV2 is the frozen v1 tuple plus the caller's per-call nonce and // issue time. It is a separate encoder; v1 identities are unchanged. func journalRequestV2(q JournalRequest) []byte { raw, _ := json.Marshal([]any{"carlos-journal-request-v2", string(journalRequestV1(q)), q.Nonce, q.IssuedAt}) return raw } func journalRequestHash(req JournalRequest) [sha256.Size]byte { identities := make([]rootstate.FileIdentity, len(req.Inventory.Entries)) present := false for i, e := range req.Inventory.Entries { identities[i] = e.Identity present = present || e.Identity != (rootstate.FileIdentity{}) } if present { base := req base.Inventory.Entries = append([]StopInventoryEntry(nil), req.Inventory.Entries...) for i := range base.Inventory.Entries { base.Inventory.Entries[i].Identity = rootstate.FileIdentity{} } raw, _ := json.Marshal([]any{"carlos-inventory-identity-v1", journalRequestHash(base), identities}) return sha256.Sum256(raw) } if req.V == 4 { raw, _ := json.Marshal([]any{"carlos-journal-request-v4", string(journalRequestV2(req)), journalServingParkV4(req.ServingPark)}) return sha256.Sum256(raw) } if req.V == 3 { raw, _ := json.Marshal([]any{"carlos-journal-request-v3", string(journalRequestV2(req)), req.RecordKey, req.RecordETag}) return sha256.Sum256(raw) } if req.V == 2 { return sha256.Sum256(journalRequestV2(req)) } return sha256.Sum256(journalRequestV1(req)) } func journalRequestID(req JournalRequest) string { sum := journalRequestHash(req) return hex.EncodeToString(sum[:16]) } func journalRequestDigest(req JournalRequest) string { sum := journalRequestHash(req) return hex.EncodeToString(sum[:]) } func validateJournalRequest(q JournalRequest) error { if q.V == 4 { return validateServingParkRequest(q) } if q.ServingPark != nil || ServingParkOperation(q.Op) { return ErrTypedJournal } // The frozen request hash does not carry this root-only response field. if q.Launch.WritersClosed || (q.V != 3 && (q.RecordKey != "" || q.RecordETag != "")) || (q.V == 3 && (q.Op != "inventory-write" || q.RecordKey == "" || q.RecordETag == "")) { return ErrTypedJournal } id, err := hex.DecodeString(q.ID) if err != nil || len(id) != 16 { return ErrTypedJournal } switch { case journalRead(q.Op): if q.V != 1 || q.Nonce != "" || q.IssuedAt != 0 { return ErrTypedJournal } return nil default: nonce, err := hex.DecodeString(q.Nonce) if (q.V != 2 && q.V != 3) || err != nil || len(nonce) != 16 || q.IssuedAt <= 0 { return ErrTypedJournal } } if q.ID != journalRequestID(q) { return ErrChanged } return nil } type journalRequestKey struct{} const inventoryVersionsSQL = "CREATE TABLE IF NOT EXISTS journal_inventory_versions(host TEXT NOT NULL, incarnation TEXT NOT NULL, stamp INTEGER NOT NULL, revision INTEGER NOT NULL CHECK(revision>0 AND typeof(revision)='integer'), request TEXT NOT NULL, PRIMARY KEY(host,incarnation,stamp))" type journalExecutor interface { ExecContext(context.Context, string, ...any) (sql.Result, error) QueryContext(context.Context, string, ...any) (*sql.Rows, error) QueryRowContext(context.Context, string, ...any) *sql.Row } type journalTransactionKey struct{} func journalExecutorFor(ctx context.Context, db *sql.DB) journalExecutor { if tx, ok := ctx.Value(journalTransactionKey{}).(*sql.Tx); ok { return tx } return db } func journalTransaction(ctx context.Context, db *sql.DB) (*sql.Tx, func() error, func() error, error) { if tx, ok := ctx.Value(journalTransactionKey{}).(*sql.Tx); ok { return tx, func() error { return nil }, func() error { return nil }, nil } tx, err := db.BeginTx(ctx, nil) if err != nil { return nil, nil, nil, err } return tx, tx.Commit, tx.Rollback, nil } func journalIdentity(path string, original os.FileInfo) error { if original == nil { return nil } now, err := os.Lstat(path) if err != nil { return err } if !now.Mode().IsRegular() || !os.SameFile(original, now) { return ErrChanged } return nil } // JournalService owns one protected database and serializes a request's replay // check, typed mutation and receipt. Local root recovery uses the same DB. A // replay never repeats an effect, even when the reply or root process was lost. // A v2 mutation executes only within JournalRequestWindow of its issue time, // so once a receipt is older than JournalReceiptRetention no request can // still need it: a replay inside the window found it, and anything later can // only answer uncertain. Mutations prune such settled receipts in bounded // batches. inventory-write needs no receipt to replay: its version row names // the last committed request. type JournalService struct { parkGate chan struct{} Launches *LaunchJournal Inventory *StopInventoryDB Now func() time.Time mu sync.Mutex } func NewJournalService(j *LaunchJournal) (*JournalService, error) { if j == nil || j.db == nil || j.remote != nil { return nil, ErrUnsupported } for _, q := range []string{inventoryVersionsSQL, "CREATE TABLE IF NOT EXISTS stop_inventory(host TEXT NOT NULL,incarnation TEXT NOT NULL,stamp INTEGER NOT NULL,body BLOB NOT NULL,PRIMARY KEY(host,incarnation,stamp))", "CREATE TABLE IF NOT EXISTS journal_requests(id TEXT PRIMARY KEY, peer TEXT NOT NULL, digest TEXT NOT NULL, response BLOB NOT NULL, created INTEGER NOT NULL DEFAULT 0, issued INTEGER NOT NULL DEFAULT 0)", "CREATE TABLE IF NOT EXISTS journal_receipt_retention(id INTEGER PRIMARY KEY CHECK(id=1), reject_before INTEGER NOT NULL)", "INSERT INTO journal_receipt_retention(id,reject_before) VALUES(1,0) ON CONFLICT(id) DO NOTHING", } { if _, err := j.db.Exec(q); err != nil { return nil, err } } // Receipts written before retention existed carry created=0 and are pruned // first; their v1 mutation IDs can no longer execute. var created int if err := j.db.QueryRow("SELECT count(*) FROM pragma_table_info('journal_requests') WHERE name='created'").Scan(&created); err != nil { return nil, err } if created == 0 { if _, err := j.db.Exec("ALTER TABLE journal_requests ADD COLUMN created INTEGER NOT NULL DEFAULT 0"); err != nil { return nil, err } } var issued int if err := j.db.QueryRow("SELECT count(*) FROM pragma_table_info('journal_requests') WHERE name='issued'").Scan(&issued); err != nil { return nil, err } if issued == 0 { if _, err := j.db.Exec("ALTER TABLE journal_requests ADD COLUMN issued INTEGER NOT NULL DEFAULT 0"); err != nil { return nil, err } } if _, err := j.db.Exec("CREATE INDEX IF NOT EXISTS journal_requests_created ON journal_requests(created)"); err != nil { return nil, err } dir, err := os.Open(filepath.Dir(j.path)) if err != nil { return nil, err } err = dir.Sync() closeErr := dir.Close() if err != nil || closeErr != nil { return nil, errors.Join(err, closeErr) } return &JournalService{parkGate: make(chan struct{}, 1), Launches: j, Inventory: &StopInventoryDB{db: j.db, shared: true}}, nil } // Replay returns only a byte-identical request by the same authenticated owner. func (s *JournalService) Replay(ctx context.Context, peer string, q JournalRequest) (JournalResponse, bool, error) { out := JournalResponse{V: 1, ID: q.ID} if err := validateJournalRequest(q); err != nil { return out, false, err } if journalRead(q.Op) { return out, false, nil } digestExpected := journalRequestDigest(q) var priorPeer, digest string var body []byte err := s.Launches.db.QueryRowContext(ctx, "SELECT peer,digest,response FROM journal_requests WHERE id=?", q.ID).Scan(&priorPeer, &digest, &body) if errors.Is(err, sql.ErrNoRows) { return out, false, nil } if err != nil { return out, false, err } if priorPeer != peer || digest != digestExpected { return out, false, ErrChanged } err = json.Unmarshal(body, &out) return out, true, err } func (s *JournalService) Execute(ctx context.Context, peer string, q JournalRequest) (JournalResponse, error) { return s.execute(ctx, peer, q, false) } func (s *JournalService) CompletePendingInventory(ctx context.Context, peer string, q JournalRequest) (JournalResponse, error) { if q.V != 3 || q.Op != "inventory-write" { return JournalResponse{}, ErrTypedJournal } return s.execute(ctx, peer, q, true) } func (s *JournalService) ExecuteRootInventory(ctx context.Context, peer string, q JournalRequest, in StopInventory, pending bool) (JournalResponse, error) { if q.V != 3 || q.Op != "inventory-write" || q.Inventory.Host != in.Host || q.Inventory.Incarnation != in.Incarnation || q.Inventory.InstallationStamp != in.InstallationStamp { return JournalResponse{}, ErrTypedJournal } return s.executeWithInventory(ctx, peer, q, pending, &in) } func (s *JournalService) execute(ctx context.Context, peer string, q JournalRequest, pendingInventory bool) (JournalResponse, error) { return s.executeWithInventory(ctx, peer, q, pendingInventory, nil) } func (s *JournalService) executeWithInventory(ctx context.Context, peer string, q JournalRequest, pendingInventory bool, inventory *StopInventory) (JournalResponse, error) { out := JournalResponse{V: 1, ID: q.ID} if err := validateJournalRequest(q); err != nil { return out, err } if peer == "" { return out, ErrTypedJournal } if journalRead(q.Op) { return s.read(ctx, q) } digest := journalRequestDigest(q) if inventory != nil { q.Inventory = *inventory } s.mu.Lock() defer s.mu.Unlock() tx, err := s.Launches.db.BeginTx(ctx, nil) if err != nil { return out, err } defer tx.Rollback() var priorPeer, priorDigest string var prior []byte err = tx.QueryRowContext(ctx, "SELECT peer,digest,response FROM journal_requests WHERE id=?", q.ID).Scan(&priorPeer, &priorDigest, &prior) if err == nil { if priorPeer != peer || priorDigest != digest { return out, ErrChanged } err = json.Unmarshal(prior, &out) return out, err } if !errors.Is(err, sql.ErrNoRows) { return out, err } // Inventory's permanent revision row can prove this exact nonce committed // even after the receipt expired. It never authorizes another execution. ctx = context.WithValue(ctx, journalTransactionKey{}, tx) if q.Op == "inventory-write" { prior, e := s.Inventory.inventoryVersion(ctx, hostguard.FenceTarget{Host: q.Inventory.Host, Incarnation: q.Inventory.Incarnation, Stamp: q.Inventory.InstallationStamp}) if e != nil { return out, e } if prior.Revision == q.Revision+1 && prior.LastRequest == q.ID { out.Revision = prior.Revision return out, nil } } now := time.Now() if s.Now != nil { now = s.Now() } var rejectBefore int64 if err = tx.QueryRowContext(ctx, "SELECT reject_before FROM journal_receipt_retention WHERE id=1").Scan(&rejectBefore); err != nil { return out, err } // The rejection boundary only ever rises to the issue time of a receipt // this transaction deletes, so a pruned request can never execute again // whatever the clock does, and a forward clock jump cannot raise it past // real time: it covers only requests that actually committed. if (q.V == 2 || q.V == 3) && !pendingInventory { age := now.Sub(time.Unix(0, q.IssuedAt)) if q.IssuedAt <= rejectBefore || age > JournalRequestWindow || age < -JournalRequestWindow { return out, ErrJournalUncertain } } rejectBefore, err = journalPrune(ctx, tx, rejectBefore, now) if err != nil { return out, err } ctx = context.WithValue(ctx, journalTransactionKey{}, tx) ctx = context.WithValue(ctx, journalRequestKey{}, q.ID) switch q.Op { case "close-writers": err = s.Launches.CloseWriters(ctx, q.Target) case "create": err = s.Launches.Create(ctx, q.Launch) case "prelaunch": err = s.Launches.Prelaunch(ctx, q.Launch) case "intend": err = s.Launches.IntendRole(ctx, q.Reservation) case "reserve": err = s.Launches.Reserve(ctx, q.Reservation) case "started": err = s.Launches.Started(ctx, q.Reservation, q.PID, q.ProcessStart) case "settled": err = s.Launches.Settled(ctx, q.Reservation) case "file": err = s.Launches.RecordFile(ctx, q.Target, q.File) case "producer": err = s.Launches.RecordProducer(ctx, q.Target, q.Producer) case "complete": out.Completion, err = s.Launches.CompleteProducer(ctx, q.Target, q.Producer, q.Previous) case "inventory-write": prior, e := s.Inventory.inventoryVersion(ctx, hostguard.FenceTarget{Host: q.Inventory.Host, Incarnation: q.Inventory.Incarnation, Stamp: q.Inventory.InstallationStamp}) if e != nil { return out, e } if prior.Revision != q.Revision { return out, ErrChanged } err = s.Inventory.WriteStopInventory(ctx, q.Inventory) out.Revision = prior.Revision + 1 default: return out, ErrUnsupported } // A failed operation is rolled back, so no partial mutation acquires a receipt. if err != nil { return out, err } body, err := json.Marshal(out) if err != nil { return out, err } if _, err = tx.ExecContext(ctx, "INSERT INTO journal_requests(id,peer,digest,response,created,issued) VALUES(?,?,?,?,?,?)", q.ID, peer, digest, body, now.UnixNano(), q.IssuedAt); err != nil { return out, err } if err = tx.Commit(); err != nil { return out, err } return out, nil } // Reads neither take a write transaction nor allocate durable request receipts. // inventory-version is a single joined SELECT, so revision/body/last ID form // one SQLite snapshot even when a root writer commits concurrently. func (s *JournalService) read(ctx context.Context, q JournalRequest) (JournalResponse, error) { out := JournalResponse{V: 1, ID: q.ID} var err error switch q.Op { case "read": out.Launch, err = s.Launches.Read(ctx, q.Target) case "session": out.Launches, err = s.Launches.SessionLaunches(ctx, q.Target.Host, q.Target.Incarnation, q.Target.Stamp, q.Target.Session, q.Target.Box, q.Target.Enrollment) case "has-host": out.Found, err = s.Launches.HasHost(ctx, q.Target.Host) case "has-effects": out.Found, err = s.Launches.HasInstallationEffects(ctx, q.Target.Host, q.Target.Incarnation, q.Target.Stamp) case "inventory-version": out, err = s.Inventory.inventoryVersion(ctx, q.Target) out.V = 1 out.ID = q.ID case "inventory-read": out.Inventory, err = s.Inventory.ReadStopInventory(ctx, q.Target.Host) case "inventory-exact": out.Inventory, err = s.Inventory.ReadExactStopInventory(ctx, q.Target) default: return out, ErrUnsupported } return out, err } // journalPrune deletes one bounded batch of receipts past retention and raises // the persisted boundary to cover their requests. A request executed within // JournalRequestWindow of its issue time, so an unrecorded issue time is at // most that far past the receipt's creation. func journalPrune(ctx context.Context, tx *sql.Tx, rejectBefore int64, now time.Time) (int64, error) { cutoff := max(rejectBefore-int64(JournalRequestWindow), now.Add(-JournalReceiptRetention).UnixNano()) rows, err := tx.QueryContext(ctx, "SELECT rowid,created,issued FROM journal_requests WHERE created < ? ORDER BY created LIMIT ?", cutoff, journalPruneBatch) if err != nil { return rejectBefore, err } var ids []int64 covered := rejectBefore for rows.Next() { var id, created, issued int64 if err = rows.Scan(&id, &created, &issued); err != nil { rows.Close() return rejectBefore, err } if issued <= 0 { issued = created + int64(JournalRequestWindow) } ids = append(ids, id) covered = max(covered, issued) } if err = errors.Join(rows.Err(), rows.Close()); err != nil { return rejectBefore, err } if len(ids) == 0 { return rejectBefore, nil } if _, err = tx.ExecContext(ctx, "UPDATE journal_receipt_retention SET reject_before=? WHERE id=1", covered); err != nil { return rejectBefore, err } for _, id := range ids { if _, err = tx.ExecContext(ctx, "DELETE FROM journal_requests WHERE rowid=?", id); err != nil { return rejectBefore, err } } return covered, nil }