package host import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "io" "log" "os" "reflect" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/rootstate" ) // servingParkLedger runs behind root authorization and the journal transport. // Its callbacks are private implementation seams; requests cannot supply a // source, disposition, timestamp or completion. type servingParkLedger struct { historical bool state rootstate.Store fs servingParkFilesystem journal *hostbox.LaunchJournal now func() time.Time } type servingParkFile struct { Persistent servingParkPersistentFile `json:",omitzero"` Name string Present bool Device, Inode, Mount, Bytes uint64 Mode, Links uint32 Moved, DeleteIntent, Deleted bool } type servingParkSource struct { ParentPersistent servingParkPersistentDirectory `json:",omitzero"` Volume string `json:",omitempty"` Persistent servingParkPersistentDirectory `json:",omitzero"` BootID string `json:",omitempty"` Directory servingArtifactDirectory Parent servingArtifactDirectory Files [4]servingParkFile } type servingParkRecord struct { EventPersistent servingParkPersistentDirectory `json:",omitzero"` CollectAfter time.Time `json:",omitempty"` Namespace servingArtifactDirectory `json:",omitempty"` LegacyEvidence *servingLegacyEvidence `json:",omitempty"` Continuations []servingParkContinuation `json:",omitempty"` EventRemoveIntent, EventRemoved bool `json:",omitempty"` Version int Authorization servingParkAuthorization Source servingParkSource Receipts []hostguard.FenceReceipt Disposition servingParkDisposition Event servingArtifactDirectory Prepared, Parked, Complete, Deleting, Closed bool ParkedAt, ReapAfter time.Time Cause string } type servingParkContinuation struct { Authorization servingParkAuthorization Receipts []hostguard.FenceReceipt } func (r servingParkRecord) activeAuthorization() servingParkAuthorization { if len(r.Continuations) != 0 { return r.Continuations[len(r.Continuations)-1].Authorization } return r.Authorization } type servingLegacyConsumption struct { Collected bool `json:",omitempty"` LegacyEvidence *servingLegacyEvidence `json:",omitempty"` Receipts []hostguard.FenceReceipt `json:",omitempty"` Authorization servingParkAuthorization Source servingParkSource } type servingParkSlot struct { Version int Box, Host, ID, Pending, Last string } type servingParkFilesystem interface { // Capture validates a root-authenticated source before any slot is changed. Capture(servingParkAuthorization) (servingParkSource, error) Prepare(*servingParkRecord, func() error) error Move(*servingParkRecord, func() error) error Delete(*servingParkRecord, func() error) error CheckParked(*servingParkRecord) error } func servingParkKey(box, host string) string { sum := sha256.Sum256([]byte(box + "\x00" + host)) return hex.EncodeToString(sum[:]) } func servingParkDecode(body []byte, out any) error { d := json.NewDecoder(bytes.NewReader(body)) d.DisallowUnknownFields() if d.Decode(out) != nil || d.Decode(new(any)) != io.EOF { return hostbox.ErrChanged } return nil } func (l *servingParkLedger) write(name string, value any) error { body, err := json.Marshal(value) if err != nil { return err } return l.state.Write(name, body) } func (l *servingParkLedger) read(id string) (servingParkRecord, error) { r, err := l.readHistorical(id) if err == nil { err = l.bindRecord(&r, false) } if err == nil { err = validateServingParkRecord(r) } return r, err } func (l *servingParkLedger) readHistorical(id string) (servingParkRecord, error) { var r servingParkRecord if len(id) != 64 { return r, hostbox.ErrChanged } if _, err := hex.DecodeString(id); err != nil { return r, hostbox.ErrChanged } body, err := l.state.Read("retirement-" + id + ".json") if err != nil { return r, err } if err = servingParkDecode(body, &r); err != nil { return r, err } if err = validateServingParkRecord(r); err != nil || r.Authorization.ID != id { return r, hostbox.ErrChanged } return r, nil } func validateServingParkRecord(r servingParkRecord) error { if err := validateServingParkPersistentRecord(r); err != nil { return err } a := r.Authorization s := a.OriginalSuccessor if r.Version != 1 || a.ID != servingRetirementID(a.Predecessor, s, a.Legacy) || s.Host == "" || s.Box == "" || s.Incarnation == "" || s.Generation == "" || s.Region == "" || s.Location == "" || s.Stamp == 0 || s.Stamp != s.Session || a.Successor != s || a.Directory != r.Source.Directory || r.Source.Directory.Inode == 0 || r.Source.Directory.Device == 0 || r.Source.Directory.Mount == 0 || r.Source.Parent.Inode == 0 || r.Source.Parent.Device != r.Source.Directory.Device || r.Source.Parent.Mount != r.Source.Directory.Mount { return hostbox.ErrChanged } if a.Legacy { if a.Predecessor != (servingArtifactIdentity{}) || a.TagsFrom != 0 || r.Source.Files[1].Present { return hostbox.ErrChanged } } else { p := a.Predecessor if p.Host != s.Host || p.Box != s.Box || p.Region != s.Region || p.Incarnation != s.Incarnation || p.Location != s.Location || p.Generation == "" || p.Generation == s.Generation || p.Stamp == 0 || p.Stamp != p.Session || p.Stamp >= s.Stamp || a.TagsFrom != p.Stamp { return hostbox.ErrChanged } } if r.LegacyEvidence != nil && validateServingLegacyEvidence(a, r.LegacyEvidence) != nil { return hostbox.ErrChanged } noProcess := a.Legacy && r.LegacyEvidence != nil && r.LegacyEvidence.NoProcess != nil if (len(r.Receipts) == 0) != noProcess { return hostbox.ErrChanged } seen := make(map[string]bool) for _, receipt := range r.Receipts { t := receipt.Target if t.Host != s.Host || t.Box != s.Box || t.Region != s.Region || t.Incarnation != s.Incarnation || t.Stamp == 0 || t.Stamp >= s.Stamp || t.Session >= s.Session || t.Op != "" || t.Phase != "" || t.ObligationID != "" || t.FenceID == "" || seen[t.FenceID] || verifyRetirementReceipt(context.Background(), receipt) != nil { return hostbox.ErrChanged } if a.Legacy && r.LegacyEvidence != nil && t.Session != r.LegacyEvidence.Selector.Session { return hostbox.ErrChanged } if !a.Legacy && (t.Stamp != a.Predecessor.Stamp || t.Session != a.Predecessor.Session) { return hostbox.ErrChanged } seen[t.FenceID] = true } for i, suffix := range []string{"", ".gen", "-wal", "-shm"} { f := r.Source.Files[i] if f.Name != s.Host+".db"+suffix { return hostbox.ErrChanged } if f.Present { if f.Device != r.Source.Directory.Device || f.Mount != r.Source.Directory.Mount || f.Inode == 0 || f.Mode != 0100000 || f.Links != 1 { return hostbox.ErrChanged } } else if f.Device != 0 || f.Inode != 0 || f.Mount != 0 || f.Mode != 0 || f.Links != 0 || f.Bytes != 0 || f.Moved || f.DeleteIntent || f.Deleted { return hostbox.ErrChanged } if f.Deleted && (!f.DeleteIntent || !r.Deleting) || f.DeleteIntent && !f.Moved { return hostbox.ErrChanged } } if r.Disposition != servingParkRetain && r.Disposition != servingParkDiscard && r.Disposition != servingParkArtifactFree { return hostbox.ErrChanged } if (r.Disposition == servingParkArtifactFree) == r.Source.Files[0].Present { return hostbox.ErrChanged } if r.Prepared && (r.Event.Inode == 0 || r.Event.Device != r.Source.Directory.Device || r.Event.Mount != r.Source.Directory.Mount) { return hostbox.ErrChanged } if r.Namespace != (servingArtifactDirectory{}) && (!r.Prepared || r.Namespace.Inode == 0 || r.Namespace.Device != r.Source.Parent.Device || r.Namespace.Mount != r.Source.Parent.Mount || r.Namespace == r.Source.Parent || r.Namespace == r.Event) { return hostbox.ErrChanged } if r.Parked { if !r.Prepared || r.ParkedAt.IsZero() || !r.ReapAfter.Equal(r.ParkedAt.Add(7*24*time.Hour)) { return hostbox.ErrChanged } for _, f := range r.Source.Files { if f.Present && !f.Moved { return hostbox.ErrChanged } } } else if !r.ParkedAt.IsZero() || !r.ReapAfter.IsZero() || r.Complete { return hostbox.ErrChanged } if r.Deleting && (r.Cause != "discard" && r.Cause != "replacement" && r.Cause != "expiry" && r.Cause != "pressure") { return hostbox.ErrChanged } if r.Deleting && !r.Parked { return hostbox.ErrChanged } if r.EventRemoved && !r.EventRemoveIntent || r.EventRemoveIntent && !r.Deleting { return hostbox.ErrChanged } if r.EventRemoveIntent { for _, item := range r.Source.Files { if item.Present && !item.Deleted { return hostbox.ErrChanged } } } if r.Closed && !r.Deleting { return hostbox.ErrChanged } if !r.CollectAfter.IsZero() && (!r.Complete || !r.Closed || !r.EventRemoved || r.Namespace == (servingArtifactDirectory{}) || r.CollectAfter.Before(r.ParkedAt.Add(hostbox.JournalReceiptRetention))) { return hostbox.ErrChanged } if r.Closed { for _, item := range r.Source.Files { if item.Present && !item.Deleted { return hostbox.ErrChanged } } } if r.Complete && r.Disposition != servingParkRetain && !r.Closed { return hostbox.ErrChanged } previous := a if len(r.Continuations) > 32 { return hostbox.ErrChanged } for _, continuation := range r.Continuations { next := continuation.Authorization want := previous want.Successor = next.Successor want.TagsFrom = next.TagsFrom identity := previous.Successor identity.Stamp, identity.Session, identity.Generation = next.Successor.Stamp, next.Successor.Session, next.Successor.Generation if next != want || next.Successor != identity || identity.Stamp <= previous.Successor.Stamp || identity.Session != identity.Stamp || identity.Generation == "" || identity.Generation == previous.Successor.Generation || (next.TagsFrom != previous.TagsFrom && next.TagsFrom != previous.Successor.Stamp && next.TagsFrom != next.Successor.Stamp) || len(continuation.Receipts) == 0 || !r.Parked { return hostbox.ErrChanged } for _, receipt := range continuation.Receipts { t := receipt.Target p := previous.Successor if t.Host != p.Host || t.Box != p.Box || t.Region != p.Region || t.Incarnation != p.Incarnation || t.Stamp != p.Stamp || t.Session != p.Session || t.Op != "" || t.Phase != "" || t.ObligationID != "" || verifyRetirementReceipt(context.Background(), receipt) != nil { return hostbox.ErrChanged } } previous = next } return nil } func (l *servingParkLedger) save(r servingParkRecord) error { if err := l.bindRecord(&r, true); err != nil { return err } if err := validateServingParkRecord(r); err != nil { return err } return l.write("retirement-"+r.Authorization.ID+".json", r) } func (l *servingParkLedger) slot(a servingParkAuthorization) (servingParkSlot, error) { s := servingParkSlot{Version: 1, Box: a.Successor.Box, Host: a.Successor.Host} b, err := l.state.Read("slot-" + servingParkKey(s.Box, s.Host) + ".json") if errors.Is(err, os.ErrNotExist) { return s, nil } if err != nil { return s, err } var saved servingParkSlot if servingParkDecode(b, &saved) != nil || saved.Version != 1 || saved.Box != s.Box || saved.Host != s.Host { return s, hostbox.ErrChanged } if saved.ID != "" { r, err := l.readHistorical(saved.ID) if err != nil || r.Disposition != servingParkRetain || r.Authorization.Successor.Box != s.Box || r.Authorization.Successor.Host != s.Host { return s, hostbox.ErrChanged } } if saved.Pending != "" { r, err := l.read(saved.Pending) if err != nil || r.Authorization.Successor.Box != s.Box || r.Authorization.Successor.Host != s.Host { return s, hostbox.ErrChanged } } if saved.Last != "" { r, err := l.readHistorical(saved.Last) if err != nil || r.Authorization.Successor.Box != s.Box || r.Authorization.Successor.Host != s.Host { return s, hostbox.ErrChanged } } return saved, nil } // withNamespace precedes the root host guard, slot lock and record lock. The // authority callback must acquire the host guard and independently authenticate // the claim, receipts, state directory and launch history while it is held. func (l *servingParkLedger) withNamespace(ctx context.Context, fn func() error) error { if ctx == nil || l == nil || l.state == nil || l.fs == nil || l.now == nil || fn == nil { return hostbox.ErrUnsupported } unlock, err := l.state.Lock(ctx, "namespace.lock") if err != nil { return err } defer unlock() if err = ctx.Err(); err != nil { return err } return fn() } // parkLocked runs inside independently verified root host authority. Fresh // capture is never substituted for the immutable source on a replay. func (l *servingParkLedger) parkLocked(ctx context.Context, a servingParkAuthorization, receipts []hostguard.FenceReceipt) (servingParkRecord, error) { return l.parkWithEvidenceLocked(ctx, a, receipts, nil) } // capture validates the full source before namespace recovery may write any // authority. The ledger repeats it before saving a fresh retirement intent. func (l *servingParkLedger) capture(ctx context.Context, a servingParkAuthorization, receipts []hostguard.FenceReceipt, legacy *servingLegacyEvidence) (servingParkRecord, error) { var zero servingParkRecord source, e := l.fs.Capture(a) if e != nil { return zero, e } disposition := servingParkArtifactFree if a.Legacy { if source.Files[1].Present { return zero, hostbox.ErrChanged } if source.Files[0].Present { disposition = servingParkRetain } } else { disposition, e = servingGenerationParkDisposition(ctx, l.journal, a.Predecessor, receipts, source.Files[0].Present) if e != nil { return zero, e } } r := servingParkRecord{Version: 1, Authorization: a, Source: source, Receipts: receipts, LegacyEvidence: legacy, Disposition: disposition} if legacy != nil && legacy.Selector.Empty { for _, f := range source.Files { if f.Present { return zero, hostbox.ErrChanged } } } if e = validateServingParkRecord(r); e != nil { return zero, e } return r, nil } func (l *servingParkLedger) parkWithEvidenceLocked(ctx context.Context, a servingParkAuthorization, receipts []hostguard.FenceReceipt, legacy *servingLegacyEvidence) (servingParkRecord, error) { var zero servingParkRecord if err := ctx.Err(); err != nil { return zero, err } unlock, err := l.state.Lock(ctx, "slot-"+servingParkKey(a.Successor.Box, a.Successor.Host)+".lock") if err != nil { return zero, err } defer unlock() slot, err := l.slot(a) if err != nil { return zero, err } if slot.Pending != "" && slot.Pending != a.ID { return zero, hostbox.ErrChanged } unlockRecord, err := l.state.Lock(ctx, "retirement-"+a.ID+".lock") if err != nil { return zero, err } defer unlockRecord() r, err := l.read(a.ID) if errors.Is(err, os.ErrNotExist) { // Last survives collection. An old request cannot reconstruct a removed // intent from a restored source in the same incarnation. The outer root // guard independently requires the current incarnation and successor. if slot.Last != "" { last, e := l.readHistorical(slot.Last) if e != nil { return zero, e } prior := last.activeAuthorization().Successor if prior.Incarnation == a.Successor.Incarnation && a.Successor.Stamp <= prior.Stamp { return zero, hostbox.ErrChanged } } captured, e := l.capture(ctx, a, receipts, legacy) if e != nil { return zero, e } r = captured source := r.Source // Consumption survives slot replacement, expiry and park tombstones. if a.Legacy { key := "legacy-" + servingParkKey(a.Successor.Box, a.Successor.Host+"\x00"+a.Successor.Incarnation) + ".json" var consumed servingLegacyConsumption want := servingLegacyConsumption{Authorization: a, Source: source, Receipts: receipts, LegacyEvidence: legacy} body, e := l.state.Read(key) if e == nil { if servingParkDecode(body, &consumed) != nil || !sameServingParkSource(consumed.Source, want.Source) { return zero, hostbox.ErrChanged } // A lost old-format intent keeps its original consumption // evidence. Never infer historical handles from current bytes. want.Source = consumed.Source if !reflect.DeepEqual(consumed, want) { return zero, hostbox.ErrChanged } r.Source = consumed.Source } else if !errors.Is(e, os.ErrNotExist) { return zero, e } if e = l.write(key, want); e != nil { return zero, e } } if e = l.save(r); e != nil { return zero, e } } else if err != nil { return zero, err } if r.Authorization != a || !sameServingLegacyEvidence(r.LegacyEvidence, legacy) || !sameServingRootStops(r.Receipts, receipts) { return zero, hostbox.ErrChanged } if r.Closed || r.Complete { return r, nil } if slot.Pending == "" { slot.Pending = a.ID slot.Last = a.ID if err = l.write("slot-"+servingParkKey(slot.Box, slot.Host)+".json", slot); err != nil { return zero, err } } if r.Deleting { if r.Disposition == servingParkRetain || r.Cause != "discard" { return zero, hostbox.ErrChanged } if err = l.deleteLocked(ctx, &r, "discard"); err != nil { return zero, err } return r, nil } if r.Disposition == servingParkRetain && slot.ID != a.ID { // A completed replay can never reacquire a slot it has lost. if r.Parked || r.Complete { return zero, hostbox.ErrChanged } fresh, e := l.fs.Capture(a) if e != nil || !sameServingParkSource(r.Source, fresh) { return zero, hostbox.ErrChanged } if slot.ID != "" { if e = l.replaceLocked(ctx, slot.ID); e != nil { return zero, e } } slot.ID = a.ID if err = l.write("slot-"+servingParkKey(slot.Box, slot.Host)+".json", slot); err != nil { return zero, err } } save := func() error { if err := ctx.Err(); err != nil { return err } return l.save(r) } if err = l.fs.Prepare(&r, save); err != nil { return zero, err } if err = l.fs.Move(&r, save); err != nil { return zero, err } if !r.Parked { r.Parked, r.ParkedAt = true, l.now().UTC() r.ReapAfter = r.ParkedAt.Add(7 * 24 * time.Hour) if err = save(); err != nil { return zero, err } } // Even discarded restores are first isolated by recorded rename into the // root-owned event. A raced source replacement is never unlinked in place. if r.Disposition != servingParkRetain && !r.Closed { if err = l.deleteLocked(ctx, &r, "discard"); err != nil { return zero, err } } return r, nil } // completeLocked follows a fresh root check of the successor's paired tags. // Its caller holds namespace and host authority, never an agent Work lock. func (l *servingParkLedger) completeLocked(ctx context.Context, id string, a servingArtifactIdentity) error { unlock, err := l.state.Lock(ctx, "slot-"+servingParkKey(a.Box, a.Host)+".lock") if err != nil { return err } defer unlock() r, err := l.read(id) if err != nil { return err } if r.activeAuthorization().Successor != a || !r.Parked { return hostbox.ErrChanged } unlockRecord, err := l.state.Lock(ctx, "retirement-"+id+".lock") if err != nil { return err } defer unlockRecord() slot, err := l.slot(r.Authorization) if err != nil { return err } if !r.Complete { if slot.Pending != id { return hostbox.ErrChanged } if err = l.fs.CheckParked(&r); err != nil { return err } if r.Disposition == servingParkRetain { if slot.ID != id || r.Closed || r.Deleting { return hostbox.ErrChanged } } else if !r.Closed { return hostbox.ErrChanged } if err = ctx.Err(); err != nil { return err } r.Complete = true if err = l.save(r); err != nil { return err } } if slot.Pending == id { slot.Pending = "" return l.write("slot-"+servingParkKey(slot.Box, slot.Host)+".json", slot) } return nil } // continueLocked records permission from a later stopped successor without // changing the original retirement, source identities or retention deadline. // Root has already rechecked the original and every continuation stop. func (l *servingParkLedger) continueLocked(ctx context.Context, id string, a servingParkAuthorization, receipts []hostguard.FenceReceipt) (servingParkRecord, error) { var zero servingParkRecord unlock, err := l.state.Lock(ctx, "slot-"+servingParkKey(a.Successor.Box, a.Successor.Host)+".lock") if err != nil { return zero, err } defer unlock() unlockRecord, err := l.state.Lock(ctx, "retirement-"+id+".lock") if err != nil { return zero, err } defer unlockRecord() r, err := l.read(id) if err != nil { return zero, err } if r.activeAuthorization() == a { return r, nil } slot, err := l.slot(a) if err != nil || slot.Pending != id || r.Complete { return zero, hostbox.ErrChanged } if err = l.fs.CheckParked(&r); err != nil { return zero, err } r.Continuations = append(r.Continuations, servingParkContinuation{Authorization: a, Receipts: receipts}) if err = ctx.Err(); err != nil { return zero, err } if err = l.save(r); err != nil { return zero, err } return r, nil } func (l *servingParkLedger) deleteLocked(ctx context.Context, r *servingParkRecord, cause string) error { if r.Closed { return nil } if cause != "discard" && (!r.Complete || r.Disposition != servingParkRetain) { return hostbox.ErrChanged } if cause == "discard" && r.Disposition == servingParkRetain { return hostbox.ErrChanged } if r.Deleting && r.Cause != cause { return hostbox.ErrChanged } save := func() error { if err := ctx.Err(); err != nil { return err } return l.save(*r) } r.Deleting, r.Cause = true, cause if err := save(); err != nil { return err } if err := l.fs.Delete(r, save); err != nil { return err } r.Closed = true return save() } func (l *servingParkLedger) replaceLocked(ctx context.Context, id string) error { unlock, err := l.state.Lock(ctx, "retirement-"+id+".lock") if err != nil { return err } defer unlock() old, err := l.readHistorical(id) if err != nil || !old.Complete { return hostbox.ErrChanged } if old.Closed { return nil } cause := "replacement" if old.Deleting { cause = old.Cause } current, err := l.read(id) if err == nil { err = l.deleteLocked(ctx, ¤t, cause) } if err != nil { log.Printf("Host %q: serving database retirement %q retained: %s.", old.Authorization.Successor.Host, id, err) } return ctx.Err() } func sameServingRootStops(a, b []hostguard.FenceReceipt) bool { if len(a) != len(b) { return false } for i := range a { if a[i].Target != b[i].Target || !a[i].EvidenceAt.Equal(b[i].EvidenceAt) || !bytes.Equal(a[i].Bytes, b[i].Bytes) { return false } } return true }