package hostops import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "os" "reflect" "strconv" "strings" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostevidence" "github.com/carlosframework/platform/internal/hostfence" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/termfence" ) type StopReceipt struct { RequestLimitNS int64 `json:"request_limit_ns"` Region string `json:"region,omitempty"` ClaimETag string `json:"claim_etag,omitempty"` ObservedBootID string `json:"observed_boot_id,omitempty"` V int `json:"v"` ID string `json:"id"` Host string `json:"host"` Incarnation string `json:"incarnation"` InstallationStamp uint64 `json:"installation_stamp"` WriterSession uint64 `json:"writer_session"` OperationID string `json:"operation_id"` Phase string `json:"phase"` ObligationID string `json:"obligation_id"` Box string `json:"box"` Enrollment string `json:"enrollment"` BootID string `json:"boot_id"` ControllerID string `json:"controller_id"` ClaimGen uint64 `json:"claim_gen"` DirectoryRevision string `json:"directory_revision"` Kind string `json:"kind"` EvidenceAt time.Time `json:"evidence_at"` FenceID string `json:"fence_id"` InventorySHA256 string `json:"inventory_sha256"` LocalJournalSHA256 string `json:"local_journal_sha256"` ProducerDrainDigest string `json:"producer_drain_digest"` CgroupIdentities []string `json:"cgroup_identities"` UnitInvocationIDs []string `json:"unit_invocation_ids"` } func evidenceDigest(raw []byte) string { sum := sha256.Sum256(raw); return hex.EncodeToString(sum[:]) } func digestOK(s string) bool { raw, err := hex.DecodeString(s) return err == nil && len(raw) == 32 && s == strings.ToLower(s) } func (a ProductionAuthority) signedRoster(ctx context.Context) (hostactors.Roster, []byte, error) { raw, _, err := a.Store.Get(ctx, hostactors.Key) if err != nil { return hostactors.Roster{}, nil, err } var roster hostactors.Roster if release.VerifyRootEnvelope(a.Directory.PublicKey, raw, &roster) != nil || roster.Validate(a.Directory.Bucket) != nil || roster.RequestLimitNS != int64(hostbox.RequestLimit) { return roster, nil, ErrContract } return roster, raw, nil } func (a ProductionAuthority) verifyEvidence(ctx context.Context, obs hostclaim.Observed, obID, opID string, ref hostclaim.RetirementEvidenceRef, bind bool) (time.Time, error) { if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || ref.Box == "" || ref.Enrollment == "" || ref.OperationID != opID || ref.ObligationID != obID || ref.EvidenceAt.IsZero() { return time.Time{}, ErrIdentity } raw, _, err := a.Store.Get(ctx, ref.Key) if err != nil { return time.Time{}, err } if evidenceDigest(raw) != ref.SHA256 { return time.Time{}, ErrIdentity } if ref.Kind == "termination" { fence := hostclaim.FenceRef{Enrollment: ref.Enrollment, Key: ref.Key, SHA256: ref.SHA256} if err = hostclaim.VerifyFenceForBox(ctx, a.Store, hostfence.ClaimFenceAuthority{Directory: a.Directory, Now: a.Now}, fence, ref.Box); err != nil { return time.Time{}, err } var record termfence.Record if json.Unmarshal(raw, &record) != nil { return time.Time{}, ErrIdentity } at, err := time.Parse(time.RFC3339Nano, record.ObservedAt) if err != nil || !at.Equal(ref.EvidenceAt) { return time.Time{}, ErrIdentity } return at, nil } if ref.Kind != "receipt" { return time.Time{}, ErrIdentity } var roster hostactors.Roster if release.VerifyRootEnvelope(a.Directory.PublicKey, ref.SignedRoster, &roster) != nil || roster.Validate(a.Directory.Bucket) != nil || roster.RequestLimitNS != int64(hostbox.RequestLimit) { return time.Time{}, ErrIdentity } actor, ok := roster.Boxes[ref.Box] if !ok || actor.Enrollment != ref.Enrollment { return time.Time{}, ErrIdentity } signature, _, err := a.Store.Get(ctx, ref.Key+".sig") if err != nil { return time.Time{}, err } if evidenceDigest(signature) != ref.SignatureSHA256 || hostevidence.Verify(actor.CensusPublicKey, ref.Key, raw, signature) != nil { return time.Time{}, ErrIdentity } var r StopReceipt if decode(raw, &r) != nil { return time.Time{}, ErrIdentity } h := obs.Claim.Holder prefix := fmt.Sprintf("control/hosts/%s/evidence/%s/%s/", obs.Host, opID, ref.Box) revision, err := strconv.ParseUint(r.DirectoryRevision, 10, 64) if err != nil || revision == 0 || revision > roster.Revision || r.V != 1 || !safeSegment(r.ID) || ref.Key != prefix+r.ID+".json" || r.Host != obs.Host || r.Incarnation != h.Incarnation || r.OperationID != opID || r.Box != ref.Box || r.Enrollment != ref.Enrollment || r.ClaimGen == 0 || r.ClaimGen > obs.Claim.Gen || r.BootID == "" || r.ControllerID == "" || r.FenceID == "" || r.Phase == "" || r.ObligationID == "" || !r.EvidenceAt.Equal(ref.EvidenceAt) || r.EvidenceAt.After(a.Now().Add(5*time.Second)) || r.RequestLimitNS != int64(hostbox.RequestLimit) || !digestOK(r.InventorySHA256) || !digestOK(r.LocalJournalSHA256) || !digestOK(r.ProducerDrainDigest) { return time.Time{}, ErrIdentity } if ref.SourceObligationID == "" || r.ObligationID != ref.SourceObligationID { return time.Time{}, ErrIdentity } if r.Kind != "stopped" && r.Kind != "empty" { return time.Time{}, ErrIdentity } if r.InstallationStamp != ref.InstallationStamp || r.WriterSession != ref.WriterSession { return time.Time{}, ErrIdentity } if bind && (ref.Box == h.WriterBox || ref.Box == obs.Claim.Op.Exec) { if r.Kind != "stopped" || r.InstallationStamp != h.Stamp || r.WriterSession != h.Session || len(r.CgroupIdentities) == 0 { return time.Time{}, ErrIdentity } } else if bind && r.Kind == "stopped" { matched := false if inventory, ok := obs.Claim.Inventory[ref.Box]; ok { for _, stamp := range inventory.Paths { if stamp == r.InstallationStamp { matched = true } } } if !matched { return time.Time{}, ErrPending } } return r.EvidenceAt, nil } func (a ProductionAuthority) chooseEvidence(ctx context.Context, obs hostclaim.Observed, obID, opID, box string) (hostclaim.RetirementEvidenceRef, error) { roster, signed, err := a.signedRoster(ctx) if err != nil { return hostclaim.RetirementEvidenceRef{}, err } actor, ok := roster.Boxes[box] if !ok { return hostclaim.RetirementEvidenceRef{}, ErrIdentity } prefix := fmt.Sprintf("control/hosts/%s/evidence/%s/%s/", obs.Host, opID, box) keys, err := a.Store.List(ctx, prefix) if err != nil { return hostclaim.RetirementEvidenceRef{}, err } var chosen hostclaim.RetirementEvidenceRef for _, key := range keys { if !strings.HasPrefix(key, prefix) || strings.Contains(strings.TrimPrefix(key, prefix), "/") { return chosen, ErrIdentity } if !strings.HasSuffix(key, ".json") || strings.HasSuffix(key, "-retirement.json") { continue } raw, _, err := a.Store.Get(ctx, key) if err != nil { return chosen, err } var receipt StopReceipt if json.Unmarshal(raw, &receipt) != nil { return chosen, ErrIdentity } if receipt.OperationID != opID || receipt.Host != obs.Host || receipt.Box != box || receipt.Incarnation != obs.Claim.Holder.Incarnation || receipt.Enrollment != actor.Enrollment { continue } signature, _, err := a.Store.Get(ctx, key+".sig") if errors.Is(err, os.ErrNotExist) { continue } if err != nil { return chosen, err } ref := hostclaim.RetirementEvidenceRef{SourceObligationID: receipt.ObligationID, InstallationStamp: receipt.InstallationStamp, WriterSession: receipt.WriterSession, Kind: "receipt", Box: box, Enrollment: actor.Enrollment, OperationID: opID, ObligationID: obID, Key: key, SHA256: evidenceDigest(raw), SignatureSHA256: evidenceDigest(signature), SignedRoster: signed, EvidenceAt: receipt.EvidenceAt} if _, err = a.verifyEvidence(ctx, obs, obID, opID, ref, true); err != nil { return chosen, err } // Earliest complete proof is stable once cited. A retry never replaces it // with a newly published receipt carrying a later wall-clock time. if chosen.Key == "" || ref.EvidenceAt.Before(chosen.EvidenceAt) || (ref.EvidenceAt.Equal(chosen.EvidenceAt) && ref.Key < chosen.Key) { chosen = ref } } if chosen.Key != "" { return chosen, nil } if opID != obs.Claim.Op.ID { return chosen, ErrPending } var fence hostclaim.FenceRef if json.Unmarshal([]byte(obs.Claim.Op.Journal["fence."+box]), &fence) != nil { return chosen, ErrPending } if err = hostclaim.VerifyFenceForBox(ctx, a.Store, hostfence.ClaimFenceAuthority{Directory: a.Directory, Now: a.Now}, fence, box); err != nil { return chosen, err } raw, _, err := a.Store.Get(ctx, fence.Key) if err != nil { return chosen, err } if evidenceDigest(raw) != fence.SHA256 { return chosen, ErrIdentity } var record termfence.Record if json.Unmarshal(raw, &record) != nil { return chosen, ErrIdentity } at, err := time.Parse(time.RFC3339Nano, record.ObservedAt) if err != nil { return chosen, err } chosen = hostclaim.RetirementEvidenceRef{Kind: "termination", Box: box, Enrollment: fence.Enrollment, OperationID: opID, ObligationID: obID, Key: fence.Key, SHA256: fence.SHA256, EvidenceAt: at} return chosen, nil } // Discovery retains its original writer census even after local chores finish. // The serving box remains live and is excluded only for that persisted census. func (a ProductionAuthority) evidenceBoxes(ctx context.Context, obs hostclaim.Observed, opID string) ([]string, error) { if obs.Claim != nil && obs.Claim.Touched.DiscoveryOp == opID && opID != "" { if !obs.Claim.Touched.DiscoveryBound { return nil, ErrPending } return append([]string(nil), obs.Claim.Touched.DiscoveryBoxes...), nil } _, boxes, err := a.Inventory(ctx, obs) return boxes, err } func (a ProductionAuthority) collectEvidence(ctx context.Context, obs hostclaim.Observed, obID, opID string) (map[string]hostclaim.RetirementEvidenceRef, time.Time, error) { boxes, err := a.evidenceBoxes(ctx, obs, opID) if err != nil { return nil, time.Time{}, err } if len(boxes) == 0 { return nil, time.Time{}, ErrPending } refs := map[string]hostclaim.RetirementEvidenceRef{} var latest time.Time for _, box := range boxes { ref, err := a.chooseEvidence(ctx, obs, obID, opID, box) if err != nil { return nil, time.Time{}, err } refs[box] = ref if ref.EvidenceAt.After(latest) { latest = ref.EvidenceAt } } return refs, latest, nil } func (a ProductionAuthority) verifyReferences(ctx context.Context, obs hostclaim.Observed, obID, opID string, refs map[string]hostclaim.RetirementEvidenceRef) (time.Time, error) { boxes, err := a.evidenceBoxes(ctx, obs, opID) if err != nil { return time.Time{}, err } if len(boxes) == 0 || len(boxes) != len(refs) { return time.Time{}, ErrPending } var latest time.Time for _, box := range boxes { ref, ok := refs[box] if !ok || ref.Box != box { return time.Time{}, ErrIdentity } at, err := a.verifyEvidence(ctx, obs, obID, opID, ref, false) if err != nil { return time.Time{}, err } if at.After(latest) { latest = at } } return latest, nil } func (a ProductionAuthority) StopBeforeTeardown(ctx context.Context, obs hostclaim.Observed) error { if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil { return ErrIdentity } if raw := obs.Claim.Op.Journal["stop_evidence"]; raw != "" { var refs map[string]hostclaim.RetirementEvidenceRef if json.Unmarshal([]byte(raw), &refs) != nil { return ErrIdentity } _, err := a.verifyReferences(ctx, obs, "", obs.Claim.Op.ID, refs) return err } refs, _, err := a.collectEvidence(ctx, obs, "", obs.Claim.Op.ID) if err != nil { return err } raw, err := json.Marshal(refs) if err != nil { return err } _, err = hostclaim.Update(ctx, a.Store, obs, func(c *hostclaim.Claim) error { if c.Op.ID != obs.Claim.Op.ID || c.Holder == nil || c.Holder.Incarnation != obs.Claim.Holder.Incarnation { return ErrIdentity } if c.Op.Journal == nil { c.Op.Journal = map[string]string{} } c.Op.Journal["stop_evidence"] = string(raw) return nil }) if err != nil { return err } // Force the caller to re-read the claim carrying the durable references. return ErrPending } type RetirementEvidence struct{ Authority ProductionAuthority } func (e RetirementEvidence) VerifyObligation(ctx context.Context, obs hostclaim.Observed, ob hostclaim.Obligation) (hostclaim.ParkScope, error) { if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || ob.Op == "" || ob.Region == "" || ob.Owner != "region:"+ob.Region || (ob.Kind != "park-replica" && ob.Kind != "park-generation") { return hostclaim.ParkScope{}, ErrIdentity } if ob.Park != nil { if err := e.VerifyBound(ctx, obs, ob); err != nil { return hostclaim.ParkScope{}, err } return *ob.Park, nil } if ob.ParkTarget == "" { return hostclaim.ParkScope{}, ErrPending } refs, _, err := e.Authority.collectEvidence(ctx, obs, ob.ID, ob.Op) if err != nil { return hostclaim.ParkScope{}, err } scope, err := retirementScope(obs.Host, obs.Claim.Holder.Incarnation, ob.ParkTarget, refs) if err != nil { return scope, err } _, err = hostclaim.Update(ctx, e.Authority.Store, obs, func(c *hostclaim.Claim) error { for i := range c.Obligations { current := &c.Obligations[i] if current.ID == ob.ID && current.Op == ob.Op { if !reflect.DeepEqual(*current, ob) { return ErrPending } current.EvidenceRefs = refs current.Park = &scope return nil } } return ErrPending }) return scope, err } func (e RetirementEvidence) VerifyBound(ctx context.Context, obs hostclaim.Observed, ob hostclaim.Obligation) error { if ob.Park == nil || ob.ParkTarget == "" || ob.Park.Location != ob.ParkTarget || ob.Park.RequestLimit != hostbox.RequestLimit || obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || ob.Park.Incarnation != obs.Claim.Holder.Incarnation { return ErrIdentity } latest, err := e.Authority.verifyReferences(ctx, obs, ob.ID, ob.Op, ob.EvidenceRefs) if err != nil { return err } if !latest.Equal(ob.Park.EvidenceAt) { return ErrIdentity } return ob.Park.Validate(obs.Host) } func (e RetirementEvidence) VerifyRetirement(context.Context, hostclaim.Observed, []lease.ObjectVersion) (string, time.Time, error) { // At-rest/holderless residue needs Plan6b's archived-incarnation census. // Current holder evidence does not authorize another generation's sweep. return "", time.Time{}, ErrPending } type combinedRegionalEvidence struct { RetirementEvidence Previous hostbox.RegionalEvidence } func (e combinedRegionalEvidence) VerifyObligation(ctx context.Context, obs hostclaim.Observed, ob hostclaim.Obligation) (hostclaim.ParkScope, error) { // Marked retirement/discovery work must never borrow the older protocol. if ob.ParkTarget != "" || len(ob.EvidenceRefs) != 0 { return e.RetirementEvidence.VerifyObligation(ctx, obs, ob) } previous, ok := e.Previous.(hostbox.RegionalObligationEvidence) if !ok || previous == nil { return hostclaim.ParkScope{}, ErrContract } return previous.VerifyObligation(ctx, obs, ob) } func (e combinedRegionalEvidence) VerifyRetirement(ctx context.Context, obs hostclaim.Observed, versions []lease.ObjectVersion) (string, time.Time, error) { if e.Previous == nil { return "", time.Time{}, ErrPending } return e.Previous.VerifyRetirement(ctx, obs, versions) } func NewClaimRegionalWorker(regional hostbox.Regional, enrollment hostbox.RegionalEnrollment, authority ProductionAuthority, prior ...hostbox.RegionalEvidence) (hostbox.RegionalWorker, error) { if len(prior) > 1 || (len(prior) == 1 && prior[0] == nil) { return hostbox.RegionalWorker{}, ErrContract } var original hostbox.RegionalEvidence if len(prior) == 1 { original = prior[0] } if enrollment == nil || regional.Owner == nil || regional.Engine.Authorize == nil || regional.Engine.Leases == nil || regional.Engine.View == nil || regional.Engine.Claims == nil || authority.Store == nil || authority.Now == nil { return hostbox.RegionalWorker{}, ErrContract } switch regional.Engine.Store.(type) { case hostbox.RequestStore, *hostbox.RequestStore: default: return hostbox.RegionalWorker{}, ErrContract } provider := RetirementEvidence{Authority: authority} previous := regional.Engine.Authorize regional.Engine.Authorize = func(ctx context.Context, obs hostclaim.Observed, ob hostclaim.Obligation) error { if err := previous(ctx, obs, ob); err != nil { return err } // A pre-bound move/sweep owns its original evidence protocol. The // caller's mandatory authorization above revalidates that protocol. // Never route a newly produced retirement chore through this branch. if ob.Park != nil && ob.ParkTarget == "" && len(ob.EvidenceRefs) == 0 { if ob.Op == "" || ob.Park.RequestLimit != hostbox.RequestLimit { return ErrIdentity } return ob.Park.Validate(obs.Host) } return provider.VerifyBound(ctx, obs, ob) } return hostbox.RegionalWorker{Regional: regional, Enrollment: enrollment, Evidence: combinedRegionalEvidence{RetirementEvidence: provider, Previous: original}}, nil }