package hostresolve import ( "context" "crypto/rand" "encoding/hex" "encoding/json" "errors" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfence" "github.com/carlosframework/platform/internal/release" "math" "os" "reflect" "slices" "time" ) var ErrResolve = errors.New("hostresolve: evidence or operation does not authorize resolution") type Service struct { Directory hostactors.Directory Signer release.Signer Now func() time.Time Owners map[string]OperationOwner } type Inspection struct { Challenge json.RawMessage `json:"challenge"` Details hostfence.Challenge `json:"details"` } func nonce() (string, error) { var b [16]byte _, err := rand.Read(b[:]) return hex.EncodeToString(b[:]), err } func (s Service) record(ctx context.Context, c *hostclaim.Claim) (hostfence.RecordObservation, error) { _, etag, err := s.Directory.Store.Get(ctx, c.Holder.Key) if errors.Is(err, os.ErrNotExist) { return hostfence.RecordObservation{Key: c.Holder.Key, CommitState: "absent"}, nil } if err != nil { return hostfence.RecordObservation{}, err } return hostfence.RecordObservation{Key: c.Holder.Key, ETag: etag, CommitState: "present"}, nil } // effects implements only the owners present on the Plan 8 baseline. Moves' // target-loss policy must come from the moves owner, never from an HTTP flag. func effects(c *hostclaim.Claim, box string) (hostfence.Consequences, hostclaim.LossConsequences, error) { var shown hostfence.Consequences var actual hostclaim.LossConsequences if c == nil || c.Holder == nil { return shown, actual, ErrResolve } actual.RecoveryIntent = json.RawMessage(`{"kind":"continue-parking"}`) switch c.Op.Kind { case hostclaim.KindServing: shown.Transition = "destroyed-local-only" actual.RecoveryIntent = json.RawMessage(`{"kind":"destroyed-local-only"}`) for _, o := range c.Obligations { if o.Kind == "park-local" && o.Box == box { ref := hostclaim.ObligationRef{ID: o.ID, Op: o.Op, Retirement: o.Retirement} shown.DischargeLocal = append(shown.DischargeLocal, ref) actual.DischargeLocal = append(actual.DischargeLocal, ref) key := o.Op + "/" + o.ID + "/" + o.Retirement shown.DestroyedStorageAssertion = append(shown.DestroyedStorageAssertion, key) actual.DestroyedStorage = append(actual.DestroyedStorage, key) } } if len(actual.DischargeLocal) == 0 { return shown, actual, ErrResolve } case hostclaim.KindDeleting, hostclaim.KindParking: if c.Op.ID == "" || (!slices.Contains(c.Op.Participants, box) && c.Op.Exec != box) { return shown, actual, ErrResolve } shown.Transition = "continue-parking" shown.PossibleWriteLoss = true shown.PossibleTimerLoss = true actual.RevokeExecution = c.Op.Exec == box actual.RemoveParticipation = slices.Contains(c.Op.Participants, box) actual.WritesPotentiallyLost = true actual.TimersPotentiallyLost = true for _, o := range c.Obligations { if (o.Kind == "park-replica" || o.Kind == "park-generation") && o.Owner == "box:"+box { actual.ReassignRegional = append(actual.ReassignRegional, hostclaim.ObligationRef{ID: o.ID, Op: o.Op, Retirement: o.Retirement}) shown.Chores = append(shown.Chores, o.Op+"/"+o.ID+"/"+o.Retirement) } } default: return shown, actual, ErrResolve } return shown, actual, nil } func (s Service) Inspect(ctx context.Context, actor, host, name string) (Inspection, error) { ref, b, err := s.Directory.Resolve(ctx, name) if err != nil { return Inspection{}, err } if b.State == "retired" { return Inspection{}, ErrResolve } o, err := hostclaim.Read(ctx, s.Directory.Store, host) if err != nil { return Inspection{}, err } if o.Absent || o.Claim.Holder == nil { return Inspection{}, ErrResolve } policy, err := s.operationOwner(o.Claim.Op.Kind) if err != nil { return Inspection{}, err } record, err := s.record(ctx, o.Claim) if err != nil { return Inspection{}, err } shown, _, err := inspectOwner(ctx, policy, o, record, ref) if err != nil { return Inspection{}, err } id, err := nonce() if err != nil { return Inspection{}, err } c := o.Claim challenge, err := hostfence.PrepareChallenge(s.Now(), hostfence.Challenge{ID: id, Actor: actor, Host: host, HolderIncarnation: c.Holder.Incarnation, HolderStamp: c.Holder.Stamp, HolderSession: c.Holder.Session, ClaimGen: c.Gen, ClaimETag: o.ETag, Op: hostfence.Operation{ID: c.Op.ID, Kind: c.Op.Kind, Phase: c.Op.Phase}, Record: record, Box: ref, Enrollment: b.Enrollment, Cloud: hostfence.CloudBinding{Provider: b.Cloud.Provider, Account: b.Cloud.Account, Region: b.Cloud.Region, InstanceID: b.Cloud.InstanceID}, Consequences: shown}) if err != nil { return Inspection{}, err } wire, err := hostfence.SignChallenge(s.Signer, challenge) return Inspection{wire, challenge}, err } type owner struct{} func (owner) ValidateDeclareLost(c hostclaim.Claim, d hostclaim.LostDeclaration) error { _, want, err := effects(&c, d.Box) if err != nil { return err } if !reflect.DeepEqual(want, d.Consequences) { return ErrResolve } return nil } func (s Service) Declare(ctx context.Context, actor, host string, in hostfence.Submission) (hostclaim.Observed, error) { now := s.Now().UTC() var c hostfence.Challenge err := release.VerifyRootEnvelope(s.Directory.PublicKey, in.Challenge, &c) if err != nil { return hostclaim.Observed{}, err } if c.Actor != actor || c.Host != host || in.Confirmation != c.ID || !in.Attestation.MaintainComputeFence || !in.Attestation.ColdQuarantineStartOnly || !reflect.DeepEqual(in.Attestation.Disks, c.Consequences.DestroyedStorageAssertion) { return hostclaim.Observed{}, ErrResolve } // A retry uses the durable accepted declaration even after its claim CAS. // Inspect a new challenge only when no accepted declaration exists. prior, readErr := hostclaim.ReadLossRecord(ctx, s.Directory.Store, s.Directory.PublicKey, c.ID) if readErr == nil { if prior.Record.Actor != actor || prior.Record.Host != host || prior.Record.Enrollment != c.Enrollment { return hostclaim.Observed{}, ErrResolve } return s.finish(ctx, prior) } if !errors.Is(readErr, hostclaim.ErrLossRecordAbsent) { return hostclaim.Observed{}, readErr } // Acceptance survives a crash before either quarantine or loss materialization. acceptance, readErr := s.readAcceptance(ctx, c.ID) if readErr == nil { if acceptance.Loss.Actor != actor || acceptance.Loss.Host != host || acceptance.Loss.Enrollment != c.Enrollment || acceptance.Loss.Box != c.Box || acceptance.Loss.ExpectedClaimETag != c.ClaimETag { return hostclaim.Observed{}, ErrResolve } return s.accept(ctx, acceptance) } if !errors.Is(readErr, os.ErrNotExist) { return hostclaim.Observed{}, readErr } if _, err = hostfence.VerifyChallenge(s.Directory.PublicKey, in.Challenge, now); err != nil { return hostclaim.Observed{}, err } if err = hostfence.ValidateObservation(c, in.Observation, now); err != nil { return hostclaim.Observed{}, err } roster, err := s.Directory.Read(ctx) if err != nil { return hostclaim.Observed{}, err } b, ok := roster.Roster.Boxes[c.Box] if !ok || b.Enrollment != c.Enrollment || b.State == "retired" || b.Cloud != (hostactors.Cloud{Provider: c.Cloud.Provider, Account: c.Cloud.Account, Region: c.Cloud.Region, InstanceID: c.Cloud.InstanceID}) { return hostclaim.Observed{}, ErrResolve } observed, err := hostclaim.Read(ctx, s.Directory.Store, host) if err != nil { return hostclaim.Observed{}, err } if observed.Absent || observed.ETag != c.ClaimETag || observed.Claim.Gen != c.ClaimGen { return hostclaim.Observed{}, ErrResolve } record, err := s.record(ctx, observed.Claim) if err != nil || record != c.Record { return hostclaim.Observed{}, ErrResolve } policy, err := s.operationOwner(observed.Claim.Op.Kind) if err != nil { return hostclaim.Observed{}, err } shown, actual, err := inspectOwner(ctx, policy, observed, record, c.Box) if err != nil || !reflect.DeepEqual(shown, c.Consequences) { return hostclaim.Observed{}, ErrResolve } if b.QuarantineRevision == math.MaxUint64 || slices.Contains(b.LossIDs, c.ID) { return hostclaim.Observed{}, ErrResolve } loss := hostclaim.LossRecord{V: 1, ID: c.ID, State: "prepared", Actor: actor, AcceptedAt: now, Host: host, HolderIncarnation: c.HolderIncarnation, HolderStamp: c.HolderStamp, HolderSession: c.HolderSession, ExpectedClaimGen: c.ClaimGen, ExpectedClaimETag: c.ClaimETag, OpID: c.Op.ID, OpKind: c.Op.Kind, OpPhase: c.Op.Phase, Box: c.Box, Enrollment: c.Enrollment, QuarantineRevision: b.QuarantineRevision + 1, FencedAt: now, ConfirmedConsequences: actual, Timers: hostclaim.TimerLoss{Knowledge: hostclaim.TimersUnknown}} loss.WriteSources, err = ownerSources(ctx, policy, observed, actual) if err != nil { return hostclaim.Observed{}, err } loss.CloudObservation = hostclaim.LossCloudObservation{ObservedAt: in.Observation.ObservedAt, Account: in.Observation.Account, Region: in.Observation.Region, InstanceID: in.Observation.InstanceID, State: in.Observation.State, RequestID: in.Observation.RequestID} loss.OperatorAttestation = hostclaim.LossOperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true, DestroyedStorage: in.Attestation.Disks} if err := loss.Validate(); err != nil { return hostclaim.Observed{}, err } acceptance = lossAcceptance{V: 1, Bucket: s.Directory.Bucket, RosterETag: roster.ETag, Loss: loss} wire, err := release.SignRootEnvelope(s.Signer, acceptance) if err != nil { return hostclaim.Observed{}, err } key, err := acceptanceKey(c.ID) if err != nil { return hostclaim.Observed{}, err } if err := release.CreateExact(ctx, s.Directory.Store, key, wire); err != nil { return hostclaim.Observed{}, err } return s.accept(ctx, acceptance) } type lossAcceptance struct { V int `json:"v"` Bucket string `json:"bucket"` RosterETag string `json:"roster_etag"` Loss hostclaim.LossRecord `json:"loss"` } func acceptanceKey(id string) (string, error) { if _, err := hostclaim.LossRecordKey(id); err != nil { return "", err } return "control/host-loss-acceptances/" + id + ".json", nil } func (s Service) readAcceptance(ctx context.Context, id string) (lossAcceptance, error) { key, err := acceptanceKey(id) if err != nil { return lossAcceptance{}, err } raw, _, err := s.Directory.Store.Get(ctx, key) if err != nil { return lossAcceptance{}, err } var a lossAcceptance if release.VerifyRootEnvelope(s.Directory.PublicKey, raw, &a) != nil || a.V != 1 || a.Bucket != s.Directory.Bucket || a.RosterETag == "" || a.Loss.ID != id || a.Loss.State != "prepared" || a.Loss.Validate() != nil { return lossAcceptance{}, ErrResolve } return a, nil } func (s Service) accept(ctx context.Context, a lossAcceptance) (hostclaim.Observed, error) { if _, err := s.operationOwner(a.Loss.OpKind); err != nil { return hostclaim.Observed{}, err } roster, err := s.Directory.Read(ctx) if err != nil { return hostclaim.Observed{}, err } loss := a.Loss b, ok := roster.Roster.Boxes[loss.Box] if !ok || b.Enrollment != loss.Enrollment || b.State == "retired" || b.Cloud.Account != loss.CloudObservation.Account || b.Cloud.Region != loss.CloudObservation.Region || b.Cloud.InstanceID != loss.CloudObservation.InstanceID { return hostclaim.Observed{}, ErrResolve } if !slices.Contains(b.LossIDs, loss.ID) { // Retry unrelated roster changes with a fresh CAS. A declaration for // this same box defeats the accepted revision and needs new inspection. if b.QuarantineRevision == math.MaxUint64 || b.QuarantineRevision+1 != loss.QuarantineRevision { return hostclaim.Observed{}, ErrResolve } roster, err = s.Directory.Quarantine(ctx, s.Signer, roster, loss.Box, loss.Enrollment, loss.ID) if err != nil { return hostclaim.Observed{}, err } b = roster.Roster.Boxes[loss.Box] } if b.State != "quarantined" || b.QuarantineRevision < loss.QuarantineRevision { return hostclaim.Observed{}, ErrResolve } // The global deny remains in force through every subsequent failure. prepared, err := hostclaim.CreateLossRecord(ctx, s.Directory.Store, s.Signer, loss) if err != nil { return hostclaim.Observed{}, err } return s.finish(ctx, prepared) } func (s Service) finish(ctx context.Context, loss hostclaim.ObservedLossRecord) (hostclaim.Observed, error) { policy, err := s.operationOwner(loss.Record.OpKind) if err != nil { return hostclaim.Observed{}, err } observed, err := hostclaim.Read(ctx, s.Directory.Store, loss.Record.Host) if err != nil { return hostclaim.Observed{}, err } if loss.Record.State == "conflicted" { return hostclaim.Observed{}, hostclaim.ErrConflict } // A retained loss reference proves an earlier successful CAS even if // bookkeeping failed. Otherwise a changed ETag fences every stale retry. if loss.Record.State == "prepared" && (observed.Absent || !slices.Contains(observed.Claim.LossRefs, loss.Record.ID)) && (observed.Absent || observed.ETag != loss.Record.ExpectedClaimETag || observed.Claim.Gen != loss.Record.ExpectedClaimGen) { r := loss.Record r.State = "conflicted" if _, err := hostclaim.ReplaceLossRecord(ctx, s.Directory.Store, s.Signer, loss, r); err != nil { return hostclaim.Observed{}, err } return hostclaim.Observed{}, hostclaim.ErrConflict } if loss.Record.State == "prepared" && !slices.Contains(observed.Claim.LossRefs, loss.Record.ID) { if err := ownerHook(ctx, policy, observed, loss.Record, false); err != nil { return hostclaim.Observed{}, err } } next, err := hostclaim.DeclareLost(ctx, s.Directory.Store, s.Directory.PublicKey, observed, loss.Record.ID, policy) if err != nil { return hostclaim.Observed{}, err } if loss.Record.State == "committed" { return next, ownerHook(ctx, policy, next, loss.Record, true) } r := loss.Record r.State = "committed" r.ResultClaimGen = next.Claim.Gen _, err = hostclaim.ReplaceLossRecord(ctx, s.Directory.Store, s.Signer, loss, r) if err != nil { return next, err } return next, ownerHook(ctx, policy, next, r, true) }