package hostops import ( "bytes" "context" "crypto/ed25519" "encoding/json" "errors" "reflect" "slices" "time" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfence" "github.com/carlosframework/platform/internal/hostresolve" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/release" ) // The declare-lost grant for a move's target (claim Plan 10, J17 with J4). // The console's resolve doors call these two typed ControlWriter methods; // neither takes a store, a callback or a transition. Each builds the generic // hostresolve service with only the moves owner registered, for the move // kinds only, over a private store that never leaves the method: // // - inspection reads the raw bucket through a store that refuses every // write; // - a declaration writes through lossScopedStore, bound to the exact // declaration the signed challenge names (loss id, host, op, kind, box, // claim ETag, incarnation, account and app). It admits exactly the // service's acceptance, quarantine of that box, and loss record for that // id, the claim's declare-lost CAS and the owner's release of it, and // the owner's two record transitions, each recomputed from the bucket // and compared with the bytes offered. The move's admitted registration // is reread at every acceptance, quarantine, claim and record write. // lossBinding is one declaration's scope. type lossBinding struct { lossID, host, opID, kind, box, claimETag, incarnation, account, app, bucket string } // moveLossService is the service a typed method runs, with the moves owner // for the move kinds and nothing else. func moveLossService(store release.Store, pub ed25519.PublicKey, signer release.Signer, bucket string, now func() time.Time) hostresolve.Service { owner := &moveLossOwner{store: store, pub: pub, now: now} owners := map[string]hostresolve.OperationOwner{} for _, kind := range moveLossKinds { owners[kind] = owner } return hostresolve.Service{Directory: hostactors.Directory{Store: store, PublicKey: pub, Bucket: bucket}, Signer: signer, Now: now, Owners: owners} } func (w *ControlWriter) lossSigner(signer release.Signer) error { if err := w.ready(); err != nil { return err } if !bytes.Equal(signer.RootPublicKey(), w.pub) { return ErrContract } return nil } // moveLossClaim reads the claim a door names and holds it to the door's // account and app and to a move kind this owner serves. func (w *ControlWriter) moveLossClaim(ctx context.Context, account, app, host string) (hostclaim.Observed, error) { o, err := hostclaim.Read(ctx, w.raw, host) if err != nil { return hostclaim.Observed{}, err } if o.Absent || o.Claim == nil || o.Claim.Holder == nil { return hostclaim.Observed{}, hostresolve.ErrResolve } h := o.Claim.Holder if !operationHomeMatches(w.homeRegion, o.Claim) { return hostclaim.Observed{}, ErrIdentity } if h.Family != "instance" || h.Account != account || h.App != app || h.Key != provision.RecordKey(account, app, host) { return hostclaim.Observed{}, ErrIdentity } return o, nil } // InspectMoveLoss answers the signed challenge for declaring box, a move's // target, lost on host. Read-only. func (w *ControlWriter) InspectMoveLoss(ctx context.Context, signer release.Signer, bucket, actor, account, app, host, box string) (hostresolve.Inspection, error) { if err := w.lossSigner(signer); err != nil { return hostresolve.Inspection{}, err } o, err := w.moveLossClaim(ctx, account, app, host) if err != nil { return hostresolve.Inspection{}, err } if !slices.Contains(moveLossKinds, o.Claim.Op.Kind) { return hostresolve.Inspection{}, hostresolve.ErrResolve } if err := moveAdmission(ctx, w.raw, host, o.Claim); err != nil { return hostresolve.Inspection{}, err } return moveLossService(readOnlyStore{w.raw}, w.pub, signer, bucket, w.now).Inspect(ctx, actor, host, box) } // DeclareMoveLoss accepts and completes a declaration for the challenge in // the submission. A new declaration needs the move's exact admitted // registration (closed admission and an expired registration still drain // an accepted move); a retry of an accepted one replays it from its durable // records, even after its claim CAS. It answers the claim the service last // observed. func (w *ControlWriter) DeclareMoveLoss(ctx context.Context, signer release.Signer, bucket, actor, account, app, host, box string, in hostfence.Submission) (hostclaim.Observed, error) { if err := w.lossSigner(signer); err != nil { return hostclaim.Observed{}, err } var c hostfence.Challenge if err := release.VerifyRootEnvelope(w.pub, in.Challenge, &c); err != nil { return hostclaim.Observed{}, hostresolve.ErrResolve } if _, err := hostclaim.LossRecordKey(c.ID); err != nil || c.Host != host || c.Actor != actor || c.Op.ID == "" || !slices.Contains(moveLossKinds, c.Op.Kind) { return hostclaim.Observed{}, hostresolve.ErrResolve } d := hostactors.Directory{Store: readOnlyStore{w.raw}, PublicKey: w.pub, Bucket: bucket} ref, _, err := d.Resolve(ctx, box) if err != nil || ref != c.Box { return hostclaim.Observed{}, hostresolve.ErrResolve } o, err := w.moveLossClaim(ctx, account, app, host) if err != nil { return hostclaim.Observed{}, err } if o.Claim.Holder.Incarnation != c.HolderIncarnation { return hostclaim.Observed{}, hostresolve.ErrResolve } if o.Claim.Op.ID == c.Op.ID { if err := moveAdmission(ctx, w.raw, host, o.Claim); err != nil { return hostclaim.Observed{}, err } } else if prior, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, c.ID); err != nil || prior.Record.Host != host || prior.Record.OpID != c.Op.ID { // The operation has moved on: only an accepted declaration replays. return hostclaim.Observed{}, hostresolve.ErrResolve } b := lossBinding{bucket: bucket, lossID: c.ID, host: host, opID: c.Op.ID, kind: c.Op.Kind, box: c.Box, claimETag: c.ClaimETag, incarnation: c.HolderIncarnation, account: account, app: app} scoped := &lossScopedStore{raw: w.raw, pub: w.pub, now: w.now, b: b, homeRegion: w.homeRegion} if _, err := moveLossService(scoped, w.pub, signer, bucket, w.now).Declare(ctx, actor, host, in); err != nil { return hostclaim.Observed{}, err } // The service answers its claim CAS; the owner's release wrote after it. return hostclaim.Read(ctx, w.raw, host) } // readOnlyStore refuses every write. type readOnlyStore struct{ raw release.Store } func (s readOnlyStore) Get(ctx context.Context, key string) ([]byte, string, error) { return s.raw.Get(ctx, key) } func (s readOnlyStore) List(ctx context.Context, prefix string) ([]string, error) { return s.raw.List(ctx, prefix) } func (s readOnlyStore) ListDirs(ctx context.Context, prefix string) ([]string, error) { return s.raw.ListDirs(ctx, prefix) } func (readOnlyStore) Put(context.Context, string, []byte, string) (string, error) { return "", errMoveLossScope } func (readOnlyStore) Delete(context.Context, string) error { return errMoveLossScope } // lossScopedStore is one declaration's private store (see above). type lossScopedStore struct { homeRegion string raw release.Store pub ed25519.PublicKey now func() time.Time b lossBinding } func (s *lossScopedStore) Get(ctx context.Context, key string) ([]byte, string, error) { return s.raw.Get(ctx, key) } func (s *lossScopedStore) List(ctx context.Context, prefix string) ([]string, error) { return s.raw.List(ctx, prefix) } func (s *lossScopedStore) ListDirs(ctx context.Context, prefix string) ([]string, error) { return s.raw.ListDirs(ctx, prefix) } func (s *lossScopedStore) Delete(context.Context, string) error { return errMoveLossScope } func (s *lossScopedStore) Put(ctx context.Context, key string, body []byte, etag string) (string, error) { lossKey, err := hostclaim.LossRecordKey(s.b.lossID) if err != nil { return "", errMoveLossScope } claimKey, err := hostclaim.Key(s.b.host) if err != nil { return "", errMoveLossScope } switch { case key == "control/host-loss-acceptances/"+s.b.lossID+".json": err = s.acceptance(ctx, body, etag) case key == lossKey: err = s.lossRecord(ctx, body, etag) case key == hostactors.Key: err = s.quarantine(ctx, body, etag) case key == claimKey: err = s.claim(ctx, body, etag) case provision.IsRecordKey(key): err = s.record(ctx, key, body, etag) default: err = errMoveLossScope } if err != nil { return "", err } if provision.IsRecordKey(key) { return putSequencedRecord(ctx, s.raw, key, body, etag) } return s.raw.Put(ctx, key, body, etag) } // bound holds a loss record to the declaration. func (s *lossScopedStore) bound(r hostclaim.LossRecord) bool { b := s.b return r.ID == b.lossID && r.Host == b.host && r.OpID == b.opID && r.OpKind == b.kind && r.Box == b.box && r.ExpectedClaimETag == b.claimETag && r.HolderIncarnation == b.incarnation && r.Validate() == nil } // registered rereads the claim and the move's admitted registration. The // claim must still be this holder's, under this op. func (s *lossScopedStore) registered(ctx context.Context) (hostclaim.Observed, error) { o, err := hostclaim.Read(ctx, s.raw, s.b.host) if err != nil { return hostclaim.Observed{}, err } if o.Absent || o.Claim == nil || o.Claim.Holder == nil { return hostclaim.Observed{}, ErrIdentity } h := o.Claim.Holder if !operationHomeMatches(s.homeRegion, o.Claim) { return hostclaim.Observed{}, ErrIdentity } if h.Account != s.b.account || h.App != s.b.app || h.Incarnation != s.b.incarnation || o.Claim.Op.ID != s.b.opID { return hostclaim.Observed{}, ErrIdentity } if err := moveAdmission(ctx, s.raw, s.b.host, o.Claim); err != nil { return hostclaim.Observed{}, err } return o, nil } func (s *lossScopedStore) acceptance(ctx context.Context, body []byte, etag string) error { if etag != "" { return errMoveLossScope } var a struct { V int `json:"v"` Bucket string `json:"bucket"` RosterETag string `json:"roster_etag"` Loss hostclaim.LossRecord `json:"loss"` } if release.VerifyRootEnvelope(s.pub, body, &a) != nil || a.V != 1 || s.b.bucket == "" || a.Bucket != s.b.bucket || a.RosterETag == "" || a.Loss.State != "prepared" || !s.bound(a.Loss) { return errMoveLossScope } _, err := s.registered(ctx) return err } func (s *lossScopedStore) lossRecord(ctx context.Context, body []byte, etag string) error { var r hostclaim.LossRecord if release.VerifyRootEnvelope(s.pub, body, &r) != nil || !s.bound(r) { return errMoveLossScope } if _, err := s.registered(ctx); err != nil { return err } before, err := hostclaim.ReadLossRecord(ctx, s.raw, s.pub, s.b.lossID) if errors.Is(err, hostclaim.ErrLossRecordAbsent) { if etag != "" || r.State != "prepared" { return errMoveLossScope } return nil } if err != nil { return err } if before.ETag != etag { return release.ErrCASConflict } if hostclaim.ValidateLossRecordReplacement(before, r) != nil { return errMoveLossScope } return nil } // quarantine admits exactly Directory.Quarantine of the bound box for the // bound declaration: one revision, that box's state, revision and loss ids, // nothing else. func (s *lossScopedStore) quarantine(ctx context.Context, body []byte, etag string) error { if _, err := s.registered(ctx); err != nil { return err } raw, tag, err := s.raw.Get(ctx, hostactors.Key) if err != nil { return err } if tag != etag { return release.ErrCASConflict } var before, after hostactors.Roster if release.VerifyRootEnvelope(s.pub, raw, &before) != nil || release.VerifyRootEnvelope(s.pub, body, &after) != nil { return errMoveLossScope } old, ok := before.Boxes[s.b.box] next, ok2 := after.Boxes[s.b.box] if !ok || !ok2 || after.Revision != before.Revision+1 || next.State != "quarantined" || next.QuarantineRevision != old.QuarantineRevision+1 || !reflect.DeepEqual(next.LossIDs, append(append([]string(nil), old.LossIDs...), s.b.lossID)) { return errMoveLossScope } rest := hostactors.Clone(after) rest.Revision = before.Revision box := next box.State, box.QuarantineRevision, box.LossIDs = old.State, old.QuarantineRevision, old.LossIDs rest.Boxes[s.b.box] = box a, err1 := json.Marshal(rest) b, err2 := json.Marshal(before) if err1 != nil || err2 != nil || !bytes.Equal(a, b) { return errMoveLossScope } return nil } // claim admits exactly the declaration's claim CAS (hostclaim.DeclareLost // of the bound loss id, while the loss is prepared) or the owner's release // of the lost target (once the claim carries the declaration), each // recomputed from the bucket's current claim at the offered ETag. func (s *lossScopedStore) claim(ctx context.Context, body []byte, etag string) error { o, err := s.registered(ctx) if err != nil { return err } if o.ETag != etag { return release.ErrCASConflict } loss, err := hostclaim.ReadLossRecord(ctx, s.raw, s.pub, s.b.lossID) if err != nil { return err } if !s.bound(loss.Record) { return errMoveLossScope } owner := &moveLossOwner{store: readOnlyStore{s.raw}, pub: s.pub, now: s.now} var want []byte switch { case loss.Record.State == "prepared" && !slicesContain(o.Claim.LossRefs, s.b.lossID): want, err = simulateClaim(ctx, s.raw, o, func(sim release.Store) error { _, err := hostclaim.DeclareLost(ctx, sim, s.pub, o, s.b.lossID, owner) return err }) case loss.Record.State == "committed" && slicesContain(o.Claim.LossRefs, s.b.lossID): in, _, ierr := lossIntentOf(loss.Record) if ierr != nil { return ierr } rel := moveLossRelease(in, loss.Record, o.Claim.Holder, s.b.host) want, err = simulateClaim(ctx, s.raw, o, func(sim release.Store) error { _, err := hostclaim.Update(ctx, sim, o, hostclaim.ReleaseLostTarget(rel)) return err }) default: return errMoveLossScope } if err != nil { return err } if !bytes.Equal(want, body) { return errMoveLossScope } return nil } // record admits exactly the owner's two record transitions for the holder's // record: the prepared declaration's decision at the inspected record, or // the committed declaration's clean release once the claim has released // the target. Only move fields, the record sequence and the move's update // time may differ from the recomputation. func (s *lossScopedStore) record(ctx context.Context, key string, body []byte, etag string) error { o, err := s.registered(ctx) if err != nil { return err } if key != o.Claim.Holder.Key { return errMoveLossScope } if err := CheckCreateFreeze(ctx, s.raw, s.pub); err != nil { return err } cur, err := ReadRecord(ctx, s.raw, key) if err != nil { return err } if cur.ETag != etag { return release.ErrCASConflict } if !matches(cur, o.Claim.Holder) { return ErrIdentity } loss, err := hostclaim.ReadLossRecord(ctx, s.raw, s.pub, s.b.lossID) if err != nil { return err } if !s.bound(loss.Record) { return errMoveLossScope } in, ref, err := lossIntentOf(loss.Record) if err != nil { return err } _, seq, err := recordSequence(cur.Body) if err != nil { return err } _, next, err := recordSequence(body) if err != nil || next != seq+1 { return errMoveLossScope } want, err := provision.DecodeRecord(cur.Body) if err != nil { return ErrIdentity } switch { case loss.Record.State == "prepared" && o.ETag == loss.Record.ExpectedClaimETag && cur.ETag == in.RecordETag && seq == in.RecordSeq: if in.Supersedes != "" { // A rebind is admitted only while the superseded declaration is // still signed conflicted and the intent recomputes exactly. again, ierr := lossIntentAt(ctx, readOnlyStore{s.raw}, s.pub, o, cur, in.Target) if ierr != nil || !reflect.DeepEqual(again, in) { return errMoveLossScope } } err = prepareMoveLoss(&want, in, ref, loss.Record.FencedAt) case loss.Record.State == "committed" && hostclaim.LostTargetReleased(o.Claim, s.b.lossID): err = provision.MoveRollbackLossCommitted(provision.MoveGuard{ID: in.Move, Incarnation: in.Incarnation})(&want) default: return errMoveLossScope } if err != nil { return err } expected, err := json.Marshal(want) if err != nil { return err } if !sameExceptSequenceAndUpdate(expected, body) { return errMoveLossScope } return nil } // sameExceptSequenceAndUpdate compares two record bodies as JSON values, // ignoring record_seq and move.updated_at only. func sameExceptSequenceAndUpdate(a, b []byte) bool { strip := func(raw []byte) (map[string]any, bool) { var v map[string]any dec := json.NewDecoder(bytes.NewReader(raw)) dec.UseNumber() if dec.Decode(&v) != nil { return nil, false } delete(v, "record_seq") if m, ok := v["move"].(map[string]any); ok { delete(m, "updated_at") } return v, true } x, ok1 := strip(a) y, ok2 := strip(b) return ok1 && ok2 && reflect.DeepEqual(x, y) } // simulateClaim runs a claim write against a private copy of the bucket's // current claim and returns the bytes it would Put. Every other key reads // through to the bucket; no write reaches it. func simulateClaim(ctx context.Context, raw release.Store, o hostclaim.Observed, write func(release.Store) error) ([]byte, error) { key, err := hostclaim.Key(o.Host) if err != nil { return nil, err } sim := &claimSimulation{raw: raw, key: key, etag: o.ETag} if err := write(sim); err != nil { return nil, err } if sim.put == nil { return nil, errMoveLossScope } return sim.put, nil } type claimSimulation struct { raw release.Store key, etag string put []byte } func (s *claimSimulation) Get(ctx context.Context, key string) ([]byte, string, error) { return s.raw.Get(ctx, key) } func (s *claimSimulation) List(ctx context.Context, prefix string) ([]string, error) { return s.raw.List(ctx, prefix) } func (s *claimSimulation) ListDirs(ctx context.Context, prefix string) ([]string, error) { return s.raw.ListDirs(ctx, prefix) } func (s *claimSimulation) Delete(context.Context, string) error { return errMoveLossScope } func (s *claimSimulation) Put(_ context.Context, key string, body []byte, etag string) (string, error) { if key != s.key || etag != s.etag || s.put != nil { return "", errMoveLossScope } s.put = append([]byte(nil), body...) return "simulated", nil }