package host import ( "context" "errors" "fmt" "slices" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/stopevidence" ) // ErrClaimWritersStopped refuses new writer work on an attempt whose // writers StopWriters has stopped. var ErrClaimWritersStopped = errors.New("host claims: the session's writers are stopped") // The producers a writers-only stop keeps running: the session's // replicator, its generation marker publisher and region HA's files // publisher (claim ยง6.2a, L13). Every other producer is a writer: a role // launch, or a timer delivery ("timer-" ids, ClaimTimerDelivery). const ( ClaimProducerReplicator = "replicator" ClaimProducerPublisher = "publisher" ClaimProducerFiles = "files" ) func claimRetainedProducer(id string) bool { return id == ClaimProducerReplicator || id == ClaimProducerPublisher || id == ClaimProducerFiles } // ClaimWritersDrained is StopWriters' witness for one exact writer session // (claim Plan 11, R2). It is no stop evidence: nothing is fenced, signed // or published, and the claim is untouched. Root's writers-stopped // observation (R3) and root's full stop still follow. type ClaimWritersDrained struct { // Target is the session's launch tuple: host, incarnation, stamp, // writer session, box, enrollment, boot and controller. Target hostguard.FenceTarget Generation string WriterBox string // FenceClosed: the attempt and its producers admit no new writer. FenceClosed bool // LaunchJoined: every role launch producer was cancelled and joined. LaunchJoined bool // WorkDrained: every in-flight timer delivery was cancelled and joined. WorkDrained bool // SpawnsSettled: the launch journal holds no unsettled reservation. SpawnsSettled bool // CgroupEmpty: the session's containment was emptied and reads empty. CgroupEmpty bool // Cgroup is the containment's path relative to the cgroup root. Cgroup string DBPaths []string // Drained and Retained are the producer ids joined and still running. Drained, Retained []string CheckedAt time.Time } // Valid reports a complete witness. func (d ClaimWritersDrained) Valid() bool { t := d.Target return t.Host != "" && t.Incarnation != "" && t.Stamp != 0 && t.Session != 0 && t.Box != "" && t.Enrollment != "" && t.BootID != "" && t.ControllerID != "" && d.FenceClosed && d.LaunchJoined && d.WorkDrained && d.SpawnsSettled && d.CgroupEmpty && d.Cgroup != "" && len(d.DBPaths) != 0 && !d.CheckedAt.IsZero() } // For checks the witness still names o's writer session: the same host, // holder, stamp, session, generation and writer box. Any other session, // including a later one on this box, gets nothing from it. func (d ClaimWritersDrained) For(o hostclaim.Observed) error { if !d.Valid() || o.Claim == nil || o.Claim.Holder == nil { return hostbox.ErrChanged } h := o.Claim.Holder if o.Host != d.Target.Host || h.Incarnation != d.Target.Incarnation || h.Stamp != d.Target.Stamp || h.Session != d.Target.Session || h.Generation != d.Generation || h.WriterBox != d.WriterBox { return hostbox.ErrChanged } return nil } // claimWritersJournal is the launch journal as StopWriters uses it. type claimWritersJournal interface { CloseWriters(context.Context, hostguard.FenceTarget) error Read(context.Context, hostguard.FenceTarget) (hostbox.LaunchInventory, error) Settled(context.Context, hostbox.LaunchReservation) error } // claimWritersCgroup is the session's containment (the launch's parent). type claimWritersCgroup interface { Populated() (bool, error) KillAndEmpty(context.Context) error } // admitsWriters refuses once the attempt's writers are stopped. Every new // launch, reservation, restore and producer of the attempt asks it. func (a *claimWakeAttempt) admitsWriters() error { a.mu.Lock() defer a.mu.Unlock() if a.writersClosed { return ErrClaimWritersStopped } return nil } // StopWriters stops the attempt's writers and keeps its replicator (claim // Plan 11, R2). In order: close admission; cancel and join the role launch // and the schedule runner's in-flight deliveries for the host (both are // producers of the session); empty the session's containment; settle every // spawn reservation the journal still holds; check the containment reads // empty. The replicator, the generation publisher and the files publisher // keep running, and the lease stays held. It never releases the claim or // publishes evidence; root's stop does that. Once it has succeeded it // returns the same witness again without repeating any effect; after a // failure the next call resumes, each step being idempotent. func (a *claimWakeAttempt) StopWriters(ctx context.Context) (ClaimWritersDrained, error) { if a == nil || ctx == nil { return ClaimWritersDrained{}, hostwrite.ErrUnavailable } a.writersMu.Lock() defer a.writersMu.Unlock() a.mu.Lock() if a.writers != nil { done := a.writers.clone() a.mu.Unlock() return done, nil } observed, in, owner := a.observed, a.inputs, a.owner launch := a.launch if launch == nil { launch = a.prelaunch } journal, group := a.writersJournal, a.writersCgroup a.mu.Unlock() if launch == nil || owner == nil || owner.Producers == nil || journal == nil || group == nil || in.DBPath == "" { return ClaimWritersDrained{}, hostwrite.ErrUnavailable } target := launch.Target if observed.Claim == nil || observed.Claim.Holder == nil { return ClaimWritersDrained{}, hostbox.ErrChanged } h := observed.Claim.Holder if observed.Host != target.Host || h.Incarnation != target.Incarnation || h.Stamp != target.Stamp || h.Session != target.Session || target.Session == 0 { return ClaimWritersDrained{}, hostbox.ErrChanged } // 1. Admission: the attempt refuses new launches and restores, and the // session's producers refuse every writer from here on. a.mu.Lock() a.writersClosed = true a.mu.Unlock() producers, err := owner.Producers(ctx, observed) if err != nil { return ClaimWritersDrained{}, err } if err := journal.CloseWriters(ctx, target); err != nil { return ClaimWritersDrained{}, err } // 2. The role launch and the timer deliveries, joined. drained, err := producers.StopWriters(ctx, target) if err != nil { return ClaimWritersDrained{}, err } // 3. The containment: whatever the roles left is killed and waited for. if err := group.KillAndEmpty(ctx); err != nil { return ClaimWritersDrained{}, err } // 4. Spawn reservations: with the containment empty, nothing reserved // or started can still run. An intended role is root's to resolve. inventory, err := journal.Read(ctx, target) if err != nil { return ClaimWritersDrained{}, err } for _, r := range inventory.Reservations { switch r.State { case "settled": case "reserved", "started": if err := journal.Settled(ctx, r); err != nil { return ClaimWritersDrained{}, err } default: return ClaimWritersDrained{}, fmt.Errorf("%w: reservation %s is %s", ErrClaimStopInventory, r.ID, r.State) } } settled, err := journal.Read(ctx, target) if err != nil { return ClaimWritersDrained{}, err } for _, r := range settled.Reservations { if r.State != "settled" { return ClaimWritersDrained{}, fmt.Errorf("%w: reservation %s is %s", ErrClaimStopInventory, r.ID, r.State) } } // 5. The containment reads empty after the journal is settled. populated, err := group.Populated() if err != nil { return ClaimWritersDrained{}, err } if populated { return ClaimWritersDrained{}, fmt.Errorf("%w: the session's cgroup is still populated", ErrClaimStopInventory) } var retained []string for _, id := range producers.Running() { if !claimRetainedProducer(id) { return ClaimWritersDrained{}, fmt.Errorf("%w: producer %s still runs", ErrClaimStopIdentity, id) } retained = append(retained, id) } // Every producer but the retained ones was joined in step 2: the role // launch's and the timer deliveries' ("timer-" ids) alike. done := ClaimWritersDrained{Target: target, Generation: h.Generation, WriterBox: h.WriterBox, FenceClosed: true, LaunchJoined: true, WorkDrained: true, SpawnsSettled: true, CgroupEmpty: true, Cgroup: launch.ParentPath, DBPaths: []string{in.DBPath}, Drained: slices.Clip(drained), Retained: retained, CheckedAt: time.Now().UTC()} if !done.Valid() { return ClaimWritersDrained{}, hostwrite.ErrUnavailable } a.mu.Lock() cached := done.clone() a.writers = &cached a.mu.Unlock() return done, nil } // recordReplicatorHints takes this process's own descriptors on the // database files immediately before and after open, and keeps those that // appeared: the replicator's descriptors (R3, the agent half). They are a // narrowing hint for root, never evidence. func recordReplicatorHints(ctx context.Context, procRoot string, pid int, db string, open func() error) ([]stopevidence.DescriptorHint, error) { paths := []string{db, db + "-wal", db + "-shm"} before, beforeErr := stopevidence.OwnDescriptors(ctx, procRoot, pid, paths) if err := open(); err != nil { return nil, err } if beforeErr != nil { return nil, beforeErr } after, err := stopevidence.OwnDescriptors(ctx, procRoot, pid, paths) if err != nil { return nil, err } var hints []stopevidence.DescriptorHint for _, d := range after { if !slices.Contains(before, d) { hints = append(hints, d) } } return hints, nil } func (d ClaimWritersDrained) clone() ClaimWritersDrained { d.DBPaths = slices.Clone(d.DBPaths) d.Drained = slices.Clone(d.Drained) d.Retained = slices.Clone(d.Retained) return d }