package host import ( "bytes" "context" "encoding/json" "fmt" "io" "net" "net/http" "slices" "time" "github.com/carlosframework/platform/internal/controlsock" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/stopevidence" ) // WritersStoppedRequest asks root for the writers-stopped observation of one // exact writer session on this box (claim Plan 11, R3; L10). It names the // session and the claim it was read at, and carries the agent's narrowing // hint. It carries no pid, process start, path or cgroup: root takes the // agent's process from the socket's peer credentials, the database paths // from the installation inventory and the containment from the launch // journal. type WritersStoppedRequest struct { Region string `json:"region"` Host string `json:"host"` Incarnation string `json:"incarnation"` InstallationStamp uint64 `json:"installation_stamp"` WriterSession uint64 `json:"writer_session"` Box string `json:"box"` Enrollment string `json:"enrollment"` BootID string `json:"boot_id"` ControllerID string `json:"controller_id"` ExpectedClaimETag string `json:"expected_claim_etag"` ExpectedDirectoryRevision string `json:"expected_directory_revision"` RosterRevision uint64 `json:"roster_revision"` Replicator []stopevidence.DescriptorHint `json:"replicator"` } // WritersStoppedEvidence is root's answer: what its own observation found // for the session. It is not stop evidence and is never fenced, signed or // published; root's stop receipt still follows the release. type WritersStoppedEvidence struct { Host string `json:"host"` Incarnation string `json:"incarnation"` InstallationStamp uint64 `json:"installation_stamp"` WriterSession uint64 `json:"writer_session"` Box string `json:"box"` Enrollment string `json:"enrollment"` BootID string `json:"boot_id"` ControllerID string `json:"controller_id"` ClaimETag string `json:"claim_etag"` ClaimGen uint64 `json:"claim_gen"` AgentPID int `json:"agent_pid"` AgentStart string `json:"agent_start"` CheckedAt time.Time `json:"checked_at"` CheckedPaths []string `json:"checked_paths"` CgroupIdentities []string `json:"cgroup_identities"` Retained []stopevidence.Descriptor `json:"retained"` } // WritersKernelEvidence is the kernel half's observation. type WritersKernelEvidence struct { CheckedPaths []string CgroupIdentities []string Retained []stopevidence.Descriptor } // stopWritersKernel is root's kernel half of the writers-stopped // observation: the session's journaled spawns settled and its containment // empty, and every holder of the installation's database files one of the // hinted replicator descriptors in the authenticated agent process. type stopWritersKernel interface { VerifyWritersStopped(context.Context, hostguard.FenceTarget, StopInventory, StopPeer, []stopevidence.DescriptorHint) (WritersKernelEvidence, error) } // WritersStopped is root's writers-stopped observation for req (R3). Root // trusts no agent-authored list: the hint can only narrow what its own scan // exempts, and the process it exempts in is the authenticated peer. func (c ClaimStopCoordinator) WritersStopped(ctx context.Context, req WritersStoppedRequest) (WritersStoppedEvidence, error) { if ctx == nil || c.CarlosUID == 0 || c.Peers == nil || c.Claims == nil || c.Inventory == nil || c.Kernel == nil || c.Now == nil || !stopComponent(req.Host) || !stopComponent(req.Box) || !stopComponent(req.Enrollment) || req.Incarnation == "" || req.InstallationStamp == 0 || req.WriterSession == 0 || req.BootID == "" || req.ControllerID == "" || req.ExpectedClaimETag == "" || req.ExpectedDirectoryRevision == "" || req.RosterRevision == 0 { return WritersStoppedEvidence{}, ErrClaimStopUnavailable } kernel, ok := c.Kernel.(stopWritersKernel) if !ok { return WritersStoppedEvidence{}, ErrClaimStopUnavailable } peer, err := c.Peers.AuthenticatedPeer(ctx) if err != nil { return WritersStoppedEvidence{}, err } if peer.UID != c.CarlosUID || peer.PID <= 0 || peer.ProcessStart == "" || peer.BootID != req.BootID || peer.ControllerID != req.ControllerID { return WritersStoppedEvidence{}, ErrClaimStopIdentity } observed, err := c.Claims.Read(ctx, req.Host) if err != nil { return WritersStoppedEvidence{}, err } if observed.Absent || observed.Claim == nil || observed.Claim.Holder == nil || observed.ETag != req.ExpectedClaimETag { return WritersStoppedEvidence{}, ErrClaimStopIdentity } claim, holder := observed.Claim, observed.Claim.Holder target := hostguard.FenceTarget{Region: req.Region, ClaimGen: claim.Gen, Host: req.Host, Incarnation: req.Incarnation, Stamp: req.InstallationStamp, Session: req.WriterSession, Op: claim.Op.ID, Phase: claim.Op.Phase, Box: req.Box, Enrollment: req.Enrollment, BootID: req.BootID, ControllerID: req.ControllerID, ClaimETag: req.ExpectedClaimETag, DirectoryRevision: req.ExpectedDirectoryRevision, RosterRevision: req.RosterRevision} // The session is this box's writer: the holder names it, and the claim // gives this box the authority a stop of it would need. if claim.Host != req.Host || holder.WriterBox != req.Box || !hostguard.StopAuthorityMatches(claim, target) { return WritersStoppedEvidence{}, ErrClaimStopIdentity } inv, err := c.Inventory.ReadStopInventory(ctx, req.Host) if err != nil { return WritersStoppedEvidence{}, err } if inv.Host != req.Host || inv.Incarnation != req.Incarnation || inv.InstallationStamp != req.InstallationStamp || inv.Box != req.Box || inv.Enrollment != req.Enrollment || inv.DirectoryRevision != req.ExpectedDirectoryRevision { return WritersStoppedEvidence{}, ErrClaimStopInventory } seen, err := kernel.VerifyWritersStopped(ctx, target, inv, peer, slices.Clone(req.Replicator)) if err != nil { return WritersStoppedEvidence{}, err } // The observation holds for the claim it was asked at. again, err := c.Claims.Read(ctx, req.Host) if err != nil { return WritersStoppedEvidence{}, err } if again.ETag != req.ExpectedClaimETag { return WritersStoppedEvidence{}, ErrClaimStopIdentity } return WritersStoppedEvidence{Host: req.Host, Incarnation: req.Incarnation, InstallationStamp: req.InstallationStamp, WriterSession: req.WriterSession, Box: req.Box, Enrollment: req.Enrollment, BootID: req.BootID, ControllerID: req.ControllerID, ClaimETag: req.ExpectedClaimETag, ClaimGen: claim.Gen, AgentPID: peer.PID, AgentStart: peer.ProcessStart, CheckedAt: c.Now().UTC(), CheckedPaths: seen.CheckedPaths, CgroupIdentities: seen.CgroupIdentities, Retained: seen.Retained}, nil } // serveWritersStopped answers POST /claim/writers-stopped for an // authenticated peer. func (h ClaimStopHTTP) serveWritersStopped(w http.ResponseWriter, r *http.Request, peer StopPeer) { decoder := json.NewDecoder(io.LimitReader(r.Body, 64<<10)) decoder.DisallowUnknownFields() var req WritersStoppedRequest if decoder.Decode(&req) != nil || decoder.Decode(new(any)) != io.EOF { http.Error(w, "refused", http.StatusBadRequest) return } c := h.Coordinator c.Peers = stopPeerValue{peer} ev, err := c.WritersStopped(r.Context(), req) if err != nil { http.Error(w, "refused", http.StatusConflict) return } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(ev) } // CollectWritersStopped asks root over the instance-control socket at path // for the writers-stopped observation of req's session. func CollectWritersStopped(ctx context.Context, path string, req WritersStoppedRequest) (WritersStoppedEvidence, error) { ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() body, err := json.Marshal(req) if err != nil { return WritersStoppedEvidence{}, err } transport := &http.Transport{DisableKeepAlives: true, DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "unix", path) }} defer transport.CloseIdleConnections() request, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://control/claim/writers-stopped", bytes.NewReader(body)) if err != nil { return WritersStoppedEvidence{}, err } response, err := (&http.Client{Transport: transport}).Do(request) if err != nil { return WritersStoppedEvidence{}, err } defer response.Body.Close() if response.StatusCode != http.StatusOK { return WritersStoppedEvidence{}, fmt.Errorf("%w: writers-stopped: %s", ErrClaimStopUnavailable, response.Status) } decoder := json.NewDecoder(io.LimitReader(response.Body, 1<<20)) decoder.DisallowUnknownFields() var ev WritersStoppedEvidence if err := decoder.Decode(&ev); err != nil { return WritersStoppedEvidence{}, err } if ev.Host != req.Host || ev.Incarnation != req.Incarnation || ev.InstallationStamp != req.InstallationStamp || ev.WriterSession != req.WriterSession || ev.Box != req.Box || ev.Enrollment != req.Enrollment || ev.BootID != req.BootID || ev.ControllerID != req.ControllerID || ev.ClaimETag != req.ExpectedClaimETag || ev.CheckedAt.IsZero() { return WritersStoppedEvidence{}, fmt.Errorf("%w: writers-stopped evidence names another session", ErrClaimStopIdentity) } return ev, nil } // WritersStopped is the agent's whole writers-stopped step for host's // admitted attempt: StopWriters (idempotent), then root's observation of // the witnessed session at the current claim, with the replicator hint // recorded when it opened. It never releases the claim or the lease and // publishes nothing. func (a ClaimAssembly) WritersStopped(ctx context.Context, host string) (ClaimWritersDrained, WritersStoppedEvidence, error) { read := func(ctx context.Context, host string) (hostclaim.Observed, error) { return hostclaim.Read(ctx, a.Claims, host) } return a.writersStopped(ctx, host, read, func(ctx context.Context, req WritersStoppedRequest) (WritersStoppedEvidence, error) { return CollectWritersStopped(ctx, controlsock.Path, req) }) } func (a ClaimAssembly) writersStopped(ctx context.Context, host string, read func(context.Context, string) (hostclaim.Observed, error), collect func(context.Context, WritersStoppedRequest) (WritersStoppedEvidence, error)) (ClaimWritersDrained, WritersStoppedEvidence, error) { if a.Wake == nil || read == nil || collect == nil { return ClaimWritersDrained{}, WritersStoppedEvidence{}, ErrClaimStopUnavailable } att := a.Wake.get(host) if att == nil { return ClaimWritersDrained{}, WritersStoppedEvidence{}, ErrClaimStopUnavailable } drained, err := att.StopWriters(ctx) if err != nil { return ClaimWritersDrained{}, WritersStoppedEvidence{}, err } observed, err := read(ctx, host) if err != nil { return ClaimWritersDrained{}, WritersStoppedEvidence{}, err } if err := drained.For(observed); err != nil { return ClaimWritersDrained{}, WritersStoppedEvidence{}, err } local, err := a.localStopIdentity(ctx) if err != nil { return ClaimWritersDrained{}, WritersStoppedEvidence{}, err } t := drained.Target if t.Box != local.Actor.Box || t.Enrollment != local.Actor.Enrollment || t.BootID != local.BootID || t.ControllerID != local.ControllerID { return ClaimWritersDrained{}, WritersStoppedEvidence{}, hostbox.ErrChanged } revision := t.DirectoryRevision if revision == "" { revision = local.Actor.Binding } rosterRevision := t.RosterRevision if rosterRevision == 0 { rosterRevision = local.Actor.Revision } att.mu.Lock() hints := slices.Clone(att.replicatorFDs) att.mu.Unlock() req := WritersStoppedRequest{Region: local.Actor.Region, Host: host, Incarnation: t.Incarnation, InstallationStamp: t.Stamp, WriterSession: t.Session, Box: t.Box, Enrollment: t.Enrollment, BootID: t.BootID, ControllerID: t.ControllerID, ExpectedClaimETag: observed.ETag, ExpectedDirectoryRevision: revision, RosterRevision: rosterRevision, Replicator: hints} ev, err := collect(ctx, req) if err != nil { return ClaimWritersDrained{}, WritersStoppedEvidence{}, err } return drained, ev, nil }