package host import ( "context" "encoding/json" "errors" "fmt" "github.com/carlosframework/platform/internal/hostactors" execlease "github.com/carlosframework/platform/internal/hostexec" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootparticipation" "os" "path/filepath" "slices" "time" ) type guardTransaction struct { Roster json.RawMessage `json:"roster"` ID string `json:"id"` Actor string `json:"actor"` Enrollment string `json:"enrollment"` } func (a *systemApplier) guarded() bool { return a.floor != nil } func (a *systemApplier) activeFloor() (bool, error) { if !a.guarded() { return false, nil } p, _, err := a.floor.Load() return p.Mode != "unpublished", err } func (a *systemApplier) guardImage(path string) error { _, err := a.guardImageHash(path) return err } // guardImageHash is guardImage, returning the verified image's SHA-256 // ("" when no floor guards this box). func (a *systemApplier) guardImageHash(path string) (string, error) { if !a.guarded() { return "", nil } open := hostfloor.OpenProtected if a.guardOpen != nil { open = a.guardOpen } f, err := open(path) if err != nil { return "", err } defer f.Close() hash, err := a.floor.VerifyFile(f, a.platform) if err != nil { return "", err } p, _, err := a.floor.Load() if err != nil || p.Mode == "unpublished" { return hash, err } tx, err := a.guardTransaction() if err != nil { if os.IsNotExist(err) { if j, e := readSystemJournal(a.optDir); e == nil && j == nil && path == systemBinaryPath(a.optDir) { return hash, nil } } return "", err } var roster hostactors.Roster if release.VerifyRootEnvelope(a.floor.PublicKey, tx.Roster, &roster) != nil || roster.Validate(a.floor.Bucket) != nil { return "", hostfloor.ErrFloor } reservation, ok := roster.Activations[tx.ID] _, floor, err := a.floor.Load() if err != nil { return "", err } if !ok || reservation.FloorSHA256 != floor.Digest || reservation.Actors[tx.Actor] != tx.Enrollment || !slices.Contains(reservation.Hashes[tx.Actor], hash) { return "", hostfloor.ErrFloor } return hash, nil } // guardTarget is the transaction's target image: the middle of the three // hashes prepareGuard selected the reservation by, read from the signed // roster root persisted with the transaction. func (a *systemApplier) guardTarget(tx guardTransaction) (string, error) { var roster hostactors.Roster if release.VerifyRootEnvelope(a.floor.PublicKey, tx.Roster, &roster) != nil || roster.Validate(a.floor.Bucket) != nil { return "", hostfloor.ErrFloor } hashes := roster.Activations[tx.ID].Hashes[tx.Actor] if len(hashes) != 3 || hashes[1] == "" { return "", hostfloor.ErrFloor } return hashes[1], nil } // guardFinished reports whether a transaction with no journal open has // finished: the verified image on disk is its target, or root's marker // records that it rolled back (Opus re-review M1-4). A transaction that is // neither never changed the box: it was refused or deferred before its // first destructive step, or abandoned at the rename. func (a *systemApplier) guardFinished(tx guardTransaction) (bool, error) { hash, err := a.guardImageHash(systemBinaryPath(a.optDir)) if err != nil { return false, err } begun, err := hostfloor.ReadBegun(a.floor.Dir) if err == nil { if begun.ID != tx.ID { return false, hostfloor.ErrFloor } if begun.RolledBack { return true, nil } } else if !errors.Is(err, os.ErrNotExist) { return false, err } target, err := a.guardTarget(tx) if err != nil { return false, err } return hash == target, nil } // settleGuard closes a transaction with no journal open. A finished one is // completed, once no controller is active, as before. One that never // changed the box records no completion: root withdraws its own marker and // transaction, so neither wedges the box nor reports the old image as the // result, and the console's reservation stays held for the agent's retry // or an operator. The withdrawal does not wait for controllers: it gives up // only root's intent, on a box it did not change. func (a *systemApplier) settleGuard(ctx context.Context, tx guardTransaction, beforeCompletion func() error) error { finished, err := a.guardFinished(tx) if err != nil { return err } if !finished { // The marker goes inside the settlement, after root's census, so a // census that refuses leaves it with the rest of the transaction. return a.floor.EndParticipation(ctx, tx.ID, func() error { if err := hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "update-begun.json")); err != nil { return err } return hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "activation.json")) }) } if beforeCompletion != nil { if err := beforeCompletion(); err != nil { return err } } // Completion is durable before the marker disappears. Replaying either // write is harmless; readers never report completion with a marker present. if err = hostfloor.Durable(filepath.Join(a.floor.Dir, "completed.json"), []byte(tx.ID), 0644); err != nil { return err } return a.floor.EndParticipation(ctx, tx.ID, func() error { return hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "update-begun.json")) }) } func (a *systemApplier) guardTransaction() (guardTransaction, error) { raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "activation.json")) if err != nil { return guardTransaction{}, err } var tx guardTransaction if json.Unmarshal(raw, &tx) != nil || tx.ID == "" || tx.Enrollment == "" || tx.Actor == "" { return guardTransaction{}, hostfloor.ErrFloor } return tx, nil } func (a *systemApplier) beginGuard(ctx context.Context, target *release.SystemPointer) error { if !a.guarded() { return nil } p, _, err := a.floor.Load() if err != nil { return err } if p.Mode == "unpublished" { return nil } if err := a.prepareGuard(ctx, target); err != nil { return err } tx, err := a.guardTransaction() if err != nil { return err } // The signed reservation is persisted by the root approval step, never by // the worker's request. A new transaction must still find it in the roster. store, pub, err := a.openStore(ctx) if err != nil { return err } _, local, err := a.floor.Load() if err != nil { return err } remote, err := hostfloor.Read(ctx, store, pub, a.floor.Bucket) if err != nil || remote.Digest != local.Digest { return hostfloor.ErrFloor } d := hostactors.Directory{Store: store, PublicKey: pub, Bucket: a.floor.Bucket} live, err := d.Read(ctx) if err != nil { return err } h, ok := live.Roster.Activations[tx.ID] if !ok || h.Actors[tx.Actor] != tx.Enrollment { return hostfloor.ErrFloor } var saved hostactors.Roster if release.VerifyRootEnvelope(pub, tx.Roster, &saved) != nil { return hostfloor.ErrFloor } x, _ := json.Marshal(h) y, _ := json.Marshal(saved.Activations[tx.ID]) if string(x) != string(y) { return hostfloor.ErrFloor } return a.floor.Begin(ctx, tx.ID, a.dataDir) } // checkGuardRecovery runs with the apply lock held. No-journal markers can // only precede the first destructive step or follow durable journal removal. // Both are terminal after the selected current image has been verified. func (a *systemApplier) checkGuardRecovery(ctx context.Context) error { if !a.guarded() { return a.checkUnarmedParticipationRecovery(ctx) } p, _, err := a.floor.Load() if err != nil { return err } if p.Mode == "unpublished" { return a.checkUnarmedParticipationRecovery(ctx) } j, err := readSystemJournal(a.optDir) if err != nil { return err } marker := filepath.Join(a.floor.Dir, "update-begun.json") raw, err := os.ReadFile(marker) if errors.Is(err, os.ErrNotExist) && j == nil { tx, txErr := a.guardTransaction() if errors.Is(txErr, os.ErrNotExist) { return nil } if txErr != nil { return txErr } completed, readErr := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")) if readErr == nil && string(completed) == tx.ID { // Completion recorded and root's marker gone. With root's update // record gone too the transaction is settled: a replay has // nothing left to close, and runs no census. open, err := a.updateRecordOpen(ctx) if err != nil || !open { return err } return a.settleWithoutJournal(ctx, tx.ID, a.floor.EndParticipation(ctx, tx.ID, nil)) } if readErr != nil && !errors.Is(readErr, os.ErrNotExist) { return readErr } return a.settleWithoutJournal(ctx, tx.ID, a.settleGuard(ctx, tx, func() error { return a.controllersRecovered(ctx) })) } if err != nil { return err } tx, err := a.guardTransaction() if err != nil { return err } var begun struct { ID string `json:"id"` } if json.Unmarshal(raw, &begun) != nil || begun.ID != tx.ID { return hostfloor.ErrFloor } if j == nil { return a.settleWithoutJournal(ctx, tx.ID, a.settleGuard(ctx, tx, func() error { return a.controllersRecovered(ctx) })) } if err := a.controllersRecovered(ctx); err != nil { return err } if j.Phase == systemPhaseBackingUp { return a.guardImage(systemBinaryPath(a.optDir)) } return a.guardImage(systemPrevPath(a.optDir)) } // errSettlementDeferred marks a settlement that root's generation census // refused with no journal open. No apply is half done and the files are // whole, so there is nothing for recovery to restore and no reason to keep // the edge down (its drop-in Requires= the recover unit). The transaction // evidence stays as it was: the reservation, root's marker and update // record, so every new claim acquisition stays refused, and the next apply, // or recover on the next edge start, settles it. Run under --recover logs // it and exits 0; the apply returns it as before. beginExecutableChange // treats a refused census as a deferral too (systemparticipation.go). var errSettlementDeferred = errors.New("participation settlement deferred") // errAcquisitionBarrier marks a deferral that could not hold claim // acquisitions refused (Ledger.HoldUpdate failed). It is never swallowed: // recover must not exit 0 with acquisitions open (Astra rounds 2 and 3). var errAcquisitionBarrier = errors.New("acquisition barrier") // settleWithoutJournal is the result of one settlement made with no // journal open: the pending-controller recovery, the completion or // withdrawal, and the end of root's update record, whose census runs again // inside EndParticipation; and, after an apply's journal is durably // removed, the settlement that follows it (closeGuardJournal). A result // that is only a census refusal, on a box with root participation, is // wrapped in errSettlementDeferred, once acquisitions are held refused: // root's update record is kept, or, where the refused census was the one // that would have begun it, written under barrier's id (Ledger.HoldUpdate). // Anything else is returned unchanged: an image that does not verify, // malformed state, a live or unreadable controller, a durable write that // failed, a barrier that could not be written, and every refusal on a box // without a census, which sees exactly main's recovery (ruling 6). func (a *systemApplier) settleWithoutJournal(ctx context.Context, barrier string, err error) error { if err == nil || !onlyCensusRefusal(err) || ctx.Err() != nil { return err } required, checkErr := a.participationDisk().ParticipationRequired(ctx) if checkErr != nil { // Whether a barrier is needed could not be established (the store // did not open, a retained census did not read): that is a barrier // failure too, never a plain refusal a caller may log (Astra // round 4). Both errors are kept. return errors.Join(err, fmt.Errorf("%w: participation required: %w", errAcquisitionBarrier, checkErr)) } if !required { return err } if holdErr := a.participationDisk().WithParticipation(ctx, func(ctx context.Context, l rootparticipation.Ledger) error { return l.HoldUpdate(ctx, barrier) }, func() error { return rootparticipation.ErrUnavailable }); holdErr != nil { return errors.Join(err, fmt.Errorf("%w: %w", errAcquisitionBarrier, holdErr)) } return fmt.Errorf("%w: no journal is open, so the files are whole; root's update record holds claim acquisitions refused until the settlement passes: %w", errSettlementDeferred, err) } // updateRecordOpen reports whether root's update record is there, on a box // with root participation. Without one it is false: main's legacy path. func (a *systemApplier) updateRecordOpen(ctx context.Context) (bool, error) { open := false err := a.participationDisk().WithParticipation(ctx, func(_ context.Context, l rootparticipation.Ledger) error { _, err := l.UpdateID() if errors.Is(err, os.ErrNotExist) { return nil } open = err == nil return err }, func() error { open = true; return nil }) return open, err } // deferSettlement reports whether Run, under --recover, leaves err's // settlement to a later run instead of failing: only an errSettlementDeferred. // It logs why. func (a *systemApplier) deferSettlement(recover bool, err error) bool { if !recover || !errors.Is(err, errSettlementDeferred) { return false } a.ops.logf("system apply: recover: %v; the edge may start, and the next apply or recover settles it", err) return true } // settleMarker settles the update-begun marker at the end of a run // (settleUpdateBegun) and decides what its outcome means for the run: a // deferral under --recover is logged and left to a later run; a barrier // that could not be written joins the run's error, whatever the mode, // because acquisitions would otherwise be open with the settlement // unmade; a deferral under the apply is logged, as every marker failure // was on main, with acquisitions held refused by the barrier. func (a *systemApplier) settleMarker(ctx context.Context, recover bool, retErr *error) { err := a.settleUpdateBegun(ctx) if err == nil || a.deferSettlement(recover, err) { return } if errors.Is(err, errAcquisitionBarrier) { *retErr = errors.Join(*retErr, err) return } a.ops.logf("system apply: %v", err) } func (a *systemApplier) endGuard(ctx context.Context) error { if !a.guarded() { return nil } p, _, err := a.floor.Load() if err != nil || p.Mode == "unpublished" { return err } tx, err := a.guardTransaction() if err != nil { return err } if j, err := readSystemJournal(a.optDir); err != nil { return err } else if j != nil { return hostfloor.ErrFloor } return a.settleGuard(ctx, tx, nil) } func (a *systemApplier) controllersIdle(ctx context.Context) error { return a.participationDisk().ControllersIdle(ctx, a.dataDir) } var recoverPendingControllers = recoverRootExecution // controllersRecovered lets root recover a provably dead pending controller // before refusing, whether or not a journal is open; a live or unreadable one // still refuses. func (a *systemApplier) controllersRecovered(ctx context.Context) error { err := a.controllersIdle(ctx) if err == nil { return nil } guardDir := systemGuardDir(a.optDir) if a.floor != nil { guardDir = a.floor.Dir } required, checkErr := a.participationDisk().ParticipationRequired(ctx) if checkErr != nil { return errors.Join(err, checkErr) } recover := recoverPendingControllers if !required { recover = execlease.RecoverPending } if recoveryErr := recover(ctx, a.dataDir, guardDir); recoveryErr != nil { return errors.Join(err, recoveryErr) } return a.controllersIdle(ctx) } func (a *systemApplier) closeGuardJournal(ctx context.Context) error { required, err := a.participationDisk().ParticipationRequired(ctx) if err != nil { return err } if !required { if !a.guarded() { a.removeJournal() return nil } if err := hostfloor.RemoveDurable(a.file("journal.json")); err != nil { return err } return a.endGuard(ctx) } active, err := a.activeFloor() if err != nil { return err } if err := hostfloor.RemoveDurable(a.file("journal.json")); err != nil { return err } // The journal is gone and the files are whole: from here a settlement // the census refuses is deferred as with no journal open. if !active { return a.settleWithoutJournal(ctx, "recover-legacy", a.clearUpdateBegun(ctx)) } err = a.endGuard(ctx) if err == nil || !onlyCensusRefusal(err) { return err } tx, txErr := a.guardTransaction() if txErr != nil { return errors.Join(err, txErr) } return a.settleWithoutJournal(ctx, tx.ID, err) } func (a *systemApplier) guardRun(ctx context.Context, wait time.Duration, name string, args ...string) (string, error) { if id, required, err := a.rootMoves.read(); err != nil { return "", err } else if required { binary, err := readRootFloorBinary(a.optDir, name) if err != nil { return "", err } floor, err := readEffectiveMovesFloor(a.rootMoves, a.optDir, a.dataDir) if err != nil { return "", err } if !binary.supports(floor) { repair := name == systemBinaryPath(a.optDir) || name == filepath.Join(systemGuardDir(a.optDir), "carlos") if name == systemPrevPath(a.optDir) && rootFloorAllowsPrevIdentity(a.optDir, id) == nil { repair = true } if !binary.supports(nil) || !repair { return "", errRootFloorHistory } rootFloorAlert() } } if !a.guarded() { return a.ops.run(ctx, wait, name, args...) } if err := a.guardImage(name); err != nil { return "", err } bounded, cancel := context.WithTimeout(ctx, wait) defer cancel() return a.floor.Run(bounded, name, a.platform, args...) } func installFloorHook(a *systemApplier) error { dir := systemGuardDir(a.optDir) fi, err := os.Lstat(dir) if errors.Is(err, os.ErrNotExist) { return nil } if err != nil { return err } if !fi.IsDir() { return hostfloor.ErrFloor } entries, err := os.ReadDir(dir) if err != nil { return err } // Main's participation marker predates floor arming, is written through // update-begun.json.tmp, and leaves an empty directory after completion. // A torn or leftover write of it is still legacy state; only a marker // naming a floor transaction ("id") or a floor artifact needs a policy. if legacyGuardEntries(dir, entries) { return nil } d, err := floorDisk(a.optDir) if err != nil { return err } a.floor = &d _, _, err = d.Load() return err } func (a *systemApplier) prepareGuard(ctx context.Context, target *release.SystemPointer) error { enrollment, err := hostfloor.ReadEnrollment(filepath.Join(a.etcDir, "carlos", "host-actor.json")) if err != nil { return err } store, pub, err := a.openStore(ctx) if err != nil { return err } raw, _, err := store.Get(ctx, hostactors.Key) if err != nil { return err } var roster hostactors.Roster if release.VerifyRootEnvelope(pub, raw, &roster) != nil || roster.Validate(a.floor.Bucket) != nil { return hostfloor.ErrFloor } b, ok := roster.Boxes[enrollment.Box] if !ok || b.Enrollment != enrollment.Enrollment || b.Cloud.InstanceID != enrollment.InstanceID { return hostfloor.ErrFloor } actor := "box:" + enrollment.Box id := "" for name, h := range roster.Activations { if h.Publication != nil && target.ActivationID != name { continue } hashes := h.Hashes[actor] if h.Actors[actor] == enrollment.Enrollment && len(hashes) == 3 && hashes[1] == target.Current.Artifacts[a.platform].SHA256 { if id != "" { return hostfloor.ErrFloor } id = name } } if id == "" { return hostfloor.ErrFloor } identityKey, err := release.IdentityKey(target.Current.ReleaseTag) if err != nil { return err } identity, _, err := store.Get(ctx, identityKey) if err != nil { return err } _, floor, err := a.floor.Load() if err != nil { return err } if err = hostfloor.CheckImage(pub, floor, identity, target.Current.Version, a.platform, target.Current.Artifacts[a.platform].SHA256); err != nil { return err } if err = hostfloor.Durable(filepath.Join(a.floor.Dir, "identities", target.Current.ReleaseTag+".json"), identity, 0644); err != nil { return err } txRaw, _ := json.Marshal(guardTransaction{raw, id, actor, enrollment.Enrollment}) return hostfloor.Durable(filepath.Join(a.floor.Dir, "activation.json"), txRaw, 0644) } // legacyGuardEntries reports whether the guard directory holds nothing but // main's region participation marker, its temporary file, or both, with no // floor transaction ID in the marker. func legacyGuardEntries(dir string, entries []os.DirEntry) bool { for _, e := range entries { if !e.Type().IsRegular() || (e.Name() != updateBegunFile && e.Name() != updateBegunFile+".tmp") { return false } if e.Name() == updateBegunFile { raw, err := os.ReadFile(filepath.Join(dir, e.Name())) if err != nil { return false } var marker map[string]json.RawMessage if json.Unmarshal(raw, &marker) == nil { if _, ok := marker["id"]; ok { return false } } } } return true }