package host import ( "context" "crypto/ed25519" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "github.com/carlosframework/platform/internal/genselect" "log" "os" "path/filepath" "strings" "sync" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/controlsock" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfence" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootstate" ) var openClaimRootAuthority = func() (rootstate.Store, error) { return rootstate.Open("/var/lib/carlos-claims/authority/fences", true, false) } type ClaimRuntimeResources struct { Mode string Registry *registry.Registry Manager *cgroup.Manager DataDir, OptDir, UnitsDir string // MigrationWindow is journal-migrate's --window: the attended offline // window's directory, whose held unit files let the migration census // accept that window's own masks. Empty everywhere else. MigrationWindow string } // ClaimOwnerBindings are the provider's typed owners. Serving feeds the // claim's own serving input, admission and publication owners. Moves is the // moves integration's owner set for moving, landing, reverting and restoring, // bound only with CARLOS_MOVES_PHASE1 on (claim Plan 9 D5). type ClaimOwnerBindings struct { moveRead *claimMoveReadAccess wakeUnavailable error cleanupEnrollment *claimCleanupEnrollment RetirementAuthority *hostops.ProductionAuthority EvidenceKey ed25519.PrivateKey Identity hostbox.IdentityProvider Local hostbox.LocalAuthority Controllers StopControllerReader Data lease.VersionedStore // Claims-off callers keep the legacy owners. Claimed serving has its own // bindings; the moves flag extends that dispatch with operation owners. Inputs ClaimWakeInputs AdmissionEvidence func(context.Context, hostclaim.Observed, hostclaim.Admission, *genselect.Plan) error PublicationEvidence func(context.Context, hostclaim.Observed) error Serving ClaimServingSources Moves ClaimMovesOwners InputPolicy func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error MovesPolicy func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error Evidence hostbox.RegionalEvidence LeaseSet hostbox.LeaseSet View func(context.Context, hostclaim.Observed) (park.View, error) Authorize func(context.Context, hostclaim.Observed, hostclaim.Obligation) error Close func(context.Context) error // Relocation is region HA's typed owner set for relocating, bound only // with CARLOS_REGION_HA and the moves C1 integration on (claim Plan 11). Relocation ClaimRelocationOwners // PolicyRefusals are append-only policy checks (claim Plan 11, R8 as // accepted): each runs after the whole base policy passed and can only // refuse. They are fixed here, before the box identity is verified, and // nothing later can remove or replace one. PolicyRefusals []hostbox.PolicyRefusal } type ClaimOwnerProvider interface { OpenClaimOwners(context.Context, release.Store, ClaimRuntimeResources) (ClaimOwnerBindings, error) } var claimOwnerRegistration struct { sync.Mutex Provider ClaimOwnerProvider } // The Plan 8/8b and moves integration registers its typed adapters once at // process startup. It cannot replace local construction or add a permissive fallback. func RegisterProductionClaimOwners(p ClaimOwnerProvider) error { claimOwnerRegistration.Lock() defer claimOwnerRegistration.Unlock() if p == nil || claimOwnerRegistration.Provider != nil { return hostwrite.ErrUnavailable } claimOwnerRegistration.Provider = p return nil } func productionClaimOwners() ClaimOwnerProvider { claimOwnerRegistration.Lock() defer claimOwnerRegistration.Unlock() return claimOwnerRegistration.Provider } func withClaimRuntimeResources(ctx context.Context, r ClaimRuntimeResources) context.Context { entry, _ := ctx.Value(claimEntryKey{}).(claimEntry) entry.Resources = r return context.WithValue(ctx, claimEntryKey{}, entry) } func ensureProductionClaimAssembly(ctx context.Context) (context.Context, error) { entry, _ := ctx.Value(claimEntryKey{}).(claimEntry) if !entry.Enabled { return ctx, nil } if entry.Assembly != nil { return ctx, nil } if entry.Resources.Registry == nil || entry.Resources.Mode == "" { return ctx, &ClaimDependencyError{Contracts: []string{"box registry and mode-specific runtime resources"}} } provider := productionClaimOwners() if provider == nil { if err := RegisterProductionClaimOwners(ProductionClaimOwnerProvider{}); err != nil { return ctx, err } provider = productionClaimOwners() } assembly, err := OpenProductionClaimAssembly(ctx, entry.Store, entry.Resources, provider) if err != nil { return ctx, err } entry.Assembly = assembly return context.WithValue(ctx, claimEntryKey{}, entry), nil } func OpenProductionClaimAssembly(ctx context.Context, store release.Store, r ClaimRuntimeResources, provider ClaimOwnerProvider) (result *ClaimAssembly, resultErr error) { if ctx == nil || store == nil || provider == nil || r.Registry == nil || (r.Mode != "agent" && r.Mode != "adopt" && r.Mode != "instance-control") || !filepath.IsAbs(r.DataDir) || !filepath.IsAbs(r.OptDir) || !filepath.IsAbs(r.UnitsDir) { return nil, hostwrite.ErrUnavailable } if err := claimActivationPreflight(r.Registry); err != nil { return nil, err } if err := claimActivationVerified(ctx, store, r); err != nil { return nil, err } owners, err := provider.OpenClaimOwners(ctx, store, r) if err != nil { return nil, err } if owners.Close == nil { return nil, hostwrite.ErrUnavailable } defer func() { if resultErr != nil { resultErr = errors.Join(resultErr, owners.Close(context.WithoutCancel(ctx))) } }() movesC1, err := claimMovesC1(claimEntryEnv(ctx), owners.Moves) if err != nil { owners.wakeUnavailable = errors.Join(owners.wakeUnavailable, &ClaimDependencyError{Contracts: []string{"moves wake owners and operation attempts"}}, err) movesC1 = false } if owners.Local == nil || owners.Controllers == nil || owners.Data == nil || owners.InputPolicy == nil || owners.MovesPolicy == nil || owners.Evidence == nil || owners.LeaseSet == nil || owners.View == nil || owners.Authorize == nil { return nil, hostwrite.ErrUnavailable } servingEnabled := hostwrite.Enabled(claimEntryEnv(ctx)) if servingEnabled { if err := checkClaimServingSources(owners.Serving); err != nil { owners.wakeUnavailable = errors.Join(owners.wakeUnavailable, err) } } else if owners.Inputs == nil || owners.AdmissionEvidence == nil || owners.PublicationEvidence == nil { return nil, hostwrite.ErrUnavailable } callerCtx := ctx ctx, finishStartup := context.WithTimeout(ctx, 2*time.Minute) defer finishStartup() actor, err := owners.actorForConstruction(ctx, r.Mode) if err != nil { return nil, err } stop, err := owners.claimEvidenceCoordinator(ctx, store, r.Mode, actor) if err != nil { return nil, err } if r.Mode == "agent" && r.Manager == nil { return nil, hostwrite.ErrUnavailable } local := filepath.Join(r.DataDir, ".hostclaims") if r.Mode == "agent" { if err = prepareClaimStateDirectory(local); err != nil { return nil, err } } var launches *hostbox.LaunchJournal var inventory *hostbox.StopInventoryDB var journals *hostbox.JournalService if r.Mode != "agent" { // The legacy journal check may wait on the migration lock while an // operator runs the offline import (carlos system journal-migrate), so // it keeps caller cancellation rather than construction's two minutes. finishStartup() ctx = callerCtx launches, err = openClaimRootJournal() if err != nil { return nil, err } journals, err = hostbox.NewJournalService(launches) if err != nil { launches.Close() return nil, err } inventory = journals.Inventory if err = checkClaimJournalMigration(ctx, journals, local); err != nil { launches.Close() return nil, err } ctx, finishStartup = context.WithTimeout(callerCtx, 2*time.Minute) defer finishStartup() fresh, refreshErr := owners.actorForConstruction(ctx, r.Mode) if refreshErr != nil || fresh.Box != actor.Box || fresh.Enrollment != actor.Enrollment || fresh.Bucket != actor.Bucket { launches.Close() return nil, errors.Join(refreshErr, hostwrite.ErrUnavailable) } actor = fresh if err := recoverRootInventoryStartup(ctx, store, r, owners, actor, journals, launches); err != nil { launches.Close() return nil, err } if err := verifyRootMigrationStartup(ctx, rootFloorIdentity{Deployment: actor.ControlBucket, Box: actor.Box, Enrollment: actor.Enrollment}, journals.Inventory); err != nil { launches.Close() return nil, err } stop, err = owners.claimEvidenceCoordinator(ctx, store, r.Mode, actor) if err != nil { launches.Close() return nil, err } } else { launches, inventory = hostbox.RemoteJournals(rootJournalClient(controlsock.Path)) } defer func() { if resultErr != nil { resultErr = errors.Join(resultErr, inventory.Close(), launches.Close()) } }() boot, err := os.ReadFile("/proc/sys/kernel/random/boot_id") if err != nil { return nil, err } processStart, err := claimConstructionProcessStart() if err != nil { return nil, err } controller := "" if r.Mode == "agent" { controller, err = owners.Controllers.VerifiedController(ctx, os.Getpid()) if err != nil || controller == "" { return nil, errors.Join(err, hostwrite.ErrUnavailable) } } newID := func() string { var b [16]byte if _, err := rand.Read(b[:]); err != nil { return "" } return hex.EncodeToString(b[:]) } roots := map[string]string{"data": r.DataDir, "opt": r.OptDir, "units": r.UnitsDir} guard := hostguard.Guard{DataDir: r.DataDir, Claims: hostguard.StoreReader{Store: store}} var authority rootstate.Store if r.Mode == "agent" { guard.FenceView = rootFenceViewClient(rootJournalClient(controlsock.Path)) } else { authority, err = openClaimRootAuthority() if err != nil { return nil, err } defer func() { if resultErr != nil { _ = authority.Close() } }() guard.Root = hostguard.NewRootAuthorityWithInventory(authority, func(ctx context.Context, host string, c hostguard.Captured) (uint64, error) { _, revision, err := inventory.Snapshot(ctx, host, c) return revision, err }) } producers := &ClaimProducerSet{CloseWriters: launches.CloseWriters} inputs, admissionEvidence, publicationEvidence := owners.Inputs, owners.AdmissionEvidence, owners.PublicationEvidence var serving *claimServingOwners var publisher ClaimFencedPublication var groupD claimGroupD if servingEnabled { serving = &claimServingOwners{HomeRegion: actor.HomeRegion, Data: hostbox.RequestStore{Store: owners.Data, Check: func(ctx context.Context) error { _, err := owners.Identity.Verify(ctx); return err }}, Bucket: actor.Bucket, Region: park.PlacementRegion(actor.Region), Box: actor.Box, NewID: newID, Now: time.Now, Sources: owners.Serving} serving.emptyRecovery = rootEmptyRecovery(guard) if files, ok := serving.Sources.Files.(*movesFiles); ok { files.emptyRecovery = serving.emptyRecovery } dispatch := ClaimOwnerDispatch{Serving: serving.owners()} if movesC1 { groupD, err = claimGroupDWiring(claimEntryEnv(ctx), serving.owners(), owners.Moves, owners.Relocation) if err != nil { return nil, err } dispatch = groupD.Dispatch } inputs, admissionEvidence, publicationEvidence = dispatch, dispatch.AdmissionEvidence, dispatch.PublicationEvidence publisher = dispatch } wake := &ClaimWakeAssembly{Ready: claimSocketReady, Manager: r.Manager, Inputs: inputs, LeaseRegion: actor.Cloud.Region, Publication: publisher} wake.RequireServingEnrollment = servingEnabled wake.Producers = func(ctx context.Context, o hostclaim.Observed) (*ClaimProducers, error) { if o.Claim == nil || o.Claim.Holder == nil || controller == "" { return nil, hostwrite.ErrUnavailable } h := o.Claim.Holder target := hostguard.FenceTarget{Host: o.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Op: o.Claim.Op.ID, Phase: o.Claim.Op.Phase, Box: actor.Box, Region: actor.Region, Enrollment: actor.Enrollment, BootID: strings.TrimSpace(string(boot)), ControllerID: controller, ClaimETag: o.ETag, DirectoryRevision: actor.Binding, RosterRevision: actor.Revision} return producers.Ensure(target, processStart, launches.RecordProducer, launches.CompleteProducer) } wake.Roles = &ClaimRoleAssembly{Guard: guard, Claims: store, Manager: r.Manager, Journal: launches, Inventory: inventory, ManagedRoots: roots, Identity: owners.Identity, BootID: strings.TrimSpace(string(boot)), ControllerID: controller} if r.Mode == "agent" { wake.Roles.Root = NewClaimRoleRootClient(controlsock.Path) } wake.AfterAdmission = coldAdmissionHook(guard, owners.Identity, controlsock.Path) wake.RetireArtifacts = func(context.Context, hostclaim.Observed, ClaimLaunchInputs, string) error { return hostbox.ErrChanged } // Region HA N6 is wired whenever claim mode is on, whatever // CARLOS_REGION_HA says (ruling 82's reasoning): it serves a relocation // that already completed, and without a completion record it does nothing. completed := &relocationCompletedHandover{Guard: guard, Identity: owners.Identity, Socket: controlsock.Path, Box: actor.Box} handover := claimCrossBoxHandover{RecoveryRebind: recoveryRebindClient(controlsock.Path), Guard: guard, Evidence: store, Box: actor.Box, Inventory: inventory, Identity: owners.Identity, Registry: r.Registry, Launches: launches} if owners.RetirementAuthority != nil { handover.PublicKey = owners.RetirementAuthority.Directory.PublicKey handover.Directory = owners.RetirementAuthority.Directory handover.FenceAuthority = hostfence.ClaimFenceAuthority{Directory: owners.RetirementAuthority.Directory, Now: time.Now} } var relocation *ClaimRelocationOwners if movesC1 && groupD.Dispatch.Relocation != nil { relocation = &owners.Relocation } wireProductionAdmission(wake, actor.Box, movesC1, handover, relocation, completed) if movesC1 { retirement := park.Local{Authorize: func(ctx context.Context, c *hostclaim.Claim, o hostclaim.Obligation) error { return owners.Authorize(ctx, hostclaim.Observed{Host: c.Host, Claim: c}, o) }, Guard: guard, Claims: store, DataDir: r.DataDir, Box: actor.Box} wake.RetireArtifacts = claimArtifactRetirement{Local: retirement, Continue: hostclaim.SessionContinues}.Retire wake.RecoverArtifacts = claimArtifactRetirement{Local: retirement, Continue: hostclaim.SessionContinues}.Recover wake.Continue = claimWakeContinue wake.Operations = groupD.Operations wake.ProbeReady = groupD.ProbeReady } if servingEnabled { retirement := &claimServingArtifactRetirement{Claims: store, Guard: guard, DataDir: r.DataDir, Box: actor.Box, Region: actor.Region, Root: hostbox.ServingParkClient{Call: rootJournalClient(controlsock.Path)}} if err := bindServingArtifactRetirement(wake, retirement); err != nil { return nil, err } } wake.Retain = wake.retainCaptured wake.ObserveLoss = func(o hostclaim.Observed, err error) { log.Printf("host claim lease loss for %s: %v", o.Host, err) } wake.Retries = activator.ServingRetries{Claims: store, Box: actor.Box, Enrollment: actor.Enrollment, BootID: strings.TrimSpace(string(boot)), ControllerID: controller, Now: time.Now, NewID: newID} wake.Retries.Inputs = func(ctx context.Context, rt registry.Route) (string, error) { o, err := hostclaim.Read(ctx, store, rt.Host) if err != nil { return "", err } in, err := owners.Inputs.Select(ctx, rt, o, activator.StartPolicy{MaxStarts: 5, Window: 30 * time.Second}) if err != nil { return "", err } raw, err := json.Marshal(struct { Identity hostwrite.Identity Executable, Digest string Args, Env []string }{in.Identity, in.Executable, in.ExecutableSHA256, in.Args, in.Env}) if err != nil { return "", err } digest := sha256.Sum256(raw) return hex.EncodeToString(digest[:]), nil } if servingEnabled { wake.ServingPrerequisites = servingFilesPrerequisites(store, owners.Serving.Files) wake.Retries.BeforeReserve = func(ctx context.Context, rt registry.Route, o hostclaim.Observed) error { if err := ctx.Err(); err != nil { return err } if err := wake.ServingPrerequisites(ctx, rt, o); err != nil { return err } return claimServingRoom(r.DataDir) } wake.Retries.Inputs = serving.launchDigest } // ClaimAssembly.Build binds Retries.StopAttempt to the attempt's persisted // boot and controller tuple; a wake runtime cannot be wrapped without it. reconcile := &hostbox.ReconcileAdapter{Registry: r.Registry, Inventory: inventory, Retirement: hostbox.TypedRetirement{JournalDir: claimRetireJournalDir(r.Mode, r.DataDir)}} // Region HA R5-3 (Task 10b): the source's row held at its fence's // close, with CARLOS_REGION_HA on only. Build copies the adapter and // sets the reconciler's other fields; it keeps these two. relocationSourceRows(claimEntryEnv(ctx), &reconcile.Reconciler, guard, r.Registry) stop.Guard = guard stop.Claims = hostguard.StoreReader{Store: store} stop.Inventory = inventory stop.NewID = newID assembly := &ClaimAssembly{moveRead: owners.moveRead, wakeUnavailable: owners.wakeUnavailable, Journals: journals, cleanupEnrollment: owners.cleanupEnrollment, RetirementAuthority: owners.RetirementAuthority, SharedPaths: claimGlobalSharedPaths(r), Mode: r.Mode, Producers: producers, Launches: launches, ManagedRoots: roots, Reconcile: reconcile, RootJournal: launches, Controllers: owners.Controllers, Claims: store, Guard: guard, Identity: owners.Identity, Local: owners.Local, Policy: hostbox.Policy{Inputs: owners.InputPolicy, Moves: owners.MovesPolicy, Relocation: groupD.Relocation, Held: hostbox.RegistryHoldPolicy(r.Registry)}.WithRefusals(owners.PolicyRefusals...), Data: owners.Data, Wake: wake, Stop: stop, Evidence: owners.Evidence, LeaseSet: owners.LeaseSet, View: owners.View, Authorize: owners.Authorize, AdmissionEvidence: admissionEvidence, PublicationEvidence: publicationEvidence, NewID: newID} if movesC1 && owners.RetirementAuthority != nil { assembly.LossAdmission = &hostbox.LossAdmission{PublicKey: owners.RetirementAuthority.Directory.PublicKey, Owner: owners.Moves.LossRecovery, Sources: owners.Moves.LossSources} } var closeMu sync.Mutex closed := false assembly.Close = func(ctx context.Context) error { closeMu.Lock() defer closeMu.Unlock() if closed { return nil } if r.Mode == "agent" { if err := wake.Close(ctx); err != nil { return err } if err := wake.closeRetained(ctx); err != nil { return err } } if err := owners.Close(ctx); err != nil { return err } closed = true var rootErr error if authority != nil { rootErr = authority.Close() } return errors.Join(rootErr, inventory.Close(), launches.Close()) } return assembly, nil } // claimGroupD is the moves C1 (claim Plan 9 Group D) owner wiring, with // region HA's relocation entry when it is bound (claim Plan 11). type claimGroupD struct { Dispatch ClaimOwnerDispatch Operations OperationAttempts ProbeReady ClaimProbeReadiness Relocation func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error } // claimGroupDWiring builds the Group D owners once moves C1 is on. With the // relocation owners unbound it is exactly Plan 9's wiring: the moves owner // set, the moves attempt journal and probe check, and no relocation policy. // Bound, the dispatcher gains the typed Relocation entry, a router sends // landing attempts to moves and relocating attempts to region HA, and the // relocation policy owner is set. func claimGroupDWiring(getenv func(string) string, serving ClaimKindOwners, moves ClaimMovesOwners, relocation ClaimRelocationOwners) (claimGroupD, error) { movesOwners := moves.ClaimKindOwners movesReady := moves.ProbeReady if movesReady == nil { movesReady = claimProbeSocketReady } out := claimGroupD{Dispatch: ClaimOwnerDispatch{Serving: serving, Moves: &movesOwners}, Operations: moves.Operations, ProbeReady: movesReady} bound, err := claimRelocationC1(getenv, true, relocation) if err != nil || !bound { return out, err } relocationOwners := relocation.ClaimKindOwners relocationReady := relocation.ProbeReady if relocationReady == nil { relocationReady = claimProbeSocketReady } out.Dispatch.Relocation = &relocationOwners out.Operations = claimOperationRouter{ hostclaim.KindLanding: {Attempts: moves.Operations, ProbeReady: movesReady}, hostclaim.KindReverting: {Attempts: moves.Operations, ProbeReady: movesReady}, hostclaim.KindRestoring: {Attempts: moves.Operations, ProbeReady: movesReady}, hostclaim.KindRelocating: {Attempts: relocation.Operations, ProbeReady: relocationReady}, } out.Relocation = relocation.Policy return out, nil } // claimRetireJournalDir is root's retirement and cold-handover journal // namespace. It never lives under the agent-owned data directory: root parks // only what its own journals and stop inventory name. The agent retires // through root's socket and never reads it. func claimRetireJournalDir(string, string) string { return "/var/lib/carlos-claims/retire" } func claimConstructionProcessStart() (string, error) { raw, err := os.ReadFile("/proc/self/stat") if err != nil { return "", err } end := strings.LastIndexByte(string(raw), ')') if end < 0 { return "", hostwrite.ErrUnavailable } fields := strings.Fields(string(raw)[end+1:]) if len(fields) < 20 { return "", hostwrite.ErrUnavailable } return fields[19], nil } type retainedClaimWork struct { Observed hostclaim.Observed Process activator.Proc Replica activator.Rep Cause error } func (w *ClaimWakeAssembly) retainCaptured(o hostclaim.Observed, p activator.Proc, r activator.Rep, err error) { w.mu.Lock() defer w.mu.Unlock() w.retained = append(w.retained, &retainedClaimWork{o, p, r, err}) } func (w *ClaimWakeAssembly) closeRetained(ctx context.Context) error { w.mu.Lock() saved := append([]*retainedClaimWork{}, w.retained...) w.mu.Unlock() for _, work := range saved { if work.Process != nil { if err := work.Process.Stop(ctx); err != nil { return err } } if work.Replica != nil { if err := work.Replica.Close(ctx); err != nil { return err } } w.mu.Lock() for i, item := range w.retained { if item == work { w.retained = append(w.retained[:i], w.retained[i+1:]...) break } } w.mu.Unlock() } return nil } func refusingRootWake(box string, store release.Store) *activator.ClaimRuntime { return &activator.ClaimRuntime{Box: box, Read: func(ctx context.Context, h string) (hostclaim.Observed, error) { return hostclaim.Read(ctx, store, h) }, StopForeign: func(context.Context, string) error { return hostwrite.ErrUnavailable }, Prerequisites: func(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) error { return hostwrite.ErrUnavailable }, Admit: func(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) (hostclaim.Observed, error) { return hostclaim.Observed{}, hostwrite.ErrUnavailable }, Prepare: func(context.Context, registry.Route, hostclaim.Observed, activator.StartPolicy) (activator.ClaimStart, error) { return activator.ClaimStart{}, hostwrite.ErrUnavailable }, FailedAdmission: func(context.Context, hostclaim.Observed, error) error { return hostwrite.ErrUnavailable }} }