package host import ( "context" "crypto/rand" "encoding/hex" "encoding/json" "errors" "golang.org/x/sys/unix" "os" "path/filepath" "strconv" "strings" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" "github.com/carlosframework/platform/internal/safefile" ) var rootMigrationQuiesce = quiesceLegacyClaimJournals func migrateRootClaimAuthority(ctx context.Context, store release.Store, r ClaimRuntimeResources, owners ClaimOwnerBindings, actor hostbox.Actor, journals *hostbox.JournalService, launches *hostbox.LaunchJournal) error { if err := claimActivationPreflight(r.Registry); err != nil { return err } return withRootReleaseStore(ctx, func() (rootstate.Store, error) { return rootstate.Open(rootReleasePrivate, true, false) }, func() error { if err := claimActivationVerified(ctx, store, r); err != nil { return err } if err := verifyOfflineRootMigration(ctx); err != nil { return err } state, err := rootstate.Open(rootMigrationDir, true, false) if err != nil { return err } defer state.Close() execution, err := rootstate.Open(rootparticipation.Namespace, true, false) if err != nil { return err } defer execution.Close() fences, err := openClaimRootAuthority() if err != nil { return err } defer fences.Close() root := hostguard.NewRootAuthority(fences) id := rootMigrationIdentity{Deployment: owners.RetirementAuthority.Directory.Bucket, Box: actor.Box, Enrollment: actor.Enrollment, Generation: 1} collector := rootMigrationCollector{retirements: productionGenerationRetirements, suspects: func(ctx context.Context) ([]string, error) { return rootMigrationLegacySuspects(ctx, r.DataDir) }, store: store, directory: owners.RetirementAuthority.Directory, identity: id, seeds: r.Registry, installations: root.MigrationInstallations, launches: launches.GenerationLaunches, boot: func() (string, error) { raw, err := os.ReadFile("/proc/sys/kernel/random/boot_id") return strings.TrimSpace(string(raw)), err }, routes: r.Registry.List, dataDir: r.DataDir, reservations: func(ctx context.Context) ([]release.GenerationReservation, error) { state, err := rootstate.Open(release.RootGenerationsDir, false, false) if errors.Is(err, os.ErrNotExist) { return nil, nil } if err != nil { return nil, err } defer state.Close() return (&release.RootGenerations{Store: state}).Reservations(ctx) }} roots := map[string]string{"data": r.DataDir, "opt": r.OptDir, "units": r.UnitsDir} guard := hostguard.Guard{Root: root, DataDir: r.DataDir, Claims: hostguard.StoreReader{Store: store}} // Reads go through the same typed transport used by the running controller. _, inventory := hostbox.RemoteJournals(func(ctx context.Context, q hostbox.JournalRequest) (hostbox.JournalResponse, error) { return journals.Execute(ctx, "root-migration", q) }) timers := rootMigrationTimers{state: state, registry: r.Registry, inventory: inventory, guard: guard} completeInventory := func(ctx context.Context, item rootMigrationItem, target hostguard.FenceTarget) error { observed, err := (hostguard.StoreReader{Store: store}).Read(ctx, item.Installation.Host) if err != nil || observed.Claim == nil { return errRootMigrationBusy } canonical, _, err := rootClaimRecord(ctx, store, observed.Claim, r.DataDir) if err != nil { return err } return timers.adopt(ctx, item, target, canonical) } coordinator := ClaimStopCoordinator{Guard: guard, Claims: hostguard.StoreReader{Store: store}, Inventory: inventory, Kernel: newProductionStopKernel(launches, roots), Evidence: store, EvidenceSigner: owners.EvidenceKey, RequestLimit: hostbox.RequestLimit, Now: time.Now, NewID: func() string { return "migration-" + randomMigrationID() }} m := rootMigration{prepareFloors: func(context.Context) error { return prepareRootFloorDirs(r.OptDir) }, guardReady: func(context.Context) error { return verifyRootMigrationFloors(r.OptDir, rootFloorIdentity{Deployment: id.Deployment, Box: id.Box, Enrollment: id.Enrollment}) }, state: state, execution: execution, identity: id, quiesce: rootMigrationQuiesce, observe: collector.collect, deny: func(ctx context.Context, item rootMigrationItem) error { return root.RetainMigrationDeny(ctx, item.Installation, item.Session) }, prepareRecovery: func(ctx context.Context) error { // The outer exclusion owns the edge mask through census completion. The // importer must neither acquire nor release that same mask itself. return journals.MigrateLegacy(ctx, filepath.Join(r.DataDir, ".hostclaims"), actor.Box, actor.Enrollment, func(context.Context) (func(), error) { return func() {}, nil }, legacyJournalProof(store, owners.RetirementAuthority.Directory), func(context.Context) error { return nil }) }, recover: func(ctx context.Context, item rootMigrationItem) error { list, err := launches.GenerationLaunches(ctx) if err != nil { return err } for _, l := range list { target := l.Target in := item.Installation if target.Host != in.Host || target.Incarnation != in.Incarnation || target.Stamp != in.Stamp || target.Session != item.Session { continue } if pending, exists, err := timers.load(item); err != nil { return err } else if exists { if err := verifyMigrationStopped(ctx, launches, inventory, l, roots); err != nil { return err } return completeInventory(ctx, item, pending.Target) } result, err := recoverMigrationStop(ctx, coordinator, store, owners.RetirementAuthority.Directory, launches, inventory, l, item.ClaimETag, roots) if err != nil { return err } if err := publishMigrationReceipt(ctx, coordinator, result); err != nil { return err } return completeInventory(ctx, item, result.Target) } return ErrClaimStopInventory }, ready: func(ctx context.Context, s rootMigrationSnapshot) error { if err := verifyRootMigrationRetained(ctx, root, s); err != nil { return err } var all []hostbox.StopInventory for _, item := range s.Items { in, revision, err := inventory.Snapshot(ctx, item.Installation.Host, hostguard.Captured{Incarnation: item.Installation.Incarnation, Stamp: item.Installation.Stamp}) if err != nil { return err } // A missing independently established inventory cannot be filled from the // agent's markers, even after importing an otherwise valid stop proof. retained, e := root.Installation(ctx, item.Installation.Host, hostguard.Captured{Incarnation: item.Installation.Incarnation, Stamp: item.Installation.Stamp}) if e != nil { return e } if revision == 0 || retained.InventoryRevision != revision || !in.Complete { return ErrClaimStopInventory } if err := (ClaimRootHTTP{Reconcile: hostbox.ReconcileAdapter{Registry: r.Registry}}).verifyInventoryTimers(in); err != nil { return err } for _, e := range in.Entries { if e.Kind == "timer" { if err := (ClaimRootHTTP{Reconcile: hostbox.ReconcileAdapter{Registry: r.Registry}}).verifyInventoryTimer(in.Host, timers.owner(item, e.RowIdentity), e); err != nil { return err } } } all = append(all, in) } masked, err := verifyMigrationWindow(r.MigrationWindow, r) if err != nil { return err } shared, err := claimMigrationSharedPaths(r, masked, s) if err != nil { return err } if err := hostbox.CheckMigrationRoots(ctx, roots, shared, all); err != nil { return err } if err := migrationExecutionEmpty(execution); err != nil { return err } // The migration's own service-user registry child is still running; // it is this command's, not a retired generation's. if _, _, err := productionGenerationKernelCensus(context.WithValue(ctx, generationCensusParentKey{}, os.Getpid())); err != nil { return err } generations, err := rootstate.Open(release.RootGenerationsDir, true, false) if err != nil { return err } generations.Close() floorID := rootFloorIdentity{Deployment: id.Deployment, Box: id.Box, Enrollment: id.Enrollment} if err := prepareRootFloorDirs(r.OptDir); err != nil { return err } for _, dir := range []string{rootFloorDir(r.OptDir), rootSeedsFloorDir(r.OptDir)} { f, err := openRootProtocolFloors(dir, floorID, true) if err != nil { return err } f.close() } return verifyRootMigrationFloors(r.OptDir, rootFloorIdentity{Deployment: id.Deployment, Box: id.Box, Enrollment: id.Enrollment}) }, publish: func(ctx context.Context, report hostops.CensusReport) error { if err := initializeMigrationFailures(r.DataDir, execution); err != nil { return err } return hostops.PublishCensusReport(ctx, store, owners.RetirementAuthority.Directory, report, owners.EvidenceKey) }} return m.run(ctx) }) } // prepareRootFloorDirs makes the two public floor directories, guard/moves // and guard/seeds, 0755 root:root whatever the operator shell's umask // (#317's class). rootstate creates a public directory with Mkdirat(0755) // and never chmods, so under umask 077 it comes out 0700 and rootstate's // public open refuses it; an interrupted earlier attempt (mkdir done, chmod // not) leaves the same. The migration runs this under its lock and // quiesce, before guardReady's first floor read, so a rerun repairs what an // earlier attempt left instead of refusing it forever; ready runs it again // before it opens the floors. // // Nothing is changed until the path to it has been checked: see // repairRootFloorDir. func prepareRootFloorDirs(opt string) error { return prepareRootFloorDirsOwned("/", opt, 0, 0) } // prepareRootFloorDirsOwned is prepareRootFloorDirs with the directory the // walk starts from and the owner every directory on it must have; // production passes / and root. Unprivileged tests pass a scratch directory // of their own and their own uid and gid, since the real ancestors of their // scratch are not theirs. func prepareRootFloorDirsOwned(base, opt string, uid, gid uint32) error { for _, dir := range []string{rootFloorDir(opt), rootSeedsFloorDir(opt)} { if err := repairRootFloorDir(base, dir, uid, gid); err != nil { return err } } return nil } // repairRootFloorDir walks from base to dir by descriptor (walkNoFollowDir, // consoleDir's walk) before it changes anything. Every directory from base // to dir's parent must be a real directory, not a symlink, owned by uid and // not writable by group or others. Only then is dir made, if it is missing, // by Mkdirat beneath that checked parent; it is opened with O_NOFOLLOW, must // be a directory owned by uid and gid, and is Fchmodded to 0755. A symlink // or a missing directory on the way, an untrusted ancestor, or a dir that is // a symlink, a file or another owner's refuses, and nothing is made or // changed. func repairRootFloorDir(base, dir string, uid, gid uint32) error { refuse := func(err error) error { return &os.PathError{Op: "repair", Path: dir, Err: err} } if !filepath.IsAbs(base) || filepath.Clean(base) != base || !filepath.IsAbs(dir) || filepath.Clean(dir) != dir { return refuse(os.ErrPermission) } rel, err := filepath.Rel(base, dir) if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, "../") { return refuse(os.ErrPermission) } check := func(fd int, last bool) error { var st unix.Stat_t if err := unix.Fstat(fd, &st); err != nil { return err } if st.Mode&unix.S_IFMT != unix.S_IFDIR || st.Uid != uid || !last && st.Mode&0o022 != 0 || last && st.Gid != gid { return os.ErrPermission } return nil } fd, err := walkNoFollowDir(base, strings.Split(rel, "/"), func(last bool) bool { return last }, check) if err != nil { return refuse(err) } defer unix.Close(fd) var st unix.Stat_t if err := unix.Fstat(fd, &st); err != nil { return refuse(err) } if st.Mode&07777 != 0755 { if err := unix.Fchmod(fd, 0755); err != nil { return refuse(err) } if err := unix.Fstat(fd, &st); err != nil { return refuse(err) } } if st.Mode&07777 != 0755 { return refuse(os.ErrPermission) } return nil } func migrationExecutionEmpty(state rootstate.Store) error { names, err := state.Names() if err != nil { return err } for _, name := range names { if name == "census.json" || strings.HasSuffix(name, ".lock") { continue } return errRootMigrationBusy } return nil } func randomMigrationID() string { var b [16]byte if _, err := rand.Read(b[:]); err != nil { return "" } return hex.EncodeToString(b[:]) } func publishMigrationReceipt(ctx context.Context, c ClaimStopCoordinator, result StopResult) error { var receipt StopReceipt if json.Unmarshal(result.Bytes, &receipt) != nil { return ErrClaimStopReceipt } result.Key = stopEvidenceKey(receipt.Host, receipt.OperationID, receipt.Box, receipt.ID) result.Digest = digestRawStop(result.Bytes) return c.Publish(ctx, result) } func rootMigrationLegacySuspects(ctx context.Context, dd string) ([]string, error) { dir, err := safefile.OpenDir(filepath.Join(dd, "hostguard")) if errors.Is(err, os.ErrNotExist) { return nil, nil } if err != nil { return nil, err } defer dir.Close() names, err := dir.Readdirnames(-1) if err != nil { return nil, err } out := []string{} for _, name := range names { if err := ctx.Err(); err != nil { return nil, err } if strings.HasSuffix(name, ".json") { out = append(out, strings.TrimSuffix(name, ".json")) } } return out, nil } func quiesceLegacyRootMutators(ctx context.Context) error { entries, err := os.ReadDir("/proc") if err != nil { return err } for _, entry := range entries { if err := ctx.Err(); err != nil { return err } pid, err := strconv.Atoi(entry.Name()) if err != nil || pid == os.Getpid() { continue } path := "/proc/" + entry.Name() var st unix.Stat_t if err := unix.Stat(path, &st); errors.Is(err, unix.ENOENT) { continue } else if err != nil { return err } if st.Uid != 0 { continue } raw, err := os.ReadFile(path + "/cmdline") if errors.Is(err, os.ErrNotExist) { continue } if err != nil { return err } args := strings.Split(string(raw), "\x00") if len(args) > 1 && (args[1] == "adopt" || args[1] == "system" && len(args) > 2 && (args[2] == "apply" || args[2] == "guard-arm")) { return errRootMigrationBusy } } return nil } var rootMigrationOfflineUnits = []string{"carlos-adopt.service", "carlos-stage.service", "carlos-config-sync.service", "carlos-restart.service", "carlos-system-update.service", "carlos-system-recover.service"} func verifyOfflineRootMigration(ctx context.Context) error { for _, unit := range rootMigrationOfflineUnits { raw, err := claimSystemctl(ctx, "show", unit, "--property=LoadState,ActiveState", "--no-pager") if err != nil { return err } props := string(raw) if !strings.Contains(props, "LoadState=masked\n") || !(strings.Contains(props, "ActiveState=inactive\n") || strings.Contains(props, "ActiveState=failed\n")) { return errRootMigrationBusy } } return quiesceLegacyRootMutators(ctx) }