//go:build linux && rootrunner package host import ( "bytes" "context" "crypto/ed25519" "encoding/base64" "encoding/json" "errors" "fmt" "os" "os/exec" "path/filepath" "slices" "strconv" "strings" "syscall" "testing" "time" "github.com/carlosframework/platform/internal/claimactivation" "github.com/carlosframework/platform/internal/domains" "github.com/carlosframework/platform/internal/genselect" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfloor" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/park" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" ) // Step 8 readiness witness (R4). Leo's real tree, rebuilt from its metadata // snapshot, goes through the window in order: // // 1. 8.1's masks, applied with the runbook's own commands to leo's unit set // installed in this runner's /etc/systemd/system, and checked by // systemd and by install-control's verifyOfflineRootMigration; // 2. backfill-collect (claims run, production runner); // 3. journal-migrate without --window: the census refuses the masks (S2 as // found), and leaves the migration resumable; // 4. journal-migrate with --window, under umask 077 (R3): census and // migration complete, the floor directories come out 0755, and leo's // signed census report names claimtrial as its route installation; // 5. backfill-adopt: on the dev roster it refuses, because sophie is a // roster box with no census key (STOP S3). With sophie's report standing // in, it publishes, claimtrial goes to leo, and the others stay unowned; // 6. the console's adoption pass (production ReconcileClaims) gives // claimtrial a serving claim on leo; the files seed selects it; // 7. root's first step for an adopted host with no installation, the // per-host empty census, refuses claimtrial's legacy row and files // (STOP S4): nothing on main installs an adopted legacy host. // // Scaffold writes (state from before the window, never claim state on the // window's path): the roster at revision 9 and the floor, signed by a // fixture release key (steps 5 to 7 made them); the three bucket records // and their committed domain claims (the bucket as it is); leo's identity // and census key (fixture owner bindings, as every accepted root migration // witness uses); the root floor history and guard binary (guard-arm, // step 6); claimtrial's registry row (leo-routes.tsv); and, in the second // test only, sophie's census report, which nothing on dev can produce. // // On the dev roster, after step 4, backfill-done is also run against leo's // real report: it names claimtrial, so done refuses (fail-closed on // installations). func TestRootStep8WindowWitnessDevRoster(t *testing.T) { rootStep8Witness(t, step8Shape{}) } func TestRootStep8WindowWitnessSophieReported(t *testing.T) { rootStep8Witness(t, step8Shape{sophieReports: true}) } // The empty-leo flow (claim trial Option B, Paul's D1 of 2026-10-20): the // dev records are deleted before the window (B1; all of them, as the // controller ruled after the live precheck, or all but rollcheck-0261 and // wakecheck, the two-host shape), and claimtrial's and wakecheck's leftovers // on leo are moved aside out of /data and /opt/carlos (B2). Leo's real tree // then runs without claimtrial's registry row and without those leftovers, // under 8.1's masks: // // 1. 8.1's masks, as above; // 2. backfill-collect freezes the records left (zero, or two); // 3. journal-migrate without --window still refuses the masks; // 4. journal-migrate --window under umask 077 completes, and leo's signed // census report lists zero installations; // 5. backfill-done, dry run then apply: done under If-Match over the // collecting plan, sophie (no census key) unreported, and the // production CheckCreateFreeze accepts the plan; the console's adoption // pass is a no-op on it and gives no host a claim. // // Scaffold writes are those above less claimtrial's registry row, less the // deleted records and their domain claims, and with no stand-in report for // sophie. Nothing on the window's path (the plan in both phases, leo's // census report, the migration, the masks) is hand-made. func TestRootStep8WindowWitnessEmptyLeoZeroHosts(t *testing.T) { rootStep8Witness(t, step8Shape{empty: true}) } func TestRootStep8WindowWitnessEmptyLeoTwoHosts(t *testing.T) { rootStep8Witness(t, step8Shape{empty: true, keep: []string{leoRollcheck, leoWakecheck}}) } // The rerun witnesses (fix round 1, F2): the empty-leo zero-host flow, with // a public floor directory left behind by an earlier journal-migrate // attempt in the window's umask 077 shell. The rerun, also under umask // 077, repairs it under the migration's lock before the first floor read, // completes, and leaves guard/moves and guard/seeds 0755 root:root: // // - Moves0700: guard/moves (guard-arm's floor history) left 0700; // - Seeds0700: guard/seeds left 0700; // - Interrupted: guard/seeds made by an earlier build's mkdirAllExact // mkdir under umask 077, its chmod never reached (0700, empty); // - Foreign: guard/seeds left 0700 but owned by the carlos user. The // migration refuses it and leaves it as it was; once it is root's again // the rerun repairs it and completes. // // The earlier attempt's leftover is the only scaffold write these add, made // after step 3's refused attempt and checked before the rerun. func TestRootStep8WindowWitnessRerunMoves0700(t *testing.T) { rootStep8Witness(t, step8Shape{empty: true, floorDirs: "moves"}) } func TestRootStep8WindowWitnessRerunSeeds0700(t *testing.T) { rootStep8Witness(t, step8Shape{empty: true, floorDirs: "seeds"}) } func TestRootStep8WindowWitnessRerunInterrupted(t *testing.T) { rootStep8Witness(t, step8Shape{empty: true, floorDirs: "interrupted"}) } func TestRootStep8WindowWitnessRerunForeign(t *testing.T) { rootStep8Witness(t, step8Shape{empty: true, floorDirs: "foreign"}) } type step8Shape struct { // sophieReports gives sophie a census key and a stand-in report. sophieReports bool // empty is Option B: B1 and B2 done, and keep the records B1 left. empty bool keep []string // floorDirs is the floor directory an earlier attempt left behind // ("" for none; see the rerun witnesses). floorDirs string } const ( step8Window = "/etc/carlos/claim-window-cohort1-20261021T0900Z" step8Bucket = "deployment" step8Leo = "leo-climbing" step8Sophie = "sophie-knitting" step8ID = "cohort1-20261021" ) func rootStep8Witness(t *testing.T, shape step8Shape) { sophieReports := shape.sophieReports if os.Geteuid() != 0 { t.Fatal("designated runner must execute as root") } ctx := t.Context() uid, gid := generationRunnerRegistryUser(t) for _, path := range []string{"/var/lib/carlos-claims", "/var/lib/carlos-release", "/var/lib/carlos-swap-status", "/etc/carlos"} { if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { t.Fatal("runner already holds", path, err) } t.Cleanup(func() { if err := os.RemoveAll(path); err != nil { t.Error(err) } }) } for _, path := range []string{"/run/carlos", "/run/carlos/hostguard"} { if err := os.Mkdir(path, 0755); err == nil { t.Cleanup(func() { os.RemoveAll(path) }) } else if !os.IsExist(err) { t.Fatal(err) } } // Leo's /data and /opt/carlos from the snapshot, owned as on leo. scratch := generationRootRunnerDir(t) data, opt := filepath.Join(scratch, "data"), filepath.Join(scratch, "opt") for _, dir := range []string{data, opt} { if err := os.Mkdir(dir, 0755); err != nil { t.Fatal(err) } } owner := func(path, who string) error { switch who { case "root:root": return os.Lchown(path, 0, 0) case "carlos:carlos": return os.Lchown(path, uid, gid) case "root:carlos": return os.Lchown(path, 0, gid) } return fmt.Errorf("unexpected owner %q at %s", who, path) } buildLeoTree(t, leoTreeRows(t, false), leoTreeOptions{ Roots: map[string]string{"/data": data, "/opt/carlos": opt}, Owner: owner, Content: func(root, path string) []byte { if root == "/data" && path == "swap-failures.json" { return []byte(`{"schema":1,"records":null}`) } return nil }, Skip: func(r leoRow) bool { // Rebuilt by the scaffold: the registry, root's guard and // update state, the platform binary and its fallback. if r.Root == "/data" && strings.HasPrefix(r.Path, "registry.db") { return true } if shape.empty && step8MovedAside(r) { return true } return r.Root == "/opt/carlos" && (r.Path == "system" || strings.HasPrefix(r.Path, "system/") || r.Path == "carlos" || strings.HasPrefix(r.Path, "carlos.prev")) }, }) // Leo's unit set in this runner's /etc/systemd/system, then 8.1's masks. installLeoUnits(t) runStep8Shell(t, ` WINDOW='`+step8Window+`' TRIGGERS=(carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-system-update.path) SERVICES=(carlos-adopt.service carlos-stage.service carlos-config-sync.service carlos-restart.service carlos-system-update.service carlos-system-recover.service) UNITS=("${TRIGGERS[@]}" "${SERVICES[@]}" carlos-edge.service) mkdir -p -m 0755 /etc/carlos mkdir -m 0700 -- "$WINDOW" mkdir -m 0700 -- "$WINDOW/held" systemctl daemon-reload systemctl stop "${TRIGGERS[@]}" systemctl stop "${SERVICES[@]}" systemctl stop carlos-edge.service systemctl disable carlos-edge.service for unit in "${UNITS[@]}"; do mv -T -- "/etc/systemd/system/$unit" "$WINDOW/held/$unit" ln -s -- /dev/null "/etc/systemd/system/$unit" done systemctl daemon-reload systemctl show "${UNITS[@]}" -p Id -p LoadState -p ActiveState `) for _, unit := range claimWindowUnits { out, err := exec.Command("systemctl", "show", unit, "-p", "LoadState", "--value").CombinedOutput() if err != nil || strings.TrimSpace(string(out)) != "masked" { t.Fatalf("8.1: %s is %q (%v)", unit, out, err) } } if _, err := os.Lstat("/etc/systemd/system/multi-user.target.wants/carlos-edge.service"); !errors.Is(err, os.ErrNotExist) { t.Fatal("8.1: the edge is still wanted", err) } if err := verifyOfflineRootMigration(ctx); err != nil { t.Fatal("8.1: install-control's offline check", err) } // The bucket and roster as steps 5 to 7 left them (scaffold). store := release.NewDirStore(filepath.Join(scratch, "bucket")) key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize)) signer := release.RootSigner(key) directory := hostactors.Directory{Store: store, PublicKey: key.Public().(ed25519.PublicKey), Bucket: step8Bucket} leoKey := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{2}, ed25519.SeedSize)) sophieKey := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{3}, ed25519.SeedSize)) enrollment := strings.Repeat("a", 32) identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: step8Leo, Enrollment: enrollment, Revision: 9, Binding: "9", Region: "ie", ControlBucket: step8Bucket}) actor, err := identity.Verify(ctx) if err != nil { t.Fatal(err) } if err := hostfloor.Publish(ctx, store, signer, step8Bucket, hostfloor.Floor{Min: 1, MinRelease: "v2.0.0"}); err != nil { t.Fatal(err) } floor, err := hostfloor.Read(ctx, store, directory.PublicKey, step8Bucket) if err != nil { t.Fatal(err) } roster := hostactors.Roster{V: 1, DeploymentBucket: step8Bucket, Revision: 9, RequestLimitNS: int64(hostbox.RequestLimit), HomeRegion: "ie", Publication: "published", FloorSHA256: floor.Digest, Regions: map[string]hostactors.Region{"ie": {Account: actor.Cloud.Account, AWSRegion: "eu-west-1", Namespaces: []string{"platform"}, Pools: []string{"cohort1", "console"}}}, Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} roster.Boxes[step8Leo] = hostactors.Box{BoxID: step8Leo, Enrollment: enrollment, Namespace: "platform", Label: step8Leo, Region: "ie", Pool: "cohort1", State: "active", LossIDs: []string{}, Cloud: hostactors.Cloud{Provider: "aws", Account: actor.Cloud.Account, Region: actor.Cloud.Region, InstanceID: actor.Cloud.InstanceID}, EvidenceSHA256: strings.Repeat("c", 64), CensusPublicKey: leoKey.Public().(ed25519.PublicKey)} sophie := hostactors.Box{BoxID: step8Sophie, Enrollment: strings.Repeat("b", 32), Namespace: "platform", Label: step8Sophie, Region: "ie", Pool: "console", State: "active", LossIDs: []string{}, Cloud: hostactors.Cloud{Provider: "aws", Account: actor.Cloud.Account, Region: actor.Cloud.Region, InstanceID: "i-0def"}, EvidenceSHA256: strings.Repeat("d", 64)} if sophieReports { sophie.CensusPublicKey = sophieKey.Public().(ed25519.PublicKey) } roster.Boxes[step8Sophie] = sophie body, err := release.SignRootEnvelope(signer, roster) if err != nil { t.Fatal(err) } if _, err := store.Put(ctx, hostactors.Key, body, ""); err != nil { t.Fatal(err) } records := map[string]hostops.Record{} for _, r := range leoRecords(t) { if shape.empty && !slices.Contains(shape.keep, r.Holder.Host) { continue // deleted at B1 } if _, err := store.Put(ctx, r.Holder.Key, r.Body, ""); err != nil { t.Fatal(err) } if r, err = hostops.ReadRecord(ctx, store, r.Holder.Key); err != nil { t.Fatal(err) } records[r.Holder.Host] = r inc, _ := r.Incarnation() raw, _ := json.Marshal(domains.Claim{Schema: 1, Host: r.Holder.Host, Account: r.Holder.Account, App: r.Holder.App, Acquisition: r.Holder.Acquisition, RecordIncarnation: inc, OwnershipState: "committed"}) if _, err := store.Put(ctx, domains.ClaimKey(r.Holder.Host), raw, ""); err != nil { t.Fatal(err) } } // Leo's registry: its one route (leo-routes.tsv), owned as on leo; none // once B1 has retired claimtrial. reg, err := registry.Open(filepath.Join(data, "registry.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { reg.Close() }) if !shape.empty { if err := reg.Upsert(leoRoute(data)); err != nil { t.Fatal(err) } } for _, suffix := range []string{"", "-wal", "-shm"} { if err := os.Lchown(filepath.Join(data, "registry.db"+suffix), uid, gid); err != nil && !os.IsNotExist(err) { t.Fatal(err) } } // Root's floor history and guard binary, as guard-arm left them. t.Setenv("CARLOS_OPT_DIR", opt) t.Setenv("CARLOS_DATA_DIR", data) t.Setenv("CARLOS_DEPLOYMENT_BUCKET", step8Bucket) t.Setenv("CARLOS_RELEASE_PUBKEY", base64.StdEncoding.EncodeToString(directory.PublicKey)) t.Setenv("CARLOS_HOST_CLAIM", "1") t.Setenv("CARLOS_HOST_CLAIM_ORIGINS", "1") t.Setenv("CARLOS_POOLS", "cohort1") t.Setenv("TMPDIR", scratch) // guard-arm (step 6) leaves guard/ and guard/moves 0755 root:root. Make // them so explicitly, under umask 022, rather than at whatever umask the // runner's shell has, and check it: the migration below must find either // that or the leftover a rerun shape makes after step 3, never a state // the scaffold produced by accident. floorID := rootFloorIdentity{Deployment: step8Bucket, Box: actor.Box, Enrollment: actor.Enrollment} scaffoldUmask := syscall.Umask(0o022) floors, err := openRootMovesFloors(opt, floorID, true) if err != nil { syscall.Umask(scaffoldUmask) t.Fatal(err) } err = floors.reconstruct(ctx, rootFloorReconstruction{Identity: floorID, Floor: &moves.Floor{Min: 1, MinRelease: "v2.0.0"}, Source: "enrolled root history", Digest: floorTestDigest, Authorization: floorTestAuthorization(t, floors)}) floors.close() syscall.Umask(scaffoldUmask) if err != nil { t.Fatal(err) } for _, dir := range []string{systemGuardDir(opt), rootFloorDir(opt)} { st, err := os.Lstat(dir) if err != nil || !st.IsDir() || st.Mode().Perm() != 0o755 || st.Sys().(*syscall.Stat_t).Uid != 0 || st.Sys().(*syscall.Stat_t).Gid != 0 { t.Fatalf("scaffold: %s is not guard-arm's 0755 root:root directory: %v %v", dir, st.Mode(), err) } } rootFloorRunnerBinary(t, opt, systemBinaryPath(opt), "v2.0.0", key) rootFloorRunnerBinary(t, opt, filepath.Join(systemGuardDir(opt), "carlos"), "v2.0.0", key) if _, err := os.Lstat(rootSeedsFloorDir(opt)); !errors.Is(err, os.ErrNotExist) { t.Fatal("the seeds floor directory exists before journal-migrate", err) } unused := fix4UnusedDependencies{} owners := fix4Owners{ClaimOwnerBindings{ Identity: identity, RetirementAuthority: &hostops.ProductionAuthority{Store: store, Directory: directory, Now: time.Now}, EvidenceKey: leoKey, Local: unused, Controllers: unused, Data: unused, Inputs: unused, Evidence: unused, InputPolicy: func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }, MovesPolicy: func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }, LeaseSet: func(context.Context, string, string) ([]string, error) { return nil, nil }, View: func(context.Context, hostclaim.Observed) (park.View, error) { return park.View{}, nil }, Authorize: func(context.Context, hostclaim.Observed, hostclaim.Obligation) error { return nil }, AdmissionEvidence: func(context.Context, hostclaim.Observed, hostclaim.Admission, *genselect.Plan) error { return nil }, PublicationEvidence: func(context.Context, hostclaim.Observed) error { return nil }, Close: func(context.Context) error { return nil }, }} // 2. backfill-collect, dry run then apply. runner := claimactivation.Runner{Directory: directory, Signer: signer} collect := claimactivation.Plan{Phase: "backfill-collect", Bucket: step8Bucket, ExpectedRevision: 9, Backfill: &claimactivation.BackfillInput{ID: step8ID}} if res, err := runner.Run(ctx, collect, true); err != nil || len(res.Backfill.Hosts) != len(records) { t.Fatalf("backfill-collect dry run: %+v %v", res.Backfill, err) } res, err := runner.Run(ctx, collect, false) if err != nil { t.Fatal("backfill-collect", err) } t.Logf("backfill-collect: %d hosts frozen, phase %s, revision %d", len(res.Backfill.Hosts), res.Backfill.Phase, res.Backfill.DirectoryRevision) r := ClaimRuntimeResources{Mode: "instance-control", Registry: reg, DataDir: data, OptDir: opt, UnitsDir: "/etc/systemd/system"} // 3. journal-migrate as S2 found it: no --window, the masks refuse. err = migrateClaimJournalsWith(ctx, store, r, owners, true) if !errors.Is(err, hostbox.ErrTypedInventory) || !strings.Contains(err.Error(), "units entry") { t.Fatalf("journal-migrate without --window = %v, want the census to refuse a mask", err) } t.Logf("journal-migrate without --window refused: %v", err) // The floor directories are prepared under the migration's lock before // its first floor read, so even this refused attempt made guard/seeds, // 0755 root:root, and left guard/moves so. for _, dir := range []string{rootFloorDir(opt), rootSeedsFloorDir(opt)} { st, err := os.Lstat(dir) if err != nil || !st.IsDir() || st.Mode().Perm() != 0o755 || st.Sys().(*syscall.Stat_t).Uid != 0 || st.Sys().(*syscall.Stat_t).Gid != 0 { t.Fatalf("after the refused attempt: %s %v %v", dir, st.Mode(), err) } } // 4. journal-migrate --window, as the window's root shell runs it, but // under umask 077 (R3). r.MigrationWindow = step8Window masked, err := verifyJournalMigrateWindow(r) if err != nil || len(masked) != len(claimWindowUnits) { t.Fatalf("--window: %v %v", masked, err) } step8EarlierAttempt(t, opt, shape.floorDirs, uid, gid) // Step 3's refused attempt got as far as the census in ready, so its // migration.json is there; a refused rerun must leave it as it is. migrationRecord := func() []byte { state, err := rootstate.Open(rootMigrationDir, false, false) if err != nil { t.Fatal(err) } defer state.Close() raw, err := state.Read("migration.json") if err != nil { t.Fatal(err) } return raw } var before []byte if shape.floorDirs == "foreign" { before = migrationRecord() } old := syscall.Umask(0o077) err = migrateClaimJournalsWith(ctx, store, r, owners, true) syscall.Umask(old) if shape.floorDirs == "foreign" { // Not root's: refused before any floor read, and left as it was. if !errors.Is(err, os.ErrPermission) { t.Fatalf("journal-migrate --window over a carlos-owned guard/seeds = %v, want a refusal", err) } st, serr := os.Lstat(rootSeedsFloorDir(opt)) if serr != nil || st.Mode().Perm() != 0o700 || st.Sys().(*syscall.Stat_t).Uid != uint32(uid) { t.Fatalf("the refused guard/seeds changed: %v %v", st.Mode(), serr) } if !bytes.Equal(migrationRecord(), before) { t.Fatal("the refused rerun changed migration.json") } t.Logf("journal-migrate --window refused a carlos-owned guard/seeds and left it 0700: %v", err) if err := os.Lchown(rootSeedsFloorDir(opt), 0, 0); err != nil { t.Fatal(err) } old = syscall.Umask(0o077) err = migrateClaimJournalsWith(ctx, store, r, owners, true) syscall.Umask(old) } if err != nil { t.Fatal("journal-migrate --window", err) } state, err := rootstate.Open(rootMigrationDir, false, false) if err != nil { t.Fatal(err) } raw, err := state.Read("migration.json") state.Close() var saved rootMigrationRecord if err != nil || rootparticipation.Decode(raw, &saved) != nil || saved.Phase != "complete" { t.Fatalf("migration.json: %s %v", raw, err) } for path, want := range map[string]os.FileMode{"/var/lib/carlos-claims": 0o700, rootFloorDir(opt): 0o755, rootSeedsFloorDir(opt): 0o755, systemGuardDir(opt): 0o755} { st, err := os.Stat(path) if err != nil || !st.IsDir() || st.Mode().Perm() != want || st.Sys().(*syscall.Stat_t).Uid != 0 || st.Sys().(*syscall.Stat_t).Gid != 0 { t.Fatalf("mode check after journal-migrate: %s %v %v", path, st.Mode(), err) } } if shape.floorDirs != "" { t.Logf("rerun over %s: complete; guard/moves and guard/seeds are 0755 root:root", shape.floorDirs) } reportRaw, _, err := store.Get(ctx, "control/host-census/"+step8ID+"/"+step8Leo+".json") if err != nil { t.Fatal(err) } var envelope hostops.CensusEnvelope if err := json.Unmarshal(reportRaw, &envelope); err != nil { t.Fatal(err) } if shape.empty { if envelope.Report.Installations == nil || len(envelope.Report.Installations) != 0 { t.Fatalf("leo's census report: %+v", envelope.Report.Installations) } t.Logf("journal-migrate --window: complete; leo's report lists zero installations (%d hosts frozen)", len(records)) rootStep8Done(t, store, directory, runner, shape) return } inc, _ := records[leoClaimtrial].Incarnation() if len(envelope.Report.Installations) != 1 || envelope.Report.Installations[0] != (hostops.CensusInstallation{Host: leoClaimtrial, Account: "bab", App: "claimtrial", Family: "instance", Key: records[leoClaimtrial].Holder.Key, Incarnation: inc, Route: true}) { t.Fatalf("leo's census report: %+v", envelope.Report.Installations) } t.Logf("journal-migrate --window: complete; leo's report names %s as a route installation", leoClaimtrial) // 5. backfill-adopt. adopt := claimactivation.Plan{Phase: "backfill-adopt", Bucket: step8Bucket, ExpectedRevision: 9, Backfill: &claimactivation.BackfillInput{ID: step8ID}} if !sophieReports { // backfill-done over leo's real report, which names claimtrial // (B1 skipped): fail-closed, the plan stays collecting. done := claimactivation.Plan{Phase: "backfill-done", Bucket: step8Bucket, ExpectedRevision: 9, Backfill: &claimactivation.BackfillInput{ID: step8ID}} if _, doneErr := runner.Run(ctx, done, false); !errors.Is(doneErr, claimactivation.ErrBackfillInstallations) { t.Fatalf("backfill-done over leo's report naming claimtrial = %v, want the installations refusal", doneErr) } else { t.Logf("backfill-done refused over leo's report naming claimtrial: %v", doneErr) } _, err := runner.Run(ctx, adopt, false) if !errors.Is(err, os.ErrNotExist) { t.Fatalf("backfill-adopt on the dev roster = %v, want sophie's missing census report refused", err) } var plan hostops.BackfillPlan if raw, _, err := store.Get(ctx, hostops.BackfillKey); err != nil || release.VerifyRootEnvelope(directory.PublicKey, raw, &plan) != nil || plan.Phase != "collecting" { t.Fatalf("the plan after a refused adopt: %+v %v", plan, err) } t.Logf("STOP S3: backfill-adopt refused on the dev roster: %v", err) return } sophieReport := hostops.CensusReport{Box: step8Sophie, Enrollment: sophie.Enrollment, BootID: "sophie-boot", Reservation: step8ID, DirectoryRevision: 9, CapturedAt: time.Now().UTC(), Complete: true, Installations: []hostops.CensusInstallation{}, Seeds: &hostops.CensusSeeds{Journaled: []string{}, Gated: []string{}}} if err := hostops.PublishCensusReport(ctx, store, directory, sophieReport, sophieKey); err != nil { t.Fatal(err) } if res, err = runner.Run(ctx, adopt, true); err != nil { t.Fatal("backfill-adopt dry run", err) } if res, err = runner.Run(ctx, adopt, false); err != nil { t.Fatal("backfill-adopt", err) } for _, h := range res.Backfill.Hosts { want := "" if h.Host == leoClaimtrial { want = step8Leo } if h.Box != want { t.Fatalf("adopting plan: %s on %q", h.Host, h.Box) } } if !slices.Equal(res.Backfill.Unowned, []string{leoRollcheck, leoWakecheck}) { t.Fatalf("unowned: %v", res.Backfill.Unowned) } t.Logf("backfill-adopt: adopting; %s on %s; unowned %v", leoClaimtrial, step8Leo, res.Backfill.Unowned) // 6. The console's adoption pass, as production builds it. serial := 0 controller := hostops.NewBackfillController(store, directory, time.Now, func() string { serial++; return "witness-op-" + strconv.Itoa(serial) }) err = controller.ReconcileClaims(ctx) if !errors.Is(err, hostops.ErrIdentity) { t.Fatalf("adoption pass = %v, want the unowned hosts refused", err) } observed, err := hostclaim.Read(ctx, store, leoClaimtrial) if err != nil || observed.Absent || observed.Claim.Op.Kind != hostclaim.KindServing || observed.Claim.Op.Exec != step8Leo || observed.Claim.Holder.WriterBox != step8Leo { t.Fatalf("claimtrial's claim: %+v %v", observed.Claim, err) } candidates, err := filesSeedCandidates(ctx, store, step8Leo, "") if err != nil || len(candidates) != 1 || candidates[0].Host != leoClaimtrial { t.Fatalf("files seed candidates: %+v %v", candidates, err) } t.Logf("adoption: %s serving on %s (stamp %d, obligations %d); the files seed selects it", leoClaimtrial, step8Leo, observed.Claim.Holder.Stamp, len(observed.Claim.Obligations)) // 7. Root's first step for a claim with no installation on this box // (rootexecution_authority's ErrNoRecord path): the per-host empty // census, built as root's startup builds it, over the migrated box. launches, err := openClaimRootJournal() if err != nil { t.Fatal(err) } defer launches.Close() journals, err := hostbox.NewJournalService(launches) if err != nil { t.Fatal(err) } roots := map[string]string{"data": r.DataDir, "opt": r.OptDir, "units": r.UnitsDir} census := &hostbox.EmptyInstallationCensus{Inventory: journals.Inventory, Registry: r.Registry, Launches: launches, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "9", Roots: roots, SharedPaths: claimGlobalSharedPaths(r)} h := ClaimRootHTTP{Journals: journals, Launches: launches, Roots: roots, Reconcile: hostbox.ReconcileAdapter{Registry: r.Registry, Inventory: journals.Inventory, Census: census, Box: actor.Box, Enrollment: actor.Enrollment}} err = h.rootEmptyCensus().CheckLocal(ctx, observed, hostbox.RecordSelection{Host: leoClaimtrial, Key: records[leoClaimtrial].Holder.Key, ETag: records[leoClaimtrial].ETag, Incarnation: inc, Family: "instance"}) if !errors.Is(err, hostbox.ErrTypedInventory) { t.Fatalf("root's per-host census for the adopted legacy host = %v, want a refusal", err) } t.Logf("STOP S4: root refuses the adopted legacy host before any seed admission: %v", err) } // step8EarlierAttempt leaves the public floor directory an earlier // journal-migrate attempt in the window's umask 077 shell would have, and // checks it is there: the rerun witnesses' only extra scaffold write. func step8EarlierAttempt(t *testing.T, opt, shape string, uid, gid int) { t.Helper() dir := rootSeedsFloorDir(opt) owner := uint32(0) if shape == "" { return } if shape != "moves" { // Step 3's refused attempt made guard/seeds, empty; the earlier // attempt this shape stands for left it differently. if err := os.Remove(dir); err != nil { t.Fatal(err) } } switch shape { case "moves": dir = rootFloorDir(opt) if err := os.Chmod(dir, 0o700); err != nil { t.Fatal(err) } case "seeds": if err := os.Mkdir(dir, 0o700); err != nil { t.Fatal(err) } case "interrupted": // An earlier build's mkdirAllExact, first half: os.MkdirAll(dir, // 0755) under the window shell's umask; the process died before // its chmod. old := syscall.Umask(0o077) err := os.MkdirAll(dir, 0o755) syscall.Umask(old) if err != nil { t.Fatal(err) } case "foreign": if err := os.Mkdir(dir, 0o700); err != nil { t.Fatal(err) } if err := os.Lchown(dir, uid, gid); err != nil { t.Fatal(err) } owner = uint32(uid) default: t.Fatalf("unknown floor-directory shape %q", shape) } st, err := os.Lstat(dir) if err != nil || !st.IsDir() || st.Mode().Perm() != 0o700 || st.Sys().(*syscall.Stat_t).Uid != owner { t.Fatalf("earlier attempt: %s is %v (%v), want 0700 owned by %d", dir, st.Mode(), err, owner) } t.Logf("earlier attempt left %s 0700 (owner %d)", dir, owner) } // rootStep8Done is the empty-leo flow's step 5: backfill-done, dry run then // apply, through the production runner, then the console's own readers. func rootStep8Done(t *testing.T, store release.Store, directory hostactors.Directory, runner claimactivation.Runner, shape step8Shape) { t.Helper() ctx := t.Context() _, collectingTag, err := store.Get(ctx, hostops.BackfillKey) if err != nil { t.Fatal(err) } done := claimactivation.Plan{Phase: "backfill-done", Bucket: step8Bucket, ExpectedRevision: 9, Backfill: &claimactivation.BackfillInput{ID: step8ID}} dry, err := runner.Run(ctx, done, true) if err != nil || len(dry.Writes) != 1 || dry.Writes[0].BeforeETag != collectingTag || dry.Backfill.Phase != "done" { t.Fatalf("backfill-done dry run: %+v %v", dry, err) } if _, tag, err := store.Get(ctx, hostops.BackfillKey); err != nil || tag != collectingTag { t.Fatalf("backfill-done dry run wrote the plan: %v", err) } if err := hostops.CheckCreateFreeze(ctx, store, directory.PublicKey); !errors.Is(err, hostops.ErrPending) { t.Fatalf("freeze while collecting = %v", err) } res, err := runner.Run(ctx, done, false) if err != nil { t.Fatal("backfill-done", err) } if res.Backfill.Phase != "done" || len(res.Backfill.Hosts) != len(shape.keep) || !slices.Equal(res.Backfill.Unreported, []string{step8Sophie}) || len(res.Backfill.Reports) != 1 || res.Backfill.Reports[step8Leo] == "" { t.Fatalf("backfill-done: %+v", res.Backfill) } if len(res.Writes) != 1 || res.Writes[0].Key != hostops.BackfillKey || res.Writes[0].BeforeETag != collectingTag { t.Fatalf("backfill-done writes: %+v", res.Writes) } var plan hostops.BackfillPlan raw, _, err := store.Get(ctx, hostops.BackfillKey) if err != nil || release.VerifyRootEnvelope(directory.PublicKey, raw, &plan) != nil || plan.Phase != "done" || plan.Hosts == nil || len(plan.Hosts) != len(shape.keep) { t.Fatalf("the done plan: %+v %v", plan, err) } if err := hostops.CheckCreateFreeze(ctx, store, directory.PublicKey); err != nil { t.Fatalf("CheckCreateFreeze after done = %v", err) } t.Logf("backfill-done: done under If-Match; %d hosts; unreported %v; CheckCreateFreeze nil", len(plan.Hosts), res.Backfill.Unreported) // The console's adoption pass, as production builds it, does nothing. serial := 0 controller := hostops.NewBackfillController(store, directory, time.Now, func() string { serial++; return "witness-op-" + strconv.Itoa(serial) }) if err := controller.ReconcileClaims(ctx); err != nil { t.Fatalf("adoption pass over the done plan = %v", err) } if keys, err := store.List(ctx, "control/hosts/"); err != nil || len(keys) != 0 { t.Fatalf("host claims after the adoption pass: %v %v", keys, err) } t.Logf("adoption pass over the done plan: no-op, no host claim") } // installLeoUnits writes leo's carlos units, drop-ins and enablement links // into this runner's /etc/systemd/system: unit files with the embedded // bytes leo's were installed from, drop-ins with a minimal body, and the // links exactly as the snapshot names them. The runner's own vendor links // stand in for leo's. Everything written here is removed afterwards. func installLeoUnits(t *testing.T) { t.Helper() const units = "/etc/systemd/system" var made []string t.Cleanup(func() { for i := len(made) - 1; i >= 0; i-- { if err := os.Remove(made[i]); err != nil && !os.IsNotExist(err) { t.Error(err) } } for _, unit := range claimWindowUnits { // The window's masks and held files. if target, err := os.Readlink(filepath.Join(units, unit)); err == nil && target == "/dev/null" { os.Remove(filepath.Join(units, unit)) } } exec.Command("systemctl", "daemon-reload").Run() }) carlos := func(name string) bool { base := filepath.Base(name) return strings.HasPrefix(base, "carlos") || base == "data-instances.mount" || strings.HasPrefix(base, "run-carlos-") } for _, r := range leoTreeRows(t, false) { if r.Root != units || r.Path == "." { continue } path := filepath.Join(units, r.Path) switch r.Type { case "d": if !carlos(r.Path) { if err := os.Mkdir(path, 0755); err == nil { made = append(made, path) } else if !os.IsExist(err) { t.Fatal(err) } continue } if err := os.Mkdir(path, r.Mode); err != nil { t.Fatal("runner already holds", path, err) } made = append(made, path) case "f": if !carlos(r.Path) { continue } body, err := unitsFS.ReadFile("units/" + filepath.Base(r.Path)) if err != nil { body = []byte("[Service]\n") } if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { t.Fatal("runner already holds", path, err) } if err := os.WriteFile(path, body, r.Mode); err != nil { t.Fatal(err) } made = append(made, path) case "l": if !carlos(r.Path) || !strings.HasPrefix(r.Target, units+"/") { continue } if err := os.Symlink(r.Target, path); err != nil { t.Fatal("runner already holds", path, err) } made = append(made, path) } } // Directories are removed after what is in them. slices.SortStableFunc(made, func(a, b string) int { return strings.Count(a, "/") - strings.Count(b, "/") }) } func runStep8Shell(t *testing.T, script string) { t.Helper() cmd := exec.Command("bash", "-c", "set -eu\numask 077\n"+script) out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("runbook block: %v\n%s", err, out) } t.Logf("runbook block:\n%s", out) }