//go:build linux && rootrunner package host import ( "bytes" "context" "crypto/ed25519" "encoding/base64" "encoding/json" "errors" "os" "os/exec" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/moves" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/rootparticipation" "github.com/carlosframework/platform/internal/rootstate" ) func TestPlan14RootMigrationBootstrapRootRunner(t *testing.T) { testRootMigrationBootstrap(t, false, false) } func TestPlan14RootMigrationHistoricRootRunner(t *testing.T) { testRootMigrationBootstrap(t, true, false) } func TestPlan14RootMigrationFreshStopRootRunner(t *testing.T) { testRootMigrationBootstrap(t, true, true) } func testRootMigrationBootstrap(t *testing.T, historic, fresh bool) { if os.Getenv("PLAN14_P3_BOOTSTRAP_CHILD") == "" { if !historic { generationRunnerRegistryUser(t) } for _, name := range rootMigrationOfflineUnits { path := filepath.Join("/run/systemd/system", name) if err := os.Symlink("/dev/null", path); err != nil { t.Fatal(err) } t.Cleanup(func() { os.Remove(path); exec.Command("systemctl", "daemon-reload").Run() }) } unit := "/usr/lib/systemd/system/carlos-edge.service" for _, p := range []string{unit, "/etc/systemd/system/carlos-edge.service", "/run/systemd/system/carlos-edge.service"} { if _, err := os.Lstat(p); !os.IsNotExist(err) { t.Fatal("existing unit", p, err) } } if err := os.WriteFile(unit, []byte("[Service]\nType=oneshot\nExecStart=/usr/bin/true\n[Install]\nWantedBy=multi-user.target\n"), 0644); err != nil { t.Fatal(err) } t.Cleanup(func() { exec.Command("systemctl", "unmask", "--runtime", "carlos-edge.service").Run() os.Remove(unit) exec.Command("systemctl", "daemon-reload").Run() }) if out, err := exec.Command("systemctl", "daemon-reload").CombinedOutput(); err != nil { t.Fatal(err, string(out)) } if out, err := exec.Command("systemctl", "mask", "--runtime", "carlos-edge.service").CombinedOutput(); err != nil { t.Fatal(err, string(out)) } if err := verifyOfflineRootMigration(t.Context()); err != nil { t.Fatal("offline root units", err) } properties := map[string]string{} for _, name := range rootMigrationOfflineUnits { out, err := claimSystemctl(t.Context(), "show", name, "--property=LoadState,ActiveState", "--no-pager") if err != nil { t.Fatal(err) } properties[name] = string(out) if out, err := exec.Command("systemctl", "start", name).CombinedOutput(); err == nil || !bytes.Contains(out, []byte("masked")) { t.Fatal(name, string(out), err) } } savedProperties, _ := json.Marshal(properties) oldOwner := claimQuiesceOwner claimQuiesceOwner = func() (int, int, error) { return 65534, 65534, nil } defer func() { claimQuiesceOwner = oldOwner }() release, err := quiesceLegacyClaimJournals(t.Context()) if err != nil { t.Fatal(err) } defer release() // The existing generation census supports a controlled all-root namespace; // E owns the wider running-tenant/foreign-namespace production integration. name := "TestPlan14RootMigrationBootstrapRootRunner" if historic { name = "TestPlan14RootMigrationHistoricRootRunner" } if fresh { name = "TestPlan14RootMigrationFreshStopRootRunner" } if t.Name() == "TestRootRebootPlan14MigrationInstalledPhase1" || t.Name() == "TestPlan14InitialMigrationExclusionRootRunner" || t.Name() == "TestPlan14RetiredMigrationPayloadRootRunner" { name = t.Name() } cmd := exec.CommandContext(t.Context(), "unshare", "--mount", "--pid", "--fork", "--mount-proc", os.Args[0], "-test.run=^"+name+"$", "-test.v") cmd.Env = append(os.Environ(), "PLAN14_P3_BOOTSTRAP_CHILD=1", "PLAN14_OFFLINE_UNITS="+string(savedProperties)) if out, err := cmd.CombinedOutput(); err != nil { t.Fatal(err, string(out)) } else { t.Log(string(out)) } return } for _, path := range []string{"/run/carlos", "/run/carlos/hostguard"} { err := os.Mkdir(path, 0755) if err == nil { t.Cleanup(func() { os.RemoveAll(path) }) } else if !os.IsExist(err) { t.Fatal(err) } } claimSystemctl = func(ctx context.Context, args ...string) ([]byte, error) { var properties map[string]string if json.Unmarshal([]byte(os.Getenv("PLAN14_OFFLINE_UNITS")), &properties) != nil || len(args) < 2 || args[0] != "show" || properties[args[1]] == "" { return nil, os.ErrPermission } return []byte(properties[args[1]]), nil } rootMigrationQuiesce = func(context.Context) (func(), error) { return func() {}, nil } for _, path := range []string{"/var/lib/carlos-claims", "/var/lib/carlos-release"} { if err := os.Mkdir(path, 0700); err != nil { t.Fatal("existing authority", path, err) } t.Cleanup(func() { if err := os.RemoveAll(path); err != nil { t.Error(err) } }) } scratch := generationRootRunnerDir(t) t.Setenv("TMPDIR", scratch) store, r, owners, _ := fix4JournalFixture(t) // Use the production private authority, replacing only the older fixture's opener. openClaimRootAuthority = func() (rootstate.Store, error) { return rootstate.Open("/var/lib/carlos-claims/authority/fences", true, false) } actor, err := owners.Identity.Verify(t.Context()) if err != nil { t.Fatal(err) } for path := r.OptDir; path != scratch; path = filepath.Dir(path) { if err := os.Chmod(path, 0755); err != nil { t.Fatal(err) } } if err := os.Mkdir(filepath.Join(r.OptDir, "versions"), 0755); err != nil { t.Fatal(err) } t.Setenv("CARLOS_OPT_DIR", r.OptDir) t.Setenv("CARLOS_DATA_DIR", r.DataDir) t.Setenv("CARLOS_DEPLOYMENT_BUCKET", actor.ControlBucket) key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{1}, ed25519.SeedSize)) t.Setenv("CARLOS_RELEASE_PUBKEY", base64.StdEncoding.EncodeToString(key.Public().(ed25519.PublicKey))) plan := hostops.BackfillPlan{V: 1, ID: "migration", Phase: "collecting", DirectoryRevision: 1, CreatedAt: time.Now(), Hosts: map[string]hostops.BackfillHost{}} var target hostguard.FenceTarget var historicBytes []byte if historic { target, historicBytes = migrationHistoricFixture(t, store, r, owners.ClaimOwnerBindings, actor, &plan) } raw, err := release.SignRootEnvelope(release.RootSigner(key), plan) if err != nil { t.Fatal(err) } if _, err := store.Put(t.Context(), hostops.BackfillKey, raw, ""); err != nil { t.Fatal(err) } id := rootFloorIdentity{Deployment: actor.ControlBucket, Box: actor.Box, Enrollment: actor.Enrollment} floors, err := openRootMovesFloors(r.OptDir, id, true) if err != nil { t.Fatal(err) } defer floors.close() floor := moves.Floor{Min: 1, MinRelease: "v2.0.0"} if err := floors.reconstruct(t.Context(), rootFloorReconstruction{Identity: id, Floor: &floor, Source: "enrolled root history", Digest: floorTestDigest, Authorization: floorTestAuthorization(t, floors)}); err != nil { t.Fatal(err) } rootFloorRunnerBinary(t, r.OptDir, systemBinaryPath(r.OptDir), "v2.0.0", key) rootFloorRunnerBinary(t, r.OptDir, filepath.Join(systemGuardDir(r.OptDir), "carlos"), "v2.0.0", key) journal, err := hostbox.OpenLaunchJournal("/var/lib/carlos-claims/journals.db") if err != nil { t.Fatal(err) } defer journal.Close() service, err := hostbox.NewJournalService(journal) if err != nil { t.Fatal(err) } if t.Name() == "TestPlan14InitialMigrationExclusionRootRunner" { plan14InitialMigrationExclusion(t, store, r, owners.ClaimOwnerBindings, actor, service, journal) } observedAfter := time.Now().UTC() oldOwner := claimQuiesceOwner claimQuiesceOwner = func() (int, int, error) { return 65534, 65534, nil } defer func() { claimQuiesceOwner = oldOwner }() if historic { held, err := os.Open(filepath.Join(r.DataDir, "historic-data")) if err != nil { t.Fatal(err) } err = migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal) held.Close() if err == nil { t.Fatal("migration completed with an open historic payload") } if fresh { key := stopEvidenceKey(target.Host, "", actor.Box, "historic-stop") if err := store.Delete(t.Context(), key); err != nil { t.Fatal(err) } if err := store.Delete(t.Context(), key+".sig"); err != nil { t.Fatal(err) } held, err := os.Open(filepath.Join(r.DataDir, "historic-data")) if err != nil { t.Fatal(err) } err = migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal) held.Close() if err == nil { t.Fatal("fresh stop accepted held payload") } } if err := r.Registry.UpsertTimer(registry.Timer{Host: target.Host, Name: "unrecorded", Kind: "cron", Spec: "0 0 * * *", Path: "/extra"}); err != nil { t.Fatal(err) } if err := migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal); err == nil { t.Fatal("unrecorded preclaim timer accepted") } if err := r.Registry.DeleteTimer(target.Host, "unrecorded"); err != nil { t.Fatal(err) } } if err := migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal); err != nil { t.Fatal("production bootstrap", err) } if !historic { uid, gid := generationRunnerRegistryUser(t) for path := r.DataDir; path != scratch; path = filepath.Dir(path) { if err := os.Chmod(path, 0755); err != nil { t.Fatal(err) } } if err := os.Chown(r.DataDir, uid, gid); err != nil { t.Fatal(err) } reg, err := openRegistryChild(filepath.Join(r.DataDir, "registry.db"), uid, gid) if err != nil { t.Fatal(err) } if err := reg.Close(); err != nil { t.Fatal(err) } if err := withClaimRootRelease(t.Context(), true, func() (rootstate.Store, error) { return rootstate.Open(rootReleasePrivate, true, false) }, func() error { return rootFloorDowngradeIdle(t.Context()) }); err != nil { t.Fatal("offline floor idle", err) } } state, err := rootstate.Open(rootparticipation.Namespace, false, false) if err != nil { t.Fatal(err) } defer state.Close() c, err := (rootparticipation.Ledger{State: state}).Census() if err != nil || !c.Complete || c.Box != actor.Box || c.NeverEnrolled { t.Fatal("invalid root census", c, err) } keys, err := store.List(t.Context(), "control/host-census/") if err != nil || len(keys) != 1 { t.Fatal(keys, err) } body, _, err := store.Get(t.Context(), keys[0]) if err != nil { t.Fatal(err) } var envelope hostops.CensusEnvelope if json.Unmarshal(body, &envelope) != nil || envelope.Report.Seeds == nil { t.Fatal("no Seeds in actual census") } if err := migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal); err != nil { t.Fatal("completed migration replay", err) } if err := verifyRootMigrationStartup(t.Context(), id, service.Inventory); err != nil { t.Fatal("production startup", err) } if historic { fences, err := openClaimRootAuthority() if err != nil { t.Fatal(err) } defer fences.Close() root := hostguard.NewRootAuthority(fences) g := hostguard.Guard{Root: root, DataDir: r.DataDir, Claims: hostguard.StoreReader{Store: store}} if fresh { view, err := root.View(t.Context(), target.Host, hostguard.Captured{Incarnation: target.Incarnation, Stamp: target.Stamp}) if err != nil || len(view.Receipts) != 1 { t.Fatal("fresh receipt", view, err) } target = view.Receipts[0].Target } receipt, err := g.FenceReceiptFor(t.Context(), target) if err != nil || !fresh && !bytes.Equal(receipt.Bytes, historicBytes) { t.Fatal("historic bytes changed", err) } if fresh { if !receipt.EvidenceAt.After(observedAfter) { t.Fatal("fresh stop inherited historic time", receipt.EvidenceAt) } var body StopReceipt if json.Unmarshal(receipt.Bytes, &body) != nil || body.RequestLimitNS != int64(hostbox.RequestLimit) || body.ObservedBootID == target.BootID { t.Fatal("fresh observation missing horizon or current boot") } historicBytes = receipt.Bytes } view, err := root.View(t.Context(), target.Host, hostguard.Captured{Incarnation: target.Incarnation, Stamp: target.Stamp}) if err != nil || view.Installation.Artifacts.Timers != "refresh" || len(view.MigrationDenies) != 1 { t.Fatal("historic root inventory", view, err) } legacy := filepath.Join(r.DataDir, ".hostclaims") if err := os.Rename(legacy, filepath.Join(scratch, "legacy-moved")); err != nil { t.Fatal(err) } if err := os.Mkdir(legacy, 0700); err != nil { t.Fatal(err) } if err := migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal); err != nil { t.Fatal("legacy rename retry", err) } if err := os.RemoveAll(filepath.Join(r.DataDir, ".hostclaims")); err != nil { t.Fatal(err) } if err := migrateRootClaimAuthority(t.Context(), store, r, owners.ClaimOwnerBindings, actor, service, journal); err != nil { t.Fatal("legacy erased retry", err) } again, err := g.FenceReceiptFor(t.Context(), target) if err != nil || !bytes.Equal(again.Bytes, historicBytes) || !again.EvidenceAt.Equal(receipt.EvidenceAt) { t.Fatal("historic replay renewed evidence", err) } captured := hostguard.Captured{Incarnation: target.Incarnation, Stamp: target.Stamp} effects := 0 launch := func() error { effects++; return exec.Command("/usr/bin/true").Run() } allow := func(*hostclaim.Claim) error { return nil } if err := g.With(t.Context(), target.Host, captured, allow, launch); err == nil || effects != 0 { t.Fatal("old writer admitted", err, effects) } observed, err := hostclaim.Read(t.Context(), store, target.Host) if err != nil { t.Fatal(err) } observed.Claim.Holder.Session++ observed.Claim.Gen++ raw, err := json.Marshal(observed.Claim) if err != nil { t.Fatal(err) } key, _ := hostclaim.Key(target.Host) if _, err = store.Put(t.Context(), key, raw, observed.ETag); err != nil { t.Fatal(err) } if err := g.With(t.Context(), target.Host, captured, allow, launch); err != nil || effects != 1 { t.Fatal("successor writer", err, effects) } if t.Name() == "TestPlan14RetiredMigrationPayloadRootRunner" { if err := g.RetireInstallationWith(t.Context(), target.Host, captured, allow, func() error { return os.ErrPermission }); !errors.Is(err, os.ErrPermission) { t.Fatal("retirement interrupt", err) } fences.Close() if err := verifyRootMigrationStartup(t.Context(), id, service.Inventory); err != nil { t.Fatal("retired migration startup", err) } read := func(ctx context.Context) (generationInventory, error) { return generationInventoryAt(ctx, rootparticipation.Namespace, "/var/lib/carlos-claims/authority/fences", "/var/lib/carlos-claims/journals.db") } snapshot, err := read(t.Context()) if err != nil { t.Fatal(err) } if len(snapshot.Installations) != 1 || snapshot.Installations[0].Installation.Stamp != captured.Stamp || len(snapshot.Payloads) == 0 || len(snapshot.Payloads[0].Inventory.Entries) == 0 { t.Fatal("retired payload lost", snapshot) } verifier := generationReconciler{read: read, kernel: productionGenerationKernelCensus} if _, err := verifier.reconcile(t.Context(), nil); err != nil { t.Fatal("retired generation verification", err) } if _, err := os.Stat(filepath.Join(r.DataDir, "historic-data")); err != nil { t.Fatal("payload removed", err) } } if t.Name() == "TestRootRebootPlan14MigrationInstalledPhase1" || t.Name() == "TestPlan14InitialMigrationExclusionRootRunner" { if err := journal.Close(); err != nil { t.Fatal(err) } saveMigrationInstalled(t, store, r, scratch, target, receipt.Bytes) } } }