//go:build linux && rootrunner package host import ( "context" "crypto/ed25519" "crypto/sha256" "encoding/hex" "encoding/json" "net" "os" "os/exec" "os/user" "path/filepath" "strconv" "strings" "testing" "time" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/controlsock" "github.com/carlosframework/platform/internal/hostactors" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostfiles" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostops" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) type seedCLIConfiguration struct { Actor hostbox.Actor Claims, Executable string } func TestRootFilesSeedDelegatedOneShot(t *testing.T) { runServingParkPeerNamespace(t, false, false, "seed-cli") } func rootFilesSeedDelegatedCLI(t *testing.T, binary string) { for _, p := range []string{"/data", "/var/lib/carlos-claims", "/run/carlos-control", "/run/carlos", "/opt/carlos"} { if err := os.Mkdir(p, 0755); err != nil { t.Fatal("existing CLI fixture", p, err) } t.Cleanup(func() { os.RemoveAll(p) }) } var captured *servingPreparation var executor *filesSeedExecutor capture := filesSeedLiveWitness{seed: func(_ *testing.T, p *servingPreparation, e *filesSeedExecutor) { captured, executor = p, e }} runFilesSeedPreparationWithLive(t, "real-seed", "same-controller", []bool{true}, []filesSeedLiveWitness{capture}) f := captured.f f.actor.ControlBucket = "deployment" cfg := seedCLIConfiguration{Actor: f.actor, Claims: "/data/cli-claims", Executable: "/opt/carlos/versions/fixture/app"} claims := release.NewDirStore(cfg.Claims) keys, err := f.f.claims.List(t.Context(), "") if err != nil { t.Fatal(err) } for _, key := range keys { b, _, err := f.f.claims.Get(t.Context(), key) if err != nil { t.Fatal(err) } if _, err := claims.Put(t.Context(), key, b, ""); err != nil { t.Fatal(err) } } f.f.claims, f.f.evidence = claims, claims f.f.guard.Claims = hostguard.StoreReader{Store: claims} f.f.guard.LockDir = "" f.p.root.Stop.Coordinator.Claims = hostguard.StoreReader{Store: claims} f.p.root.Stop.Coordinator.Guard = f.f.guard f.p.root.Stop.Coordinator.Evidence = claims f.p.root.Stop.Coordinator.Now = time.Now production, err := NewProductionClaimStopHandler(f.p.root.Stop.Coordinator, f.f.journal, productionStopControllers{}, f.p.root.Roots) if err != nil { t.Fatal(err) } f.p.root.Stop = production.(ClaimStopHTTP) // Initial installation/roster and the no-process kernel census are fixtures. f.f.kernel.noProcess = true f.p.root.Stop.Coordinator.Kernel = f.f.kernel f.p.root.Stop.Coordinator.RetainNoProcess = f.p.retainLegacyNoProcess reg, err := registry.Open("/data/registry.db") if err != nil { t.Fatal(err) } t.Cleanup(func() { reg.Close() }) row, err := executor.registry.Get(recoveryHost) if err != nil || row == nil { t.Fatal(err) } owned, err := f.p.root.Reconcile.Registry.Get(recoveryHost) if err != nil || owned == nil || !owned.ClaimOwnership.Valid() { t.Fatal("missing root-owned fixture route", err) } row.ClaimOwnership = owned.ClaimOwnership if err := reg.Upsert(*row); err != nil { t.Fatal(err) } // Production root and agent observe the same ownership transfer. f.p.root.Reconcile.Registry = reg h := servingTransportRoot(t, f) containmentRoot := claimContainmentAgentRoot capability := newLaunchFixture(t, hostclaim.Op{Kind: hostclaim.KindServing, Exec: f.actor.Box}).root // The capability fixture assumes system.slice; this installed edge uses carlos.slice. claimContainmentAgentRoot = containmentRoot capability.Authenticate = h.Stop.Authenticate capability.SeedController = verifyFilesSeedDelegation listener, err := net.Listen("unix", controlsock.Path) if err != nil { t.Fatal(err) } if err := os.Chmod(controlsock.Path, 0666); err != nil { t.Fatal(err) } server := &instancectl.Server{ClaimStop: ClaimRootWithLaunch{Root: h, Launch: capability}, PeerOK: func(net.Conn) error { return nil }, Logf: t.Logf} ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- server.Serve(ctx, listener) }() t.Cleanup(func() { cancel() if err := <-done; err != nil { t.Error(err) } }) if err := os.MkdirAll(filepath.Dir(cfg.Executable), 0755); err != nil { t.Fatal(err) } image, err := os.ReadFile(binary) if err != nil { t.Fatal(err) } if err := os.WriteFile(cfg.Executable, image, 0755); err != nil { t.Fatal(err) } raw, _ := json.Marshal(cfg) if err := os.WriteFile("/data/seed-cli.json", raw, 0600); err != nil { t.Fatal(err) } account, err := user.Lookup("carlos") if err != nil { t.Fatal(err) } uid, _ := strconv.Atoi(account.Uid) gid, _ := strconv.Atoi(account.Gid) seedCLISharedLocks(t, "/run/carlos/hostguard", uid, gid) if err := filepath.Walk("/data", func(path string, info os.FileInfo, err error) error { if err != nil { return err } return os.Chown(path, uid, gid) }); err != nil { t.Fatal(err) } raw, err = unitsFS.ReadFile("units/carlos-edge.service") if err != nil { t.Fatal(err) } var lines []string for _, line := range strings.Split(string(raw), "\n") { if strings.HasPrefix(line, "Requires=") || strings.HasPrefix(line, "After=") || strings.HasPrefix(line, "Wants=") || strings.HasPrefix(line, "ExecStart=") || strings.HasPrefix(line, "ExecReload=") || strings.HasPrefix(line, "EnvironmentFile=") || strings.HasPrefix(line, "Restart=") { continue } lines = append(lines, line) } lines = append(lines, "[Service]", "Type=oneshot", "RemainAfterExit=yes", "Restart=no", "TimeoutStartSec=60s", "TimeoutStopSec=45s", "ExecStart="+binary+" -test.run=^TestRootFilesSeedCLIPayload$ -test.v -test.timeout=50s", "Environment=FILES_SEED_CLI=1 TMPDIR=/data CARLOS_HOST_CLAIM=1 CARLOS_HOST_CLAIM_ORIGINS=1 CARLOS_DATA_DIR=/data CARLOS_DEPLOYMENT_DIR=/data/cli-claims AWS_REGION=eu-west-1 AWS_DEFAULT_REGION=eu-west-1 AWS_ACCESS_KEY_ID=test-key AWS_SECRET_ACCESS_KEY=test-secret AWS_EC2_METADATA_DISABLED=true CARLOS_S3_ENDPOINT="+os.Getenv("CARLOS_S3_ENDPOINT"), "IPAddressDeny=any", "IPAddressAllow=127.0.0.1") unit := "/run/systemd/system/carlos-edge.service" file, err := os.OpenFile(unit, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) if err != nil { t.Fatal(err) } _, err = file.WriteString(strings.Join(lines, "\n") + "\n") file.Close() if err != nil { t.Fatal(err) } t.Cleanup(func() { exec.Command("systemctl", "stop", "carlos-edge.service").Run() os.Remove(unit) exec.Command("systemctl", "daemon-reload").Run() }) servingRunnerSystemctl(t, "daemon-reload") run := func() { out, err := exec.CommandContext(t.Context(), "systemctl", "start", "carlos-edge.service").CombinedOutput() logs, _ := exec.Command("journalctl", "-u", "carlos-edge.service", "--no-pager").CombinedOutput() t.Log(string(logs)) if err != nil { t.Fatal("delegated CLI", err, string(out)) } state, err := exec.Command("systemctl", "show", "carlos-edge.service", "-p", "ExecMainStatus", "-p", "Result", "-p", "SubState").Output() if err != nil || !strings.Contains(string(state), "ExecMainStatus=0") || !strings.Contains(string(state), "Result=success") { t.Fatal("one-shot status", string(state), err) } } run() observed := f.f.read() if observed.Claim.Origins == nil || observed.Claim.Holder.WakeRetry == nil || observed.Claim.Holder.WakeRetry.AttemptsReserved != 1 || observed.Claim.Holder.WakeRetry.State != "open" { t.Fatal("one-shot did not settle publication", observed) } files, err := executor.source.Open(t.Context(), claims, executor.assembly.Data, f.actor) if err != nil { t.Fatal(err) } if _, found, err := hostfiles.Load(t.Context(), files.(*movesFiles).Files, movesFilesPrefix(observed.Claim.Holder.Location, observed.Claim.Holder.Generation)); err != nil || !found { t.Fatal("CLI did not publish", err) } servingRunnerSystemctl(t, "stop", "carlos-edge.service") etag := observed.ETag run() if f.f.read().ETag != etag { t.Fatal("delegated established rerun mutated claim") } } func TestRootFilesSeedCLIPayload(t *testing.T) { if os.Getenv("FILES_SEED_CLI") != "1" || os.Geteuid() == 0 { t.Fatal("invoke through delegated CLI fixture") } body, err := os.ReadFile("/data/seed-cli.json") if err != nil { t.Fatal(err) } var cfg seedCLIConfiguration if err := json.Unmarshal(body, &cfg); err != nil { t.Fatal(err) } claimOwnerRegistration.Provider = seedCLIOwners{t: t, cfg: cfg} RunSystemFiles([]string{"seed", "--host", recoveryHost}) } type seedCLIOwners struct { t *testing.T cfg seedCLIConfiguration } func (o seedCLIOwners) OpenClaimOwners(ctx context.Context, store release.Store, r ClaimRuntimeResources) (ClaimOwnerBindings, error) { _, _, defaults, _ := fix4JournalFixture(o.t) owners := defaults.ClaimOwnerBindings owners.Identity = handoverRecoveryIdentity(o.t, o.cfg.Actor) owners.EvidenceKey = nil owners.RetirementAuthority = &hostops.ProductionAuthority{Store: store, Directory: hostactors.Directory{Store: store, PublicKey: ed25519.NewKeyFromSeed(make([]byte, 32)).Public().(ed25519.PublicKey), Bucket: o.cfg.Actor.ControlBucket}, Now: time.Now} data, err := lease.NewVersionedS3(ctx, map[string]string{o.cfg.Actor.Bucket: "eu-west-1"}) if err != nil { return owners, err } owners.Data = data owners.Local = &rev2Local{} owners.Controllers = productionStopControllers{} fixture := plan9D5Fixture(o.t) fixture.launch.Executable = o.cfg.Executable image, err := os.ReadFile(o.cfg.Executable) if err != nil { return owners, err } sum := sha256.Sum256(image) fixture.launch.ExecutableSHA256 = hex.EncodeToString(sum[:]) fixture.launch.Args = []string{"--socket", provision.NestedSocket(recoveryHost), "--db", filepath.Join(r.DataDir, "instances", recoveryHost, recoveryHost+".db")} fixture.launch.Limit = cgroup.TierDefault fixture.launch.Env = []string{"CARLOS_ADMIN_TOKEN=test"} owners.Serving = fixture.owners.Sources files, err := productionClaimFilesSource().Open(ctx, store, data, o.cfg.Actor) if err != nil { return owners, err } owners.Serving.Files = files return owners, nil } func TestFilesSeedCLIStopFixture(t *testing.T) { var p *servingPreparation capture := filesSeedLiveWitness{seed: func(_ *testing.T, got *servingPreparation, _ *filesSeedExecutor) { p = got }} runFilesSeedPreparationWithLive(t, "real-seed", "same-controller", nil, []filesSeedLiveWitness{capture}) f := p.f claims := release.NewDirStore(filepath.Join(t.TempDir(), "claims")) keys, err := f.f.claims.List(t.Context(), "") if err != nil { t.Fatal(err) } for _, key := range keys { body, _, err := f.f.claims.Get(t.Context(), key) if err != nil { t.Fatal(err) } if _, err := claims.Put(t.Context(), key, body, ""); err != nil { t.Fatal(err) } } f.f.claims, f.f.evidence = claims, claims f.f.guard.Claims = hostguard.StoreReader{Store: claims} f.f.guard.LockDir = filepath.Join(t.TempDir(), "hostguard") seedCLISharedLocks(t, f.f.guard.LockDir, os.Getuid(), os.Getgid()) c := f.p.root.Stop.Coordinator c.Claims, c.Guard, c.Evidence = hostguard.StoreReader{Store: claims}, f.f.guard, claims c.Now = time.Now f.f.kernel.noProcess = true c.Kernel = f.f.kernel f.p.root.Stop.Coordinator = c c.RetainNoProcess = f.p.retainLegacyNoProcess c.VerifiedRosterRevision = fixedRosterRevision(f.actor.Revision) o := f.f.read() h := o.Claim.Holder req := StopRequest{Region: f.actor.Region, Host: o.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: f.actor.Box, Enrollment: f.actor.Enrollment, BootID: f.f.peer.BootID, ControllerID: f.f.peer.ControllerID, ExpectedClaimETag: o.ETag, ExpectedDirectoryRevision: f.actor.Binding, RosterRevision: f.actor.Revision} if _, err := c.FenceAndStop(t.Context(), req); err != nil { t.Fatal("CLI fixture no-process", err) } } func seedCLISharedLocks(t *testing.T, path string, uid, gid int) { t.Helper() // Match bootstrap's carlos-owned, setgid shared-lock directory. if err := os.Mkdir(path, 0770); err != nil { t.Fatal(err) } if err := os.Chown(path, uid, gid); err != nil { t.Fatal(err) } if err := os.Chmod(path, os.ModeSetgid|0770); err != nil { t.Fatal(err) } }