//go:build linux package host import ( "context" "errors" "net" "net/http" "os" "os/exec" "path/filepath" "strings" "testing" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/provision" "golang.org/x/sys/unix" ) // serveClaimRoot runs the production instance-control mux in claim mode with // h as its claim handler, on a real unix socket. func serveClaimRoot(t *testing.T, h http.Handler) string { t.Helper() return serveClaimRootWith(t, h, nil) } // serveClaimRootWithHeldRelease serves h as the claim root with the moves // held-release route registered to it too, as the host registers it with // both flags on (whole-branch fixes, item 6). func serveClaimRootWithHeldRelease(t *testing.T, h http.Handler) string { t.Helper() return serveClaimRootWith(t, h, h) } func serveClaimRootWith(t *testing.T, h, heldRelease http.Handler) string { t.Helper() dir, err := os.MkdirTemp("", "cl") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(dir) }) sock := filepath.Join(dir, "c.sock") ln, err := net.Listen("unix", sock) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) server := &instancectl.Server{ClaimStop: h, ClaimHeldRelease: heldRelease, PeerOK: func(net.Conn) error { return nil }} done := make(chan struct{}) go func() { defer close(done); _ = server.Serve(ctx, ln) }() t.Cleanup(func() { cancel(); <-done }) return sock } func procSelfStart(t *testing.T) string { t.Helper() raw, err := os.ReadFile("/proc/self/stat") if err != nil { t.Fatal(err) } fields := strings.Fields(string(raw)[strings.LastIndexByte(string(raw), ')')+1:]) return fields[19] } func TestClaimRoleRootClientThroughTheClaimMux(t *testing.T) { f := newLaunchFixture(t, servingOp()) authenticated := 0 f.root.Authenticate = func(_ context.Context, conn net.Conn) (StopPeer, error) { if conn == nil { t.Fatal("no request connection") } authenticated++ return f.peer, nil } sock := serveClaimRoot(t, ClaimRootWithLaunch{Launch: f.root}) client := NewClaimRoleRootClient(sock) protocol, err := client.Capability(t.Context()) if err != nil || protocol != hostbox.RoleLaunchProtocol { t.Fatalf("capability=%q %v", protocol, err) } grant, err := client.Launch(t.Context(), f.req) if err != nil || grant.Check(*f.req.Reservation) != nil { t.Fatalf("grant=%+v %v", grant, err) } if _, err := client.Launch(t.Context(), f.req); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("replay over the wire: %v", err) } if err := client.Stop(t.Context(), hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStop, Reservation: f.req.Reservation}); err != nil { t.Fatal(err) } if authenticated != 4 { t.Fatalf("authenticated %d of 4 requests", authenticated) } // An unauthenticated peer gets nothing, not even the capability. f.root.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return StopPeer{}, errors.New("not the edge") } if _, err := client.Capability(t.Context()); err == nil { t.Fatal("unauthenticated capability") } // A body the protocol cannot carry is refused before authority checks. f.root.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return f.peer, nil } resp, err := (&http.Client{Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "unix", sock) }}}).Post("http://control/claim/launch", "application/json", strings.NewReader(`{"v":1,"op":"start","argv":["/bin/sh"]}`)) if err != nil { t.Fatal(err) } resp.Body.Close() if resp.StatusCode != http.StatusBadRequest { t.Fatalf("unknown field status=%d", resp.StatusCode) } } func TestClaimRoleLaunchEndToEndThroughTheRootProtocol(t *testing.T) { f := newLaunchFixture(t, servingOp()) f.root.Authenticate = func(context.Context, net.Conn) (StopPeer, error) { return f.peer, nil } base, err := os.MkdirTemp("", "e2e") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(base) }) exe := filepath.Join(base, "app") if err := os.WriteFile(exe, []byte("build"), 0o555); err != nil { t.Fatal(err) } var st unix.Stat_t if err := unix.Stat(exe, &st); err != nil { t.Fatal(err) } f.kernel.path, f.kernel.want = exe, hostbox.LaunchFile{Device: uint64(st.Dev), Inode: st.Ino} f.kernel.observed.Executable = f.kernel.want f.kernel.observed.Running = f.kernel.want f.kernel.observed.ProcessStart = procSelfStart(t) f.units.started.MainPID = os.Getpid() sock := serveClaimRoot(t, ClaimRootWithLaunch{Launch: f.root}) work, err := (hostwrite.Config{}).Select(t.Context(), f.target.Host, "spawn") if err != nil { t.Fatal(err) } f.root.DataRoot, f.root.runRoot = base, filepath.Join(base, "run") launchDir, runDir := provision.InstanceDir(base, f.target.Host), filepath.Join(f.root.runRoot, provision.InstanceID(f.target.Host)) if err := os.MkdirAll(launchDir, 0o700); err != nil { t.Fatal(err) } launch := activator.RoleLaunch{Work: work, Target: f.target, Root: NewClaimRoleRootClient(sock), LaunchDir: launchDir, RunDir: runDir, LogSocket: filepath.Join(base, "l.sock"), MemoryMax: cgroup.TierDefault.MemoryMax, Executable: func(context.Context, *exec.Cmd) (*os.File, error) { return os.Open(exe) }} attempt := hostclaim.WakeRetryAttempt{ID: "attempt-1", InvocationID: "inv-1", LaunchInputs: "inputs-1", Session: 13, Box: "box-1", Enrollment: "enrollment-1", BootID: f.peer.BootID, ControllerID: "controller-1"} build := func() *exec.Cmd { cmd := exec.Command("/opt/carlos/live/shop.example", "--socket", filepath.Join(runDir, "app.sock"), "--db", filepath.Join(launchDir, "app.db")) cmd.Env = []string{"CARLOS_ADMIN_TOKEN=test"} return cmd } launch.Spec = &hostbox.LaunchSpec{Identity: f.req.Spec.Identity, Executable: exe, Digest: f.kernel.digest, Args: build().Args[1:], Env: build().Env} attempt.LaunchInputs, _ = launch.Spec.Hash() f.claims.set(func(c *hostclaim.Claim) { a := attempt; c.Holder.WakeRetry.Attempt = &a }) cmd, _, settle, _, err := launch.Start(t.Context(), "main", attempt, build, func() error { return nil }) if err != nil { t.Fatal(err) } if cmd.Process.Pid != os.Getpid() { t.Fatalf("pid=%d", cmd.Process.Pid) } got, err := f.journal.UnitReservation(t.Context(), f.target, "attempt-1:main") if err != nil || got.State != hostbox.UnitStarted || got.InvocationID != "inv-1" || got.Inputs != attempt.LaunchInputs { t.Fatalf("root journal=%+v %v", got, err) } if err := settle(t.Context()); err != nil { t.Fatal(err) } if got, err = f.journal.UnitReservation(t.Context(), f.target, "attempt-1:main"); err != nil || got.State != hostbox.UnitSettled { t.Fatalf("settled=%+v %v", got, err) } // The same attempt cannot launch twice, even from the live controller. if _, _, _, _, err := launch.Start(t.Context(), "main", attempt, build, func() error { return nil }); err == nil { t.Fatal("spent attempt relaunched") } } func TestClaimRolePreflightAdmitsOnlyTheCompleteProtocol(t *testing.T) { var nilAssembly *ClaimRoleAssembly for name, a := range map[string]*ClaimRoleAssembly{ "nil assembly": nilAssembly, "no protocol": {}, "root refuses": {Root: fakeLaunchRoot{err: activator.ErrClaimIsolationUnavailable}}, "older protocol": {Root: fakeLaunchRoot{protocol: "isolated-unit-v0"}}, "unreachable": {Root: fakeLaunchRoot{err: errors.New("dial unix: no such file")}}, } { if err := a.Preflight(t.Context(), ClaimLaunchInputs{}); !errors.Is(err, activator.ErrClaimIsolationUnavailable) { t.Fatalf("%s: %v", name, err) } } if err := (&ClaimRoleAssembly{Root: fakeLaunchRoot{protocol: hostbox.RoleLaunchProtocol}}).Preflight(t.Context(), ClaimLaunchInputs{}); err != nil { t.Fatal(err) } } func TestClaimRoleIsolatedPathsBindTheNestedInstance(t *testing.T) { roots := map[string]string{"data": "/data"} host := "shop.example" got, err := claimIsolatedPaths(roots, host, ClaimLaunchInputs{DBPath: "/data/instances/" + iidOf(host) + "/shop.db", Limit: cgroup.TierDefault}) if err != nil { t.Fatal(err) } if got.LaunchDir != "/data/instances/"+iidOf(host) || got.RunDir != "/run/carlos/instances/"+iidOf(host) || !strings.HasSuffix(got.LogSocket, ".run.sock") || got.MemoryMax != cgroup.TierDefault.MemoryMax { t.Fatalf("paths=%+v", got) } for name, in := range map[string]ClaimLaunchInputs{ "flat database": {DBPath: "/data/shop.example.db", Limit: cgroup.TierDefault}, "other instance": {DBPath: "/data/instances/other/shop.db", Limit: cgroup.TierDefault}, "nested deeper": {DBPath: "/data/instances/" + iidOf(host) + "/x/shop.db", Limit: cgroup.TierDefault}, "not a tier": {DBPath: "/data/instances/" + iidOf(host) + "/shop.db", Limit: cgroup.Limit{MemoryMax: 7}}, "relative": {DBPath: "instances/shop.db", Limit: cgroup.TierDefault}, "unescaped parent": {DBPath: "/data/instances/" + iidOf(host) + "/../x/shop.db", Limit: cgroup.TierDefault}, } { if _, err := claimIsolatedPaths(roots, host, in); !errors.Is(err, hostbox.ErrRoleLaunchShape) { t.Fatalf("%s: %v", name, err) } } } func TestProductionClaimLaunchWrapsTheRootHandler(t *testing.T) { journal, err := hostbox.OpenLaunchJournal(filepath.Join(t.TempDir(), "launch.db")) if err != nil { t.Fatal(err) } defer journal.Close() inner := stopKernelFunc(func(context.Context, hostguard.FenceTarget, StopInventory, DrainAcknowledgement) (StopKernelEvidence, error) { return StopKernelEvidence{}, nil }) authenticate := func(context.Context, net.Conn) (StopPeer, error) { return StopPeer{}, nil } root := ClaimRootHTTP{Launches: journal, Stop: ClaimStopHTTP{Authenticate: authenticate, Coordinator: ClaimStopCoordinator{Kernel: inner, Guard: hostguard.Guard{DataDir: "/data", Claims: &launchClaims{}}}}} actor := func(context.Context) (hostbox.Actor, error) { return hostbox.Actor{}, nil } wrapped, err := claimLaunchFor(root, actor, map[string]string{"opt": "/opt/carlos", "data": "/data"}) if err != nil { t.Fatal(err) } k, ok := wrapped.Root.Stop.Coordinator.Kernel.(claimUnitStopKernel) if !ok || k.Journal != journal || k.Inner == nil { t.Fatalf("stop kernel not wrapped: %T", wrapped.Root.Stop.Coordinator.Kernel) } l := wrapped.Launch units, ok := l.Units.(instancectl.ClaimUnits) if !ok || units.Systemctl == nil || units.CgroupRoot != "/sys/fs/cgroup" || !strings.HasSuffix(units.ExeRoot, "/exe") || units.Room == nil { t.Fatalf("units=%+v", l.Units) } if kernel, ok := l.Kernel.(linuxClaimLaunchKernel); !ok || kernel.OptRoot != "/opt/carlos" || kernel.CgroupRoot != "/sys/fs/cgroup" || kernel.ProcRoot != "/proc" || kernel.Owner != 0 { t.Fatalf("kernel=%+v", l.Kernel) } if l.Journal != journal || l.Authenticate == nil || l.Actor == nil || l.Guard.DataDir != "/data" { t.Fatalf("launch root=%+v", l) } for name, bad := range map[string]func(*ClaimRootHTTP, *map[string]string){ "no journal": func(r *ClaimRootHTTP, _ *map[string]string) { r.Launches = nil }, "no authenticate": func(r *ClaimRootHTTP, _ *map[string]string) { r.Stop.Authenticate = nil }, "no stop kernel": func(r *ClaimRootHTTP, _ *map[string]string) { r.Stop.Coordinator.Kernel = nil }, "unbound data root": func(_ *ClaimRootHTTP, m *map[string]string) { (*m)["data"] = "/elsewhere" }, "no opt root": func(_ *ClaimRootHTTP, m *map[string]string) { *m = map[string]string{"data": "/data"} }, } { r := root m := map[string]string{"opt": "/opt/carlos", "data": "/data"} bad(&r, &m) if _, err := claimLaunchFor(r, actor, m); err == nil { t.Fatalf("%s accepted", name) } } if _, err := WithProductionClaimLaunch(t.Context(), http.NotFoundHandler()); err == nil { t.Fatal("wrapped a handler that is not the claim root") } if _, err := WithProductionClaimLaunch(t.Context(), root); err == nil { t.Fatal("wrapped without the production assembly") } } func iidOf(host string) string { return provision.InstanceID(host) }