//go:build linux && rootrunner package host import ( "context" "io" "net" "net/http" "os" "os/exec" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/lease/leasetest" "github.com/carlosframework/platform/internal/provision" ) func seedTwoHostNames() []string { if os.Getenv("FILES_SEED_ISOLATION_MUTATION") == "1" { return []string{"seed-negative-a.bab.oncarlos.com", "seed-negative-b.bab.oncarlos.com"} } return []string{"seed-a.bab.oncarlos.com", "seed-b.bab.oncarlos.com"} } func TestRootFilesSeedTwoHostIsolation(t *testing.T) { runServingParkPeerNamespace(t, false, false, "seed-two-host") } type seedLiveHost struct { p *servingPreparation executor *filesSeedExecutor endpoint string files *movesFiles witness filesSeedLiveWitness stop func() observed hostclaim.Observed } func rootFilesSeedTwoHost(t *testing.T, binary string) { witness := rootFilesSeedIsolatedWitness(t, binary, true) hosts := make([]*seedLiveHost, 0, 2) for i, host := range seedTwoHostNames() { h := &seedLiveHost{witness: witness} capture := filesSeedLiveWitness{host: host, shared: i > 0, seed: func(t *testing.T, p *servingPreparation, e *filesSeedExecutor) { h.p, h.executor, h.endpoint = p, e, os.Getenv("CARLOS_S3_ENDPOINT") if err := os.WriteFile(filepath.Join(filepath.Dir(p.f.db), host, "payload"), []byte("real seeded bytes:"+host), 0600); err != nil { t.Fatal(err) } }} runFilesSeedPreparationWithLive(t, "real-seed", "same-controller", []bool{true}, []filesSeedLiveWitness{capture}) hosts = append(hosts, h) } for _, h := range hosts { t.Setenv("CARLOS_S3_ENDPOINT", h.endpoint) if err := h.executor.run(t.Context(), h.p.f.f.hostName(), io.Discard); err != nil { t.Fatal("seed", err) } f := h.p.f opened, err := h.executor.source.Open(t.Context(), f.f.claims, f.f.data.(*leasetest.VersionedBucket), f.actor) if err != nil { t.Fatal(err) } h.files = opened.(*movesFiles) h.wake(t) } for _, h := range hosts { h.probe(t) } // Retire each real launched generation while the other tenant stays live. for i, h := range hosts { other := hosts[1-i] other.probe(t) h.stop() previous := h.observed.Claim.Holder.Generation h.wake(t) if h.observed.Claim.Holder.Generation == previous { t.Fatal("cold wake reused generation") } other.probe(t) h.probe(t) } for _, h := range hosts { h.stop() } if strings.HasPrefix(t.Name(), "TestRootRebootFilesSeedTwoHost") && strings.HasSuffix(t.Name(), "Phase1") { saveFilesSeedReboot(t, binary, hosts) } } func (h *seedLiveHost) wake(t *testing.T) { t.Helper() t.Setenv("CARLOS_S3_ENDPOINT", h.endpoint) f := h.p.f f.f.now = f.f.now.Add(time.Minute) p := newServingPreparation(t, f, "boot", "controller", func(w *ClaimWakeAssembly) { w.Inputs.(*claimServingOwners).Sources.Files = h.files w.ServingPrerequisites = servingFilesPrerequisites(f.f.claims, h.files) h.witness.configure(w) }) h.witness.prepare(p) admitted, start := p.prepare(t) if err := start.Restore(t.Context()); err != nil { t.Fatal("restore", err) } if err := start.Reserve(t.Context(), 0); err != nil { t.Fatal("reserve", err) } if err := start.Materialize(t.Context()); err != nil { t.Fatal("materialize", err) } stop := h.witness.start(t, p, admitted, start) h.observed = admitted h.stop = func() { stop() t.Setenv("CARLOS_S3_ENDPOINT", h.endpoint) if err := p.runtime.FailedAdmission(t.Context(), admitted, hostclaim.ErrWrongState); err != nil { t.Fatal("release", err) } } } func (h *seedLiveHost) probe(t *testing.T) { t.Helper() host := h.p.f.f.hostName() transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "unix", provision.NestedSocket(host)) }} defer transport.CloseIdleConnections() response, err := (&http.Client{Transport: transport, Timeout: time.Second}).Get("http://tenant/") if err != nil { t.Fatal("sibling unavailable", host, err) } defer response.Body.Close() body, err := io.ReadAll(response.Body) if err != nil || response.StatusCode != 200 || string(body) != "real seeded bytes:"+host { t.Fatal("sibling changed", host, string(body), err) } } func TestRootFilesSeedIsolationNegativeControl(t *testing.T) { servingRootRunner(t) binary, err := os.Executable() if err != nil { t.Fatal(err) } cmd := exec.CommandContext(t.Context(), binary, "-test.run=^TestRootFilesSeedTwoHostIsolation$", "-test.v", "-test.timeout=2m") cmd.Env = append(os.Environ(), "FILES_SEED_ISOLATION_MUTATION=1") output, err := cmd.CombinedOutput() if err == nil || !strings.Contains(string(output), "isolation mutation detected: /tmp/exposed-root") { t.Fatalf("isolation mutant not caught: %v\n%s", err, output) } t.Log("root-only mutation caught: exposed private-root file was read by the tenant") }