//go:build linux package host import ( "bytes" "context" "encoding/json" "errors" "os" "path/filepath" "strings" "testing" "time" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" execlease "github.com/carlosframework/platform/internal/hostexec" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" "golang.org/x/sys/unix" ) // Fix round 1, I1: root reads the agent-written journal the way it opens // every agent-writable path: a FIFO at the path answers at once rather // than holding root under the host guard, a symlink is refused, a file // past the cap is refused before it is decoded, and a cancelled context // reads nothing. func TestRootReaderOpensTheJournalBeneathTheDataRoot(t *testing.T) { ctx := context.Background() claim := &hostclaim.Claim{Host: "shop.example", Holder: &hostclaim.Holder{Incarnation: "inc", Stamp: 9, Session: 7, Generation: "P1", WriterBox: "dst"}, Op: hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "op1", Phase: hostclaim.PhaseRelocatingProbing, Exec: "dst", Journal: map[string]string{"install_stamp": "9", "prepared": "P1", "probe_attempts.dst": "1"}}} o := hostclaim.Observed{Host: "shop.example", Claim: claim} attempt := hostclaim.WakeRetryAttempt{ID: "a", InvocationID: "b", Session: 7, Box: "dst"} good := func(t *testing.T, dd string) relocationAttemptJournal { t.Helper() j, _ := readRelocationAttempts(dd, "shop.example", "op1") j.Sessions["7"] = 1 j.Starts["1"] = relocationStart{Session: 7, Generation: "P1", Stamp: 9, State: probeReserved, Attempt: attempt} return j } t.Run("a FIFO", func(t *testing.T) { dd := t.TempDir() path, _ := relocationAttemptsPath(dd, "op1") if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { t.Fatal(err) } if err := unix.Mkfifo(path, 0o600); err != nil { t.Fatal(err) } done := make(chan error, 1) go func() { _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o) done <- err }() select { case err := <-done: if !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a FIFO: %v", err) } case <-time.After(5 * time.Second): t.Fatal("root blocked on a FIFO at the journal's path") } }) t.Run("a symlink", func(t *testing.T) { dd := t.TempDir() elsewhere := t.TempDir() if err := good(t, elsewhere).write(elsewhere); err != nil { t.Fatal(err) } real, _ := relocationAttemptsPath(elsewhere, "op1") path, _ := relocationAttemptsPath(dd, "op1") if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { t.Fatal(err) } if err := os.Symlink(real, path); err != nil { t.Fatal(err) } if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a symlink to a sound journal was followed: %v", err) } // The same journal in place reads. if err := os.Remove(path); err != nil { t.Fatal(err) } if err := good(t, dd).write(dd); err != nil { t.Fatal(err) } if got, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); err != nil || got != attempt { t.Fatalf("the journal in place: %+v %v", got, err) } }) t.Run("a symlinked directory", func(t *testing.T) { dd := t.TempDir() elsewhere := t.TempDir() if err := good(t, elsewhere).write(elsewhere); err != nil { t.Fatal(err) } if err := os.MkdirAll(filepath.Join(dd, "relocate"), 0o700); err != nil { t.Fatal(err) } if err := os.Symlink(filepath.Join(elsewhere, "relocate", "ops"), filepath.Join(dd, "relocate", "ops")); err != nil { t.Fatal(err) } if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a symlinked ops directory was followed: %v", err) } }) t.Run("past the cap", func(t *testing.T) { dd := t.TempDir() j := good(t, dd) raw, _ := json.Marshal(j) raw = append(raw, bytes.Repeat([]byte(" "), relocationAttemptsLimit+1-len(raw))...) path, _ := relocationAttemptsPath(dd, "op1") if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(path, raw, 0o600); err != nil { t.Fatal(err) } if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a journal past the cap: %v", err) } // Trailing whitespace within the cap decodes. if err := os.WriteFile(path, raw[:relocationAttemptsLimit], 0o600); err != nil { t.Fatal(err) } if got, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(ctx, o); err != nil || got != attempt { t.Fatalf("a journal at the cap: %+v %v", got, err) } }) t.Run("a cancelled context", func(t *testing.T) { dd := t.TempDir() if err := good(t, dd).write(dd); err != nil { t.Fatal(err) } gone, cancel := context.WithCancel(ctx) cancel() if _, err := (relocationAttemptReader{DataDir: dd}).ReadAttempt(gone, o); !errors.Is(err, context.Canceled) { t.Fatalf("a cancelled read: %v", err) } }) } // Fix round 1, M4: root's authorize, with the relocation reader installed, // over a journal relocationProbeAttempts.Reserve actually wrote for P's // opening. The reservation the agent presents binds end to end: the // attempt id as the role reservation, the invocation id, the launch-input // digest, the peer's boot and controller. Another invocation, a journal // rewritten under the agent, and a journal replaced by a FIFO are refused. func TestRootAuthorizesAReservationTheRelocationJournalHolds(t *testing.T) { ctx := t.Context() dataDir := t.TempDir() f := newROFixture(t, roOptions{}) f.inputs.Launch = func(context.Context, registry.Route) (ClaimServingLaunch, error) { return ClaimServingLaunch{Identity: hostwrite.Identity{Config: "sha256:config", Release: "sha256:release"}, Executable: "/opt/carlos/versions/v1/app", ExecutableSHA256: strings.Repeat("b", 64), Version: "v1", Args: []string{"--socket", filepath.Join(provision.InstanceRunDir(roHost), "app.sock"), "--db", provision.NestedDBPath(dataDir, roHost)}, Env: []string{"CARLOS_ADMIN_TOKEN=test"}}, nil } in, err := f.inputs.Select(ctx, f.route, f.read(), roProbe) if err != nil { t.Fatal(err) } admitted := f.admit(f.read(), in) h := admitted.Claim.Holder enrollment := strings.Repeat("a", 32) boot, err := execlease.BootID() if err != nil { t.Fatal(err) } start, err := claimConstructionProcessStart() if err != nil { t.Fatal(err) } peer := StopPeer{UID: 1001, PID: os.Getpid(), ProcessStart: start, BootID: boot, ControllerID: "controller-1"} target := hostguard.FenceTarget{Region: "ie", Host: roHost, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Op: roOp, Phase: hostclaim.PhaseRelocatingProbing, Box: "dst", Enrollment: enrollment, BootID: peer.BootID, ControllerID: peer.ControllerID, ClaimETag: admitted.ETag, DirectoryRevision: "7"} // The agent's side: the journal Reserve writes for the admitted session. attempts := &relocationProbeAttempts{Claims: f.claims, DataDir: dataDir, Box: "dst", NewID: f.inputs.NewID, Now: f.inputs.Now} s := OperationStart{Admitted: admitted, Policy: roProbe, Inputs: in, Owner: target} if err := attempts.Reserve(ctx, s, 0); err != nil { t.Fatal(err) } a := attempts.CurrentAttempt(s) if a.ID == "" { t.Fatal("no reservation") } // Root's side: its own providers, the reader over the same data root. exe := hostbox.LaunchFile{Device: 5, Inode: 6} kernel := &launchKernel{path: in.Executable, want: exe, digest: in.ExecutableSHA256} root := &ClaimLaunchRoot{ValidateExecutable: func(string, string, string) error { return nil }, Local: &rev2Local{}, Clock: &rev2Clock{wall: time.Now(), boot: 100}, DataRoot: dataDir, ExecLockDir: t.TempDir(), Attempts: map[string]claimLaunchAttemptReader{hostclaim.KindRelocating: relocationAttemptReader{DataDir: dataDir}}, Guard: hostguard.Guard{DataDir: dataDir, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: f.claims}}, Actor: func(context.Context) (hostbox.Actor, error) { return hostbox.Actor{Box: "dst", Enrollment: enrollment, Revision: 7, Binding: "7", Region: "ie", Eligible: true}, nil }, Kernel: kernel} configureLaunchParticipation(t, &launchFixture{root: root, peer: peer, target: target, claims: &launchClaims{obs: admitted}}) spec := &hostbox.LaunchSpec{Identity: in.Identity, Executable: in.Executable, Digest: in.ExecutableSHA256, Args: in.Args, Env: in.Env} reservation := hostbox.LaunchReservation{ID: a.ID + ":main", Role: "main", InvocationID: a.InvocationID, Inputs: a.LaunchInputs, Target: target, Group: hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup(roHost)}, Executable: exe} q := hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStart, Spec: spec, Reservation: &reservation, Executable: in.Executable, MemoryMax: cgroup.TierDefault.MemoryMax} if err := root.authorize(ctx, peer, q, admitted.Claim); err != nil { t.Fatalf("the reservation the journal holds: %v", err) } other := reservation other.InvocationID = "another" q.Reservation = &other if err := root.authorize(ctx, peer, q, admitted.Claim); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("another invocation: %v", err) } q.Reservation = &reservation // The agent rewrites its journal: root's reservation no longer matches. j, err := readRelocationAttempts(dataDir, roHost, roOp) if err != nil { t.Fatal(err) } st := j.Starts["1"] st.Attempt.InvocationID = "rewritten" j.Starts["1"] = st if err := j.write(dataDir); err != nil { t.Fatal(err) } if err := root.authorize(ctx, peer, q, admitted.Claim); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a rewritten journal: %v", err) } // A FIFO in the journal's place refuses at once. path, _ := relocationAttemptsPath(dataDir, roOp) if err := os.Remove(path); err != nil { t.Fatal(err) } if err := unix.Mkfifo(path, 0o600); err != nil { t.Fatal(err) } done := make(chan error, 1) go func() { done <- root.authorize(ctx, peer, q, admitted.Claim) }() select { case err := <-done: if !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("a FIFO: %v", err) } case <-time.After(5 * time.Second): t.Fatal("root blocked on a FIFO in the journal's place") } }