package host import ( "context" "errors" "os" "path/filepath" "strconv" "sync" "testing" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostclaim" execlease "github.com/carlosframework/platform/internal/hostexec" "github.com/carlosframework/platform/internal/hostguard" "github.com/carlosframework/platform/internal/instancectl" ) type launchClaims struct { mu sync.Mutex obs hostclaim.Observed } func (c *launchClaims) Read(_ context.Context, host string) (hostclaim.Observed, error) { c.mu.Lock() defer c.mu.Unlock() o := c.obs if o.Claim != nil { cl := *o.Claim h := *cl.Holder cl.Holder = &h o.Claim = &cl } return o, nil } func (c *launchClaims) set(fn func(*hostclaim.Claim)) { c.mu.Lock() defer c.mu.Unlock() cl := *c.obs.Claim h := *cl.Holder cl.Holder = &h fn(&cl) c.obs.Claim = &cl } type launchUnits struct { mu sync.Mutex starts []instancectl.ClaimUnit stops []string startErr error beforeCheck func() sameErr error goneErr error // goneAfterStartErr answers Gone only once a start was attempted. goneAfterStartErr error stopErr error started instancectl.ClaimUnitStarted } func (u *launchUnits) Start(_ context.Context, in instancectl.ClaimUnit) (instancectl.ClaimUnitStarted, error) { u.mu.Lock() defer u.mu.Unlock() u.starts = append(u.starts, in) if u.startErr != nil { return instancectl.ClaimUnitStarted{}, u.startErr } return u.started, nil } func (u *launchUnits) Stop(_ context.Context, host string) error { u.mu.Lock() defer u.mu.Unlock() u.stops = append(u.stops, host) return u.stopErr } func (u *launchUnits) Gone(context.Context, string, string) error { u.mu.Lock() defer u.mu.Unlock() if u.goneAfterStartErr != nil && len(u.starts) > 0 { return u.goneAfterStartErr } return u.goneErr } func (u *launchUnits) counts() (int, int) { u.mu.Lock() defer u.mu.Unlock() return len(u.starts), len(u.stops) } type launchKernel struct { path string want hostbox.LaunchFile verifyErr error observed ClaimLaunchObserved observeErr error observe func(instancectl.ClaimUnitStarted) (ClaimLaunchObserved, error) digest string } func (k *launchKernel) Verify(path string, want hostbox.LaunchFile) error { if k.verifyErr != nil { return k.verifyErr } if path != k.path || want != k.want { return hostbox.ErrChanged } return nil } func (k *launchKernel) Observe(instancectl.ClaimUnitStarted) (ClaimLaunchObserved, error) { if k.observe != nil { return k.observe(instancectl.ClaimUnitStarted{}) } return k.observed, k.observeErr } type launchFixture struct { root *ClaimLaunchRoot claims *launchClaims units *launchUnits kernel *launchKernel peer StopPeer req hostbox.RoleLaunchRequest journal *hostbox.LaunchJournal target hostguard.FenceTarget actor *hostbox.Actor } func newLaunchFixture(t *testing.T, op hostclaim.Op) *launchFixture { return newLaunchFixtureHost(t, op, "shop.example") } func newLaunchFixtureHost(t *testing.T, op hostclaim.Op, host string) *launchFixture { t.Helper() boot, err := execlease.BootID() if err != nil { t.Skip("root launch requires Linux") } start, err := claimConstructionProcessStart() if err != nil { t.Fatal(err) } data := t.TempDir() if err := os.Mkdir(filepath.Join(data, "hostguard"), 0o770); err != nil { t.Fatal(err) } claim := &hostclaim.Claim{V: 2, Host: host, Gen: 3, Holder: &hostclaim.Holder{Incarnation: "inc-1", Stamp: 11, Session: 13, WriterBox: "box-1"}, Op: op} claims := &launchClaims{obs: hostclaim.Observed{Host: host, Claim: claim, ETag: "etag-1"}} guard := hostguard.Guard{DataDir: data, LockDir: t.TempDir(), Claims: claims} if err := guard.Activate(t.Context(), host, "box-1", nil); err != nil { t.Fatal(err) } journal, err := hostbox.OpenLaunchJournal(filepath.Join(t.TempDir(), "launch.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { journal.Close() }) previousAgentRoot := claimContainmentAgentRoot claimContainmentAgentRoot = func(context.Context) (string, error) { return "/sys/fs/cgroup/system.slice/carlos-edge.service", nil } t.Cleanup(func() { claimContainmentAgentRoot = previousAgentRoot }) target := hostguard.FenceTarget{Region: "eu", Host: host, Incarnation: "inc-1", Stamp: 11, Session: 13, Op: op.ID, Phase: op.Phase, Box: "box-1", Enrollment: "enrollment-1", BootID: boot, ControllerID: "controller-1", ClaimETag: "etag-1", DirectoryRevision: "7", RosterRevision: 7} if err = journal.Prelaunch(t.Context(), hostbox.LaunchInventory{ControllerPID: os.Getpid(), ProcessStart: start, Target: target, Groups: []hostbox.LaunchGroup{{RelativePath: filepath.Join("system.slice/carlos-edge.service", cgroup.ExecName(claimContainmentName(target))), Device: 1, Inode: 9}}, NoState: true}); err != nil { t.Fatal(err) } actor := &hostbox.Actor{Box: "box-1", Enrollment: "enrollment-1", Revision: 7, Binding: "7", Region: "eu", Eligible: true} exe := hostbox.LaunchFile{Device: 5, Inode: 6} units := &launchUnits{started: instancectl.ClaimUnitStarted{Unit: instancectl.UnitName(instancectl.RoleRun, host), InvocationID: "unit-inv-1", ControlGroup: instancectl.ClaimUnitControlGroup(host), MainPID: 4242}} kernel := &launchKernel{path: "/opt/carlos/versions/v1/app", want: exe, observed: ClaimLaunchObserved{ProcessStart: "ps-4242", TenantPID: 4243, TenantStart: "ps-4243", Group: hostbox.LaunchFile{Device: 3, Inode: 4}, Executable: exe}} peer := StopPeer{UID: 1001, PID: os.Getpid(), ProcessStart: start, BootID: boot, ControllerID: "controller-1"} root := &ClaimLaunchRoot{ValidateExecutable: func(string, string, string) error { return nil }, Guard: guard, Journal: journal, Units: units, Kernel: kernel, Actor: func(context.Context) (hostbox.Actor, error) { return *actor, nil }} reservation := hostbox.LaunchReservation{ID: "attempt-1:main", Role: "main", InvocationID: "inv-1", Inputs: "inputs-1", Target: target, Group: hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup(host)}, Executable: exe} req := hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStart, Reservation: &reservation, Executable: kernel.path, MemoryMax: cgroup.TierDefault.MemoryMax} f := &launchFixture{root: root, claims: claims, units: units, kernel: kernel, peer: peer, req: req, journal: journal, target: target, actor: actor} configureRev2Launch(t, f) return f } func servingOp() hostclaim.Op { return hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box-1"} } func (f *launchFixture) noReservation(t *testing.T, name string) { t.Helper() if starts, _ := f.units.counts(); starts != 0 { t.Fatalf("%s reached the unit start", name) } if _, err := f.journal.UnitReservation(t.Context(), f.target, f.req.Reservation.ID); err == nil { t.Fatalf("%s journaled a reservation", name) } } func TestClaimLaunchRootStartsExactlyTheReservedRole(t *testing.T) { f := newLaunchFixture(t, servingOp()) resp, err := f.root.Serve(t.Context(), f.peer, f.req) if err != nil { t.Fatal(err) } group := hostbox.LaunchGroup{RelativePath: instancectl.ClaimUnitControlGroup(f.target.Host), Device: 3, Inode: 4, InvocationID: "unit-inv-1"} if resp.V != hostbox.RoleLaunchV || resp.Protocol != hostbox.RoleLaunchProtocol || resp.Grant == nil || resp.Grant.Check(*f.req.Reservation) != nil || resp.Grant.Group != group || resp.Grant.PID != 4242 || resp.Grant.ProcessStart != "ps-4242" { t.Fatalf("grant=%+v", resp) } if len(f.units.starts) != 1 || f.units.starts[0] != (instancectl.ClaimUnit{Host: f.target.Host, Executable: f.kernel.path, MemoryMax: cgroup.TierDefault.MemoryMax}) { t.Fatalf("unit starts=%+v", f.units.starts) } got, err := f.journal.UnitReservation(t.Context(), f.target, f.req.Reservation.ID) if err != nil || got.State != hostbox.UnitStarted || got.Group != group || got.PID != 4242 || got.ProcessStart != "ps-4242" || got.Executable != f.req.Reservation.Executable { t.Fatalf("journal=%+v %v", got, err) } // A replay of the same request, after success, is refused without effect. if _, err := f.root.Serve(t.Context(), f.peer, f.req); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("replay: %v", err) } if starts, _ := f.units.counts(); starts != 1 { t.Fatal("replay started a second unit") } } func TestClaimLaunchRootSerializesConcurrentDuplicates(t *testing.T) { f := newLaunchFixture(t, servingOp()) var wg sync.WaitGroup errs := make(chan error, 4) for range 4 { wg.Add(1) go func() { defer wg.Done(); _, err := f.root.Serve(t.Context(), f.peer, f.req); errs <- err }() } wg.Wait() close(errs) ok := 0 for err := range errs { if err == nil { ok++ } } if starts, _ := f.units.counts(); ok != 1 || starts != 1 { t.Fatalf("successes=%d starts=%d", ok, starts) } } func TestClaimLaunchRootTrustsOnlyItsOwnProviders(t *testing.T) { for name, mutate := range map[string]func(*launchFixture){ "other controller": func(f *launchFixture) { f.peer.ControllerID = "controller-2" }, "other boot": func(f *launchFixture) { f.peer.BootID = "boot-2" }, "root uid": func(f *launchFixture) { f.peer.UID = 0 }, "not the record's pid": func(f *launchFixture) { f.peer.PID = 43 }, "restarted controller": func(f *launchFixture) { f.peer.ProcessStart = "start-43" }, "roster box": func(f *launchFixture) { f.actor.Box = "box-2" }, "roster enrollment": func(f *launchFixture) { f.actor.Enrollment = "enrollment-2" }, "box binding": func(f *launchFixture) { f.actor.Binding = "8" }, "roster region": func(f *launchFixture) { f.actor.Region = "us" }, "quarantined box": func(f *launchFixture) { f.actor.Quarantined = true }, "holder session": func(f *launchFixture) { f.claims.set(func(c *hostclaim.Claim) { c.Holder.Session = 14 }) }, "holder stamp": func(f *launchFixture) { f.claims.set(func(c *hostclaim.Claim) { c.Holder.Stamp = 12 }) }, "executor elsewhere": func(f *launchFixture) { f.claims.set(func(c *hostclaim.Claim) { c.Op.Exec = "box-2" }) }, "writer elsewhere": func(f *launchFixture) { f.claims.set(func(c *hostclaim.Claim) { c.Holder.WriterBox = "box-2" }) }, "box declared lost": func(f *launchFixture) { f.claims.set(func(c *hostclaim.Claim) { c.Lost = []string{"box-1"} }) }, "operation began": func(f *launchFixture) { f.claims.set(func(c *hostclaim.Claim) { c.Op.ID, c.Op.Kind = "op-9", hostclaim.KindParking }) }, "build not verified": func(f *launchFixture) { f.kernel.verifyErr = hostbox.ErrChanged }, "build path differs": func(f *launchFixture) { f.req.Executable = "/opt/carlos/versions/v2/app" }, "tier not a platform": func(f *launchFixture) { f.req.MemoryMax = 12345 }, "reservation target op": func(f *launchFixture) { r := *f.req.Reservation; r.Target.Op = "op-1"; f.req.Reservation = &r }, } { f := newLaunchFixture(t, servingOp()) mutate(f) if _, err := f.root.Serve(t.Context(), f.peer, f.req); err == nil { t.Fatalf("%s launched", name) } f.noReservation(t, name) } } func TestClaimLaunchRootRefusesWithoutThePrelaunchRecord(t *testing.T) { f := newLaunchFixture(t, servingOp()) r := *f.req.Reservation r.Target.Session = 14 f.claims.set(func(c *hostclaim.Claim) { c.Holder.Session = 14 }) f.req.Reservation = &r if _, err := f.root.Serve(t.Context(), f.peer, f.req); err == nil { t.Fatal("launched a session with no prelaunch record") } if starts, _ := f.units.counts(); starts != 0 { t.Fatal("unit started") } } func TestClaimLaunchRootProbeIsBoundToTheOperationAndInstallStamp(t *testing.T) { probing := hostclaim.Op{Kind: hostclaim.KindLanding, ID: "op-1", Phase: hostclaim.PhaseLandingProbing, Exec: "box-1", Journal: map[string]string{"install_stamp": "11"}} f := newLaunchFixture(t, probing) if _, err := f.root.Serve(t.Context(), f.peer, f.req); err != nil { t.Fatalf("probe: %v", err) } for name, mutate := range map[string]func(*hostclaim.Claim){ "install stamp": func(c *hostclaim.Claim) { c.Op.Journal = map[string]string{"install_stamp": "12"} }, "probe passed": func(c *hostclaim.Claim) { c.Op.Phase = hostclaim.PhaseLandingProbed }, "other operation": func(c *hostclaim.Claim) { c.Op.ID = "op-2" }, "relocation no pin": func(c *hostclaim.Claim) { c.Op.Kind, c.Op.Journal = hostclaim.KindRelocating, nil }, } { f := newLaunchFixture(t, probing) f.claims.set(mutate) if _, err := f.root.Serve(t.Context(), f.peer, f.req); err == nil { t.Fatalf("%s launched", name) } f.noReservation(t, name) } } func TestClaimLaunchRootRefusesAFencedSession(t *testing.T) { probing := hostclaim.Op{Kind: hostclaim.KindLanding, ID: "op-1", Phase: hostclaim.PhaseLandingProbing, Exec: "box-1"} f := newLaunchFixture(t, probing) fence := f.target fence.FenceID, fence.ObligationID = "fence-1", "obligation-1" if _, err := f.root.Guard.BeginFence(t.Context(), fence); err != nil { t.Fatal(err) } if _, err := f.root.Serve(t.Context(), f.peer, f.req); !errors.Is(err, hostguard.ErrFenced) { t.Fatalf("fenced session launched: %v", err) } f.noReservation(t, "fenced session") } func TestClaimLaunchRootCleansUpAStartItCannotVouchFor(t *testing.T) { f := newLaunchFixture(t, servingOp()) f.kernel.observed.Executable = hostbox.LaunchFile{Device: 5, Inode: 7} if _, err := f.root.Serve(t.Context(), f.peer, f.req); err == nil { t.Fatal("swapped build accepted") } if _, stops := f.units.counts(); stops != 1 { t.Fatal("swapped build left running") } got, err := f.journal.UnitReservation(t.Context(), f.target, f.req.Reservation.ID) if err != nil || got.State != hostbox.UnitSettled { t.Fatalf("journal=%+v %v", got, err) } if _, err := f.root.Serve(t.Context(), f.peer, f.req); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("spent reservation retried: %v", err) } // A failed start whose unit root cannot prove gone stays pending, so the // stop journal keeps refusing a receipt. f = newLaunchFixture(t, servingOp()) f.units.startErr = errors.New("start failed") f.units.goneAfterStartErr = errors.New("still active") if _, err := f.root.Serve(t.Context(), f.peer, f.req); err == nil { t.Fatal("failed start accepted") } got, err = f.journal.UnitReservation(t.Context(), f.target, f.req.Reservation.ID) if err != nil || got.State != hostbox.UnitIntended { t.Fatalf("unproven unit settled: %+v %v", got, err) } sj, err := f.journal.ReadStopKernelJournal(t.Context(), f.target) if err != nil || sj.Settled { t.Fatalf("stop journal settled=%v %v", sj.Settled, err) } } func TestClaimLaunchRootStopSettlesAndBurnsReservations(t *testing.T) { f := newLaunchFixture(t, servingOp()) if _, err := f.root.Serve(t.Context(), f.peer, f.req); err != nil { t.Fatal(err) } stop := hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchStop, Reservation: f.req.Reservation} other := f.peer other.ControllerID = "controller-2" if _, err := f.root.Serve(t.Context(), other, stop); err == nil { t.Fatal("another controller stopped the role") } f.units.goneErr = errors.New("still active") if _, err := f.root.Serve(t.Context(), f.peer, stop); err == nil { t.Fatal("stop reported success while the unit stayed") } f.units.goneErr = nil if _, err := f.root.Serve(t.Context(), f.peer, stop); err != nil { t.Fatal(err) } got, err := f.journal.UnitReservation(t.Context(), f.target, f.req.Reservation.ID) if err != nil || got.State != hostbox.UnitSettled { t.Fatalf("journal=%+v %v", got, err) } _, before := f.units.counts() if _, err := f.root.Serve(t.Context(), f.peer, stop); err != nil { t.Fatalf("idempotent stop: %v", err) } if _, after := f.units.counts(); after != before { t.Fatal("settled stop touched the unit again") } // A stop for a reservation root never saw burns it. g := newLaunchFixture(t, servingOp()) stop.Reservation = g.req.Reservation if _, err := g.root.Serve(t.Context(), g.peer, stop); err != nil { t.Fatal(err) } if _, err := g.root.Serve(t.Context(), g.peer, g.req); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("start after stop: %v", err) } if starts, _ := g.units.counts(); starts != 0 { t.Fatal("burnt reservation started") } } func TestClaimLaunchRootCapabilityNeedsEveryProvider(t *testing.T) { f := newLaunchFixture(t, servingOp()) q := hostbox.RoleLaunchRequest{V: hostbox.RoleLaunchV, Op: hostbox.RoleLaunchCapability} resp, err := f.root.Serve(t.Context(), f.peer, q) if err != nil || resp.Protocol != hostbox.RoleLaunchProtocol || resp.Grant != nil { t.Fatalf("capability=%+v %v", resp, err) } for name, strip := range map[string]func(*ClaimLaunchRoot){ "local": func(r *ClaimLaunchRoot) { r.Local = nil }, "clock": func(r *ClaimLaunchRoot) { r.Clock = nil }, "data root": func(r *ClaimLaunchRoot) { r.DataRoot = "" }, "units": func(r *ClaimLaunchRoot) { r.Units = nil }, "kernel": func(r *ClaimLaunchRoot) { r.Kernel = nil }, "journal": func(r *ClaimLaunchRoot) { r.Journal = nil }, "actor": func(r *ClaimLaunchRoot) { r.Actor = nil }, "guard": func(r *ClaimLaunchRoot) { r.Guard.Claims = nil }, } { g := newLaunchFixture(t, servingOp()) strip(g.root) if _, err := g.root.Serve(t.Context(), g.peer, q); !errors.Is(err, activator.ErrClaimIsolationUnavailable) { t.Fatalf("%s missing: %v", name, err) } if _, err := g.root.Serve(t.Context(), g.peer, g.req); !errors.Is(err, activator.ErrClaimIsolationUnavailable) { t.Fatalf("%s missing start: %v", name, err) } } } func TestClaimUnitStopKernelRequiresTheUnitGone(t *testing.T) { f := newLaunchFixture(t, servingOp()) if _, err := f.root.Serve(t.Context(), f.peer, f.req); err != nil { t.Fatal(err) } inner := 0 k := claimUnitStopKernel{Journal: f.journal, Units: f.units, Inner: stopKernelFunc(func(_ context.Context, target hostguard.FenceTarget, _ StopInventory, _ DrainAcknowledgement) (StopKernelEvidence, error) { inner++ j, err := f.journal.ReadStopKernelJournal(t.Context(), target) if err != nil || !j.Settled { t.Fatalf("inner verifier saw an unsettled unit: %+v %v", j, err) } return StopKernelEvidence{CgroupIdentities: []string{"cg"}}, nil })} f.units.goneErr = errors.New("still active") if _, err := k.VerifyStopped(t.Context(), f.target, StopInventory{}, DrainAcknowledgement{}); err == nil || inner != 0 { t.Fatalf("running unit verified: %v inner=%d", err, inner) } f.units.goneErr = nil ev, err := k.VerifyStopped(t.Context(), f.target, StopInventory{}, DrainAcknowledgement{}) if err != nil || inner != 1 || len(ev.UnitInvocationIDs) != 1 || ev.UnitInvocationIDs[0] != "unit-inv-1" { t.Fatalf("evidence=%+v %v inner=%d", ev, err, inner) } } func TestClaimUnitStopKernelGuardsRecovery(t *testing.T) { f := newLaunchFixture(t, servingOp()) if _, err := f.root.Serve(t.Context(), f.peer, f.req); err != nil { t.Fatal(err) } called := 0 k := claimUnitStopKernel{Journal: f.journal, Units: f.units, Inner: recoveryKernelFunc(func(context.Context, hostguard.FenceTarget, StopInventory) (StopKernelEvidence, DrainAcknowledgement, error) { called++ return StopKernelEvidence{}, DrainAcknowledgement{}, nil })} f.units.goneErr = errors.New("still active") if _, _, err := k.RecoverStopped(t.Context(), f.target, StopInventory{}); err == nil || called != 0 { t.Fatalf("recovery settled a running unit: %v called=%d", err, called) } f.units.goneErr = nil ev, _, err := k.RecoverStopped(t.Context(), f.target, StopInventory{}) if err != nil || called != 1 || len(ev.UnitInvocationIDs) != 1 { t.Fatalf("recovery=%+v %v called=%d", ev, err, called) } if _, ok := StopKernelVerifier(claimUnitStopKernel{Inner: stopKernelFunc(nil)}).(stopRecoveryKernel); !ok { t.Fatal("wrapper hides recovery") } if _, _, err := (claimUnitStopKernel{Journal: f.journal, Units: f.units, Inner: stopKernelFunc(nil)}).RecoverStopped(t.Context(), f.target, StopInventory{}); !errors.Is(err, ErrClaimStopUnavailable) { t.Fatalf("recovery without an inner recovery kernel: %v", err) } } type recoveryKernelFunc func(context.Context, hostguard.FenceTarget, StopInventory) (StopKernelEvidence, DrainAcknowledgement, error) func (f recoveryKernelFunc) VerifyStopped(context.Context, hostguard.FenceTarget, StopInventory, DrainAcknowledgement) (StopKernelEvidence, error) { return StopKernelEvidence{}, ErrClaimStopUnavailable } func (f recoveryKernelFunc) RecoverStopped(ctx context.Context, t hostguard.FenceTarget, in StopInventory) (StopKernelEvidence, DrainAcknowledgement, error) { return f(ctx, t, in) } var _ = strconv.Itoa func (u *launchUnits) StartChecked(ctx context.Context, in instancectl.ClaimUnit, _ instancectl.Launch, check func() error) (instancectl.ClaimUnitStarted, error) { if u.beforeCheck != nil { u.beforeCheck() } if err := check(); err != nil { return instancectl.ClaimUnitStarted{}, err } return u.Start(ctx, in) } func (u *launchUnits) Same(context.Context, instancectl.ClaimUnitStarted) error { return u.sameErr } func (k *launchKernel) VerifyDigest(_ context.Context, path string, want hostbox.LaunchFile, digest string) error { if digest != k.digest { return hostbox.ErrChanged } return k.Verify(path, want) } func (u *launchUnits) StartReserved(ctx context.Context, in instancectl.ClaimUnit, l instancectl.Launch, p instancectl.ClaimExecPermit, check func() error) (instancectl.ClaimUnitStarted, error) { if !p.Valid() { return instancectl.ClaimUnitStarted{}, instancectl.ErrClaimUnit } return u.StartChecked(ctx, in, l, check) }