package activator import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "os" "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" "github.com/carlosframework/platform/internal/cgroup" "github.com/carlosframework/platform/internal/registry" ) // fakeSwapBackend is fakeBackend plus the swap seam (SwapBackend). Every call is recorded in // the shared order log: raise:+, startGen:, // switch:, restore:, // startSidecarGen:. Generation children are fakeProcs labelled // gen:. type fakeSwapBackend struct { *fakeBackend gmu sync.Mutex genCalls int gens []Gen genProcs []*fakeProc genErr error genGate chan struct{} // when set, StartGen blocks until it closes or ctx ends // genHold, when set, makes StartStaged wait for it to close IGNORING ctx, // then return a live successor even though the attempt was cancelled: a // spawn that outlives cancellation, which the barrier must wait for. genHold chan struct{} raises int restores int // genSocketDir, when set, puts generation sockets there (Task 12), and // genServe binds a real server on each one, stopped with the process. genSocketDir string genServe func(g Gen) (stop func()) // Task 7: legacy sidecars started from a generation. sidecarGens []Gen sidecarGenGate chan struct{} // sidecarStopGate, when set, is installed on every sidecar started from // then on: its Stop blocks until the gate closes. sidecarStopGate chan struct{} // sidecarDescendant: a process a sidecar forked is still in its cgroup, // so KillSidecar (Task 7) cannot prove it empty. sidecarDescendant bool // sidecarKills counts KillSidecar calls: a test waits on it to know a // proof was attempted before it asserts nothing was released. sidecarKills int // tiers is each build's own tier (GenTier); a build not here is the // default tier. tiers map[string]cgroup.Limit // onRaise, when set, runs inside RaiseGenParent: a test's window between // an attempt's admission and its exec. onRaise func() // genStuck makes every generation spawned from then on a stuck // fakeProc: its Stop answers ErrStillRunning until exit(). genStuck bool // raiseErr is what RaiseGenParent answers after recording the call: a // raise whose first write landed and a later one failed. raiseErr error // restoreFails is how many more RestoreGenParent calls fail. restoreFails int // restoreGate, when set, holds every RestoreGenParent call before its // write until it closes; restoreEntered, when set, is sent to // (non-blocking) as each call reaches the gate. A held call is counted // and recorded only once it writes. restoreGate chan struct{} restoreEntered chan struct{} } func (g *fakeSwapBackend) StartStaged(ctx context.Context, _ registry.Route, gen Gen) (Gen, Proc, error) { g.record("startGen:" + gen.StagedPath) g.gmu.Lock() g.genCalls++ gate, err := g.genGate, g.genErr g.gmu.Unlock() if gate != nil { select { case <-gate: case <-ctx.Done(): return gen, nil, ctx.Err() } } if err != nil { return gen, nil, err } // The caller's exec gate at the launch point (final review F1), after // the preparation genGate stands for, as the real backend asks it. if err := runExecGate(ctx); err != nil { return gen, nil, err } g.gmu.Lock() hold := g.genHold g.gmu.Unlock() if hold != nil { <-hold } gen.Socket = fmt.Sprintf("/run/carlos/g/test.%d.sock", gen.N) if g.genSocketDir != "" { gen.Socket = filepath.Join(g.genSocketDir, fmt.Sprintf("g%d.sock", gen.N)) } gen.Control = gen.Socket + ".control" g.gmu.Lock() stuck := g.genStuck g.gmu.Unlock() p := &fakeProc{label: fmt.Sprintf("gen:%d", gen.N), rec: g.fakeBackend, stuck: stuck} var proc Proc = p if g.genServe != nil { proc = &genServingProc{fakeProc: p, stop: g.genServe(gen)} } g.gmu.Lock() g.gens = append(g.gens, gen) g.genProcs = append(g.genProcs, p) g.gmu.Unlock() return gen, proc, nil } // GenTier answers from tiers (by build), and the default tier otherwise. func (g *fakeSwapBackend) GenTier(_ registry.Route, build string) cgroup.Limit { g.gmu.Lock() defer g.gmu.Unlock() if t, ok := g.tiers[build]; ok { return t } return cgroup.TierDefault } func (g *fakeSwapBackend) GenReady(registry.Route, Gen, Proc) {} func (g *fakeSwapBackend) RaiseGenParent(_ context.Context, _ registry.Route, inc, succ cgroup.Limit) error { g.record(fmt.Sprintf("raise:%d+%d", inc.MemoryMax, succ.MemoryMax)) g.gmu.Lock() g.raises++ hook, err := g.onRaise, g.raiseErr g.gmu.Unlock() if hook != nil { hook() } return err } func (g *fakeSwapBackend) RestoreGenParent(_ context.Context, _ registry.Route, tier cgroup.Limit) error { g.gmu.Lock() gate, entered := g.restoreGate, g.restoreEntered g.gmu.Unlock() if gate != nil { if entered != nil { select { case entered <- struct{}{}: default: } } <-gate } g.record(fmt.Sprintf("restore:%d", tier.MemoryMax)) g.gmu.Lock() defer g.gmu.Unlock() g.restores++ if g.restoreFails > 0 { g.restoreFails-- return errors.New("cgroup: parent memory.max: device or resource busy") } return nil } func (g *fakeSwapBackend) GenSwitched(_ registry.Route, gen Gen, _ Proc) error { g.record(fmt.Sprintf("switch:%d", gen.N)) return nil } func (g *fakeSwapBackend) calls() (gens, raises, restores int) { g.gmu.Lock() defer g.gmu.Unlock() return g.genCalls, g.raises, g.restores } func (g *fakeSwapBackend) lastGen() (Gen, *fakeProc) { g.gmu.Lock() defer g.gmu.Unlock() if len(g.gens) == 0 { return Gen{}, nil } return g.gens[len(g.gens)-1], g.genProcs[len(g.genProcs)-1] } // genServingProc is a generation child with a real socket server behind it. // Plan 1 Task 4 already declares a servingProc test type in this package // (canonlink_test.go, wrapping a Proc and an *http.Server), so this one has // its own name. type genServingProc struct { *fakeProc stop func() } func (p *genServingProc) Stop(ctx context.Context) error { p.stop() return p.fakeProc.Stop(ctx) } // swapFixture is an awake host on a swap-enabled Service whose backend runs // generations. The incumbent is generation 1<<40 on build v1, a number // nextGen will never hand out in a test. Build v2 is staged on disk with // its real digest. type swapFixture struct { a *Service b *fakeSwapBackend clk *fakeClock reg *registry.Registry host string incumbent Gen staged string digest string rev atomic.Uint64 } func newSwapFixture(t *testing.T) *swapFixture { t.Helper() return newSwapFixtureOn(t, nil) } // newSwapFixtureOn is newSwapFixture with the Service's backend made by // wrap from the swap fake, for a test that needs one more capability. func newSwapFixtureOn(t *testing.T, wrap func(*fakeSwapBackend) Backend) *swapFixture { t.Helper() b := &fakeSwapBackend{fakeBackend: &fakeBackend{}} clk := &fakeClock{t: time.Unix(1_700_000_000, 0)} reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) if err != nil { t.Fatal(err) } var be Backend = b if wrap != nil { be = wrap(b) } a := New(Config{Bucket: "test-bucket", Idle: time.Hour}, be, reg) a.gens = true // plan 1's switch, as plan 5 sets it from CARLOS_SWAP a.now = clk.now a.genHealth = func(context.Context, Gen) error { return nil } f := &swapFixture{a: a, b: b, clk: clk, reg: reg, host: "swap.example.com"} // The route's socket and database live in the test's own directory, // never the shared /tmp. rt := testRoute(f.host) rdir := t.TempDir() rt.Socket, rt.DBPath = filepath.Join(rdir, "app.sock"), filepath.Join(rdir, "app.db") // Generation sockets too: a hold's pinned wake runs wake()'s awake // transition, which points the canonical path at the generation's socket // for real (swapCanonicalLink). b.genSocketDir = t.TempDir() arrangeAwake(t, a, rt) f.incumbent = Gen{N: 1 << 40, Socket: "/run/carlos/g/incumbent.sock", Control: "/run/carlos/g/incumbent.sock.control", Build: "v1", StagedPath: "/opt/carlos/versions/v1/app"} setCurrentGen(t, a, f.host, f.incumbent) dir := filepath.Join(t.TempDir(), "versions", "v2") if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatal(err) } f.staged = filepath.Join(dir, "app") if err := os.WriteFile(f.staged, []byte("build v2"), 0o755); err != nil { t.Fatal(err) } sum := sha256.Sum256([]byte("build v2")) f.digest = hex.EncodeToString(sum[:]) f.rev.Store(1) return f } func (f *swapFixture) attempt(mode SwapMode) Attempt { return Attempt{Host: f.host, Cause: CauseRelease, Build: "v2", StagedPath: f.staged, Digest: f.digest, Mode: mode, SchedRev: f.rev.Load()} } func (f *swapFixture) newest(string) uint64 { return f.rev.Load() } func (f *swapFixture) appSocket(t *testing.T) string { t.Helper() sock, ok := f.a.AppSocket(f.host) if !ok { t.Fatalf("AppSocket(%s) reported no current generation", f.host) } return sock } // setCurrentGen stands in for plan 1's wake, which records the generation // it spawned in e.gen (contract C2, C3). func setCurrentGen(t *testing.T, a *Service, host string, g Gen) { t.Helper() a.mu.Lock() defer a.mu.Unlock() e := a.entries[host] if e == nil || e.state != actAwake { t.Fatalf("%s is not awake", host) } e.gen, e.spawnVersion = g, g.Build } // genStateForTest is the entry's current generation and its operation set // by role. func (a *Service) genStateForTest(host string) (Gen, map[uint64]genRole) { a.mu.Lock() defer a.mu.Unlock() e := a.entries[host] if e == nil { return Gen{}, nil } out := make(map[uint64]genRole, len(e.ops.members)) for n, m := range e.ops.members { out[n] = m.role } return e.gen, out } // indexPrefix is slices.Index for the first event with prefix p. func indexPrefix(ev []string, p string) int { for i, e := range ev { if strings.HasPrefix(e, p) { return i } } return -1 } func (g *fakeSwapBackend) StartSidecarGen(_ context.Context, _ registry.Route, gen Gen) (Proc, error) { g.record("startSidecarGen:" + gen.StagedPath) g.gmu.Lock() g.sidecarGens = append(g.sidecarGens, gen) gate := g.sidecarGenGate g.gmu.Unlock() if gate != nil { <-gate } g.gmu.Lock() stopGate := g.sidecarStopGate g.gmu.Unlock() p := &fakeProc{label: "sidecar", rec: g.fakeBackend, stopGate: stopGate} g.mu.Lock() g.sidecars = append(g.sidecars, p) g.mu.Unlock() return p, nil } // KillSidecar is the sidecar cgroup's proof: it fails while a descendant // is "left" (setSidecarDescendant). func (g *fakeSwapBackend) KillSidecar(context.Context, registry.Route) error { g.gmu.Lock() defer g.gmu.Unlock() g.sidecarKills++ if g.sidecarDescendant { return errors.New("role-sidecar: still populated") } return nil } func (g *fakeSwapBackend) setSidecarDescendant(left bool) { g.gmu.Lock() g.sidecarDescendant = left g.gmu.Unlock() } func (g *fakeSwapBackend) sidecarKillCount() int { g.gmu.Lock() defer g.gmu.Unlock() return g.sidecarKills } func (g *fakeSwapBackend) sidecarGenPaths() []string { g.gmu.Lock() defer g.gmu.Unlock() out := make([]string, len(g.sidecarGens)) for i, s := range g.sidecarGens { out[i] = s.StagedPath } return out }