package activator // Tests for the zero-downtime-deploys rev 19 amendment // (docs/superpowers/specs/2026-10-13-zero-downtime-deploys-rev19-amendment.md): // a generation's socket keeps the flag-off file name in a directory of its // own, a generation launch matches a flag-off launch, the directory goes // once the generation is proven gone, and a flat host too long for a // generation path keeps its canonical socket as a foreign row does. import ( "context" "encoding/json" "errors" "maps" "os" "path/filepath" "slices" "strings" "testing" "time" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" ) // goneProc is a process whose leader has exited. type goneProc struct{} func (goneProc) Stop(context.Context) error { return nil } func (goneProc) Exited() bool { return true } // envMap is an environment list ("K=V") as a map. func envMap(t *testing.T, env []string) map[string]string { t.Helper() m := make(map[string]string, len(env)) for _, kv := range env { k, v, ok := strings.Cut(kv, "=") if !ok { continue // a multi-line value's continuation in `env` output } m[k] = v } return m } // sameLaunch checks rule 2 for one layout: the two argvs differ only in // --socket's directory, and the two environments only in // CARLOS_CONTROL_SOCKET, which names each launch's own socket + ".control". func sameLaunch(t *testing.T, offArgv, genArgv []string, offEnv, genEnv map[string]string, offSock, genSock string) { t.Helper() i := slices.Index(offArgv, "--socket") j := slices.Index(genArgv, "--socket") if i < 0 || j != i || i+1 >= len(offArgv) || len(genArgv) != len(offArgv) { t.Fatalf("argv shapes differ: flag off %v, generation %v", offArgv, genArgv) } if offArgv[i+1] != offSock || genArgv[i+1] != genSock { t.Fatalf("--socket: flag off %q (want %q), generation %q (want %q)", offArgv[i+1], offSock, genArgv[i+1], genSock) } if offSock == genSock { t.Fatalf("the generation launch bound the canonical socket %s", offSock) } if a, b := filepath.Base(offArgv[i+1]), filepath.Base(genArgv[i+1]); a != b { t.Fatalf("--socket file name: flag off %q, generation %q; rule 2 wants the same", a, b) } rest := func(argv []string) []string { return slices.Delete(slices.Clone(argv), i+1, i+2) } if !slices.Equal(rest(offArgv), rest(genArgv)) { t.Fatalf("argv differs beyond --socket: flag off %v, generation %v", offArgv, genArgv) } if offEnv["CARLOS_CONTROL_SOCKET"] != offSock+".control" || genEnv["CARLOS_CONTROL_SOCKET"] != genSock+".control" { t.Fatalf("CARLOS_CONTROL_SOCKET: flag off %q, generation %q", offEnv["CARLOS_CONTROL_SOCKET"], genEnv["CARLOS_CONTROL_SOCKET"]) } off, gen := maps.Clone(offEnv), maps.Clone(genEnv) delete(off, "CARLOS_CONTROL_SOCKET") delete(gen, "CARLOS_CONTROL_SOCKET") if !maps.Equal(off, gen) { for k := range off { if gen[k] != off[k] { t.Errorf("env %s: flag off %q, generation %q", k, off[k], gen[k]) } } for k := range gen { if _, ok := off[k]; !ok { t.Errorf("env %s: only the generation launch has it (%q)", k, gen[k]) } } t.FailNow() } if off["CARLOS_HOST"] == "" { t.Fatal("neither launch carries CARLOS_HOST") } } // Rule 2, flat: a generation's exec has the flag-off argv and agent-owned // environment apart from the socket's directory and the control path. func TestFlatGenerationLaunchMatchesTheFlagOffLaunch(t *testing.T) { t.Parallel() requireUnixSockets(t) b := testExecBackend(t) b.RunDirOverride = shortRunDir(t) host := "shop.example.com" argvPath, envPath := stageRecordingLiveFullEnv(t, b, host, "v1") rt := flatGenRoute(b, host) launch := func(n uint64, perGen bool) ([]string, map[string]string, string) { t.Helper() _ = os.Remove(argvPath) _ = os.Remove(envPath) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() var ( proc Proc g Gen ) done := make(chan struct{}) go func() { proc, g, _ = b.StartGen(ctx, rt, n, perGen) // the recorder never binds; a health error is expected close(done) }() argv := strings.Split(strings.TrimSpace(waitForFile(t, argvPath, 3*time.Second)), "\n") env := envMap(t, strings.Split(strings.TrimSpace(waitForFile(t, envPath, 3*time.Second)), "\n")) cancel() // recorded: end the health wait the recorder never answers <-done if proc != nil { _ = proc.Stop(context.Background()) } return argv, env, g.Socket } offArgv, offEnv, offSock := launch(5, false) genArgv, genEnv, genSock := launch(6, true) if want := filepath.Join(b.RunDirOverride, "g", "6", host+".sock"); genSock != want { t.Fatalf("generation socket %s, want %s", genSock, want) } sameLaunch(t, offArgv, genArgv, offEnv, genEnv, rt.Socket, genSock) if offSock != rt.Socket { t.Fatalf("flag-off handle names %s, want the canonical %s", offSock, rt.Socket) } fi, err := os.Stat(filepath.Dir(genSock)) if err != nil || !fi.IsDir() || fi.Mode().Perm() != 0o750 { t.Fatalf("generation directory %s: %v, %v; want a 0750 directory", filepath.Dir(genSock), fi, err) } } // Rule 2, nested: the generation's launch file has the flag-off argv and // agent-owned environment apart from the socket's directory and the control // path, and prepare made the generation's own directory before the start. func TestNestedGenerationLaunchMatchesTheFlagOffLaunch(t *testing.T) { t.Parallel() ctl := &fakeControl{} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) launchFile := filepath.Join(provision.InstanceDir(b.DataDir, rt.Host), "launch.run.json") var ( serveAt string dirAtStart bool stop func() ) ctl.onStart = func() { fi, err := os.Stat(filepath.Dir(serveAt)) dirAtStart = err == nil && fi.IsDir() stop = serveOK(t, serveAt) } launch := func(n uint64, perGen bool, sock, inv string) instancectl.Launch { t.Helper() serveAt, dirAtStart = sock, false ctl.mu.Lock() ctl.inv = inv ctl.mu.Unlock() unit.set("active", inv, true, true) p, g, err := b.StartGen(context.Background(), rt, n, perGen) if err != nil { t.Fatalf("StartGen(%d, %v): %v", n, perGen, err) } if g.Socket != sock { t.Fatalf("handle socket %s, want %s", g.Socket, sock) } if !dirAtStart { t.Fatalf("%s did not exist when root was asked to start the unit", filepath.Dir(sock)) } raw, err := os.ReadFile(launchFile) if err != nil { t.Fatal(err) } var l instancectl.Launch if err := json.Unmarshal(raw, &l); err != nil { t.Fatal(err) } unit.set("inactive", inv, false, false) waitFor(t, "the unit to exit", func() bool { return p.Exited() }) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() _ = p.Stop(ctx) if stop != nil { stop() stop = nil } return l } genSock := filepath.Join(filepath.Dir(rt.Socket), "g9", "app.sock") off := launch(8, false, rt.Socket, "inv1") gen := launch(9, true, genSock, "inv2") sameLaunch(t, off.Argv, gen.Argv, envMap(t, off.Env), envMap(t, gen.Env), rt.Socket, genSock) } // Rule 4: a flat host whose generation path would not fit a unix socket // address is handled as a foreign row is: no generation path, a generation // on its canonical socket, and an overlap that runs as a hold. func TestOverLongFlatHostKeepsItsCanonicalSocket(t *testing.T) { requireUnixSockets(t) b := testExecBackend(t) b.RunDirOverride = shortRunDir(t) host := strings.Repeat("a", 60-len(".example.com")) + ".example.com" // 60 bytes stageServingLive(t, b, host) rt := flatGenRoute(b, host) if l, _ := b.Layout(rt); l != LayoutFlat { t.Fatalf("fixture classifies %q, want flat", l) } if l := b.GenLayout(rt); l != LayoutForeign { t.Fatalf("GenLayout = %s, want foreign", l) } split := &SplitBackend{Exec: b} if l := split.GenLayout(rt); l != LayoutForeign { t.Fatalf("SplitBackend.GenLayout = %s, want foreign", l) } if g, ok := b.GenPaths(rt, 9); ok { t.Fatalf("GenPaths = %+v, true; want no generation path", g) } a := &Service{backend: b, entries: map[string]*actEntry{host: {route: rt}}} if m := a.EffectiveMode(host, ModeOverlap); m != ModeHold { t.Fatalf("EffectiveMode(overlap) = %s, want hold, as for a foreign row", m) } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() proc, g, err := b.StartGen(ctx, rt, 9, true) if err != nil { t.Fatalf("StartGen: %v", err) } defer proc.Stop(context.Background()) if g.Socket != rt.Socket || g.Control != rt.Socket+".control" { t.Fatalf("handle = %+v, want the canonical socket", g) } if !isSocket(rt.Socket) { t.Fatalf("%s is not a bound socket", rt.Socket) } if _, err := os.Stat(filepath.Join(b.RunDirOverride, "g")); !os.IsNotExist(err) { t.Fatalf("a generation directory was made for a host with no generation path (stat: %v)", err) } // The same host one byte shorter has generation paths. short := host[1:] if l := b.GenLayout(flatGenRoute(b, short)); l != LayoutFlat { t.Fatalf("GenLayout(%d-byte host) = %s, want flat", len(short), l) } if _, ok := b.GenPaths(flatGenRoute(b, short), 9); !ok { t.Fatalf("GenPaths(%d-byte host) has no generation path", len(short)) } } // Rule 3: once a generation's termination is proven, its own empty // directory goes. A directory its app left a socket in stays, a sibling's // directory is never touched (empty or not), and a generation on the // canonical socket touches nothing. func TestRetiredGenerationDirectoryIsRemovedAndASiblingsIsNot(t *testing.T) { root := t.TempDir() flat := registry.Route{Host: "shop.example.com", Socket: filepath.Join(root, "shop.example.com.sock")} nestedRun := filepath.Join(root, "instances", "shop.example.com") nested := registry.Route{Host: "shop.example.com", Socket: filepath.Join(nestedRun, "app.sock")} mk := func(dir string, files ...string) { t.Helper() if err := os.MkdirAll(dir, 0o750); err != nil { t.Fatal(err) } for _, f := range files { if err := os.WriteFile(filepath.Join(dir, f), nil, 0o600); err != nil { t.Fatal(err) } } } exists := func(p string) bool { _, err := os.Lstat(p); return err == nil } a := &Service{} for _, tc := range []struct { name string rt registry.Route own string // generation 9's directory siblings []string }{ {"flat", flat, filepath.Join(root, "g", "9"), []string{filepath.Join(root, "g", "8"), filepath.Join(root, "g", "10")}}, {"nested", nested, filepath.Join(nestedRun, "g9"), []string{filepath.Join(nestedRun, "g8"), filepath.Join(nestedRun, "g10")}}, } { t.Run(tc.name, func(t *testing.T) { sock := filepath.Join(tc.own, filepath.Base(tc.rt.Socket)) g := Gen{N: 9, Socket: sock, Control: sock + ".control"} mk(tc.own) mk(tc.siblings[0], filepath.Base(tc.rt.Socket)) // a running sibling's socket mk(tc.siblings[1]) // an empty sibling mk(filepath.Dir(tc.rt.Socket), filepath.Base(tc.rt.Socket)) if a.genTerminated(tc.rt, g, &maintenanceProc{}) { t.Fatal("a running generation read as terminated") } if !exists(tc.own) { t.Fatal("the directory went before the termination proof") } mk(tc.own, filepath.Base(tc.rt.Socket)) // the app left its socket behind if !a.genTerminated(tc.rt, g, goneProc{}) { t.Fatal("an exited generation read as running") } if !exists(sock) { t.Fatal("the generation's own leftover socket was removed; only an empty directory goes") } if err := os.Remove(sock); err != nil { t.Fatal(err) } if !a.genTerminated(tc.rt, g, goneProc{}) { t.Fatal("an exited generation read as running") } if exists(tc.own) { t.Fatalf("the retired generation's empty directory %s is still there", tc.own) } for _, s := range tc.siblings { if !exists(s) { t.Fatalf("a sibling's directory %s was removed", s) } } if !exists(tc.rt.Socket) { t.Fatal("the canonical socket was touched") } // A handleless generation proven gone is cleaned the same way. mk(tc.own) if !a.handlelessGenEmpty(tc.rt, g) { t.Fatal("a handleless generation with no child read as running") } if exists(tc.own) { t.Fatalf("the handleless generation's empty directory %s is still there", tc.own) } // A generation on the canonical socket has no directory of its // own: the run dir it sits in is never a candidate. canon := Gen{N: 9, Socket: tc.rt.Socket, Control: tc.rt.Socket + ".control"} if err := os.Remove(tc.rt.Socket); err != nil { t.Fatal(err) } for _, s := range tc.siblings { if err := os.RemoveAll(s); err != nil { t.Fatal(err) } } if tc.name == "flat" { if err := os.Remove(filepath.Join(root, "g")); err != nil { t.Fatal(err) } } if !a.genTerminated(tc.rt, canon, goneProc{}) { t.Fatal("an exited canonical generation read as running") } if !exists(filepath.Dir(tc.rt.Socket)) { t.Fatal("a canonical generation's run dir was removed") } }) } } // Astra, review of 19c07b2b (P2 1): a flat host too long for a generation // path binds its canonical socket with perGen true, and a canonical link a // box left from the old naming must not fail that bind, as it does not on // a flag-off spawn or StartStaged's foreign arm. func TestOverLongFlatHostClearsALeftoverCanonicalLink(t *testing.T) { requireUnixSockets(t) b := testExecBackend(t) b.RunDirOverride = shortRunDir(t) host := strings.Repeat("a", 60-len(".example.com")) + ".example.com" // 60 bytes stageServingLive(t, b, host) rt := flatGenRoute(b, host) if err := os.Symlink("/run/carlos/g/0123456789ab.3.sock", rt.Socket); err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() proc, g, err := b.StartGen(ctx, rt, 9, true) if err != nil { t.Fatalf("StartGen over a leftover canonical link: %v", err) } defer proc.Stop(context.Background()) if g.Socket != rt.Socket { t.Fatalf("handle socket %s, want the canonical %s", g.Socket, rt.Socket) } if !isSocket(rt.Socket) { t.Fatalf("%s is not a bound socket: the leftover link was not cleared", rt.Socket) } } // Astra, review of 19c07b2b (P2 2): a failed wake's generation directory // goes once failedWakeStopped's whole proof holds, the host cgroup's // included where the generation had no child of its own, and never before. func TestFailedWakeRemovesItsDirectoryOnlyOnceProven(t *testing.T) { t.Parallel() running := &sigtermIgnorer{} exited := &sigtermIgnorer{} exited.exited.Store(true) for _, row := range []struct { name string proc Proc killGen error // nil: the generation's own child was proven empty populated bool // the host cgroup still holds something want bool }{ {"leader still running", running, nil, false, false}, {"child not proven empty", exited, errors.New("kill gen: timeout"), false, false}, {"uncontained, host cgroup populated", exited, errNoGenContainment, true, false}, {"uncontained, host cgroup empty", exited, errNoGenContainment, false, true}, {"contained, child proven empty", exited, nil, true, true}, } { t.Run(row.name, func(t *testing.T) { a, b, _, _ := testActivator(t, time.Hour) kb := &killGenBackend{fakeBackend: b, err: row.killGen} a.backend = &uncontainedStartBackend{unprovenStartBackend: &unprovenStartBackend{killGenBackend: kb}, populated: row.populated} root := t.TempDir() rt := testRoute("shop.example.com") rt.Socket = filepath.Join(root, "shop.example.com.sock") own := filepath.Join(root, "g", "4") sibling := filepath.Join(root, "g", "3") for _, d := range []string{own, sibling} { if err := os.MkdirAll(d, 0o750); err != nil { t.Fatal(err) } } g := Gen{N: 4, Socket: filepath.Join(own, "shop.example.com.sock")} g.Control = g.Socket + ".control" if got := a.failedWakeStopped(rt, g, row.proc, LayoutFlat); got != row.want { t.Fatalf("failedWakeStopped = %v, want %v", got, row.want) } if _, err := os.Stat(own); os.IsNotExist(err) != row.want { t.Fatalf("own directory removed = %v, want %v (stat: %v)", os.IsNotExist(err), row.want, err) } if _, err := os.Stat(sibling); err != nil { t.Fatalf("a sibling's empty directory was touched: %v", err) } }) } }