package activator import ( "context" "encoding/json" "errors" "net" "net/http" "os" "path/filepath" "strings" "sync" "testing" "time" "golang.org/x/sys/unix" "github.com/carlosframework/platform/internal/instancectl" "github.com/carlosframework/platform/internal/provision" "github.com/carlosframework/platform/internal/registry" ) // fakeControl records verbs and lets a test drive the unit's state. type fakeControl struct { mu sync.Mutex calls []string inv string seed instancectl.SeedResponse startErr error // stopGenErr is what fakeGenControl.StopGen answers. stopGenErr error stopHook func(host string, role instancectl.Role) readyErr error // onStart runs inside Start, in the fake control's own goroutine, the // same place the real control service's trampoline would exec the app // and the app would bind its socket: prepare's stale-socket removal // races a test's OWN pre-bound fixture socket when that bind happens // before Start is even called, so tests move it here instead. onStart func() // onSeed runs inside Seed, standing in for whatever a real seed process // might have written to disk before it failed. A test uses it to give // seedOwnsDB cleanup something real to (not) remove. onSeed func() cmdInv string cmdErr error cmdStatus instancectl.CmdStatus } func (f *fakeControl) rec(s string) { f.mu.Lock(); f.calls = append(f.calls, s); f.mu.Unlock() } func (f *fakeControl) Start(_ context.Context, host string, role instancectl.Role, max int64) (string, error) { f.rec("start " + host + " " + string(role)) if f.onStart != nil { f.onStart() } return f.inv, f.startErr } func (f *fakeControl) Seed(_ context.Context, host string, max int64) (instancectl.SeedResponse, error) { f.rec("seed " + host) if f.onSeed != nil { f.onSeed() } return f.seed, nil } func (f *fakeControl) Cmd(_ context.Context, host string, max int64) (string, error) { f.rec("cmd " + host) return f.cmdInv, f.cmdErr } func (f *fakeControl) CmdStatus(_ context.Context, host, inv string) (instancectl.CmdStatus, error) { f.rec("cmdstatus " + host + " " + inv) return f.cmdStatus, nil } func (f *fakeControl) Stop(_ context.Context, host string, role instancectl.Role) error { f.rec("stop " + host + " " + string(role)) if f.stopHook != nil { f.stopHook(host, role) } return nil } func (f *fakeControl) Reset(_ context.Context, host string, role instancectl.Role) error { f.rec("reset") return nil } func (f *fakeControl) Sweep(context.Context) (instancectl.SweepResponse, error) { f.rec("sweep") return instancectl.SweepResponse{}, nil } func (f *fakeControl) Ready(context.Context) error { f.rec("ready"); return f.readyErr } func (f *fakeControl) joined() string { f.mu.Lock() defer f.mu.Unlock() return strings.Join(f.calls, "\n") } // fakeUnit is the unit's observable state: what `systemctl show` and // cgroup.events would say. type fakeUnit struct { mu sync.Mutex active string inv string exists bool populated bool } func (u *fakeUnit) set(active, inv string, exists, populated bool) { u.mu.Lock() u.active, u.inv, u.exists, u.populated = active, inv, exists, populated u.mu.Unlock() } func (u *fakeUnit) state(context.Context, string) (string, string, error) { u.mu.Lock() defer u.mu.Unlock() return u.active, u.inv, nil } func (u *fakeUnit) events(string, string) (bool, bool, error) { u.mu.Lock() defer u.mu.Unlock() return u.exists, u.populated, nil } // nestedBackend returns an ExecBackend wired for a nested host with fakes, // and the route. The data and run dirs are temp dirs standing in for // /data/instances/ and /run/carlos/instances/; provision's // helpers are not used for the temp paths, so the route is built to // classify as nested by construction through the Layout seam. func nestedBackend(t *testing.T, ctl *fakeControl, unit *fakeUnit) (*ExecBackend, registry.Route) { t.Helper() root, err := os.MkdirTemp("", "nb") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(root) }) host := "h.example.com" b := testExecBackend(t) b.DataDir = filepath.Join(root, "data") b.Control = ctl b.UnitState = unit.state b.CgroupEvents = unit.events b.ObservePoll = 20 * time.Millisecond b.RunDirOverride = filepath.Join(root, "run") stageLive(t, b, host, "v1") rt := registry.Route{Host: host, Kind: "instance", Hibernate: true, DBPath: provision.NestedDBPath(b.DataDir, host), Socket: filepath.Join(b.RunDirOverride, "instances", host, "app.sock")} return b, rt } func TestStartUnitWritesTheLaunchFileAndReturnsAUnitProc(t *testing.T) { t.Parallel() ctl := &fakeControl{inv: "inv1"} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) // The fake control does not start anything, so health must be faked: // bind a listener at rt.Socket that answers 200, from inside Start // itself, where the app's own bind would actually happen, so // prepare's stale-socket removal (which runs before Start ever calls // the fake control) cannot race it. var stopServing func() ctl.onStart = func() { stopServing = serveOK(t, rt.Socket) } defer func() { if stopServing != nil { stopServing() } }() unit.set("active", "inv1", true, true) p, ver, err := b.Start(context.Background(), rt) if err != nil { t.Fatalf("Start: %v", err) } _ = ver up, ok := p.(*unitProc) if !ok || up.invocation != "inv1" { t.Fatalf("proc = %T %+v", p, p) } raw, err := os.ReadFile(filepath.Join(provision.InstanceDir(b.DataDir, rt.Host), "launch.run.json")) if err != nil { t.Fatal(err) } var l instancectl.Launch if err := json.Unmarshal(raw, &l); err != nil { t.Fatal(err) } if l.Schema != 1 || l.Host != rt.Host || len(l.Argv) != 5 || l.Argv[0] != filepath.Join(b.OptDir, "live", rt.Host) || l.Argv[2] != rt.Socket || l.Argv[4] != rt.DBPath { t.Errorf("launch = %+v", l) } seenCtl := false for _, kv := range l.Env { if strings.HasPrefix(kv, "CARLOS_CONTROL_SOCKET=") { seenCtl = true } if strings.HasPrefix(kv, "CARLOS_ADMIN_TOKEN=") && len(kv) < 30 { t.Errorf("token missing from env: %s", kv) } } if !seenCtl { t.Error("CARLOS_CONTROL_SOCKET not in the launch env") } if !strings.Contains(ctl.joined(), "start h.example.com run") { t.Errorf("control calls: %s", ctl.joined()) } if p.Exited() { t.Error("exited before the unit went inactive") } // Termination: terminal state plus empty cgroup. unit.set("inactive", "inv1", false, false) waitFor(t, "the unit to exit", func() bool { return p.Exited() }) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() if err := p.Stop(ctx); err != nil { t.Errorf("Stop after exit: %v", err) } } func TestUnitProcDoesNotExitOnTerminalStateWhileTheCgroupIsPopulated(t *testing.T) { t.Parallel() ctl := &fakeControl{inv: "inv1"} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) var stopServing func() ctl.onStart = func() { stopServing = serveOK(t, rt.Socket) } defer func() { if stopServing != nil { stopServing() } }() unit.set("active", "inv1", true, true) p, _, err := b.Start(context.Background(), rt) if err != nil { t.Fatal(err) } unit.set("failed", "inv1", true, true) // a process survived SIGKILL time.Sleep(150 * time.Millisecond) if p.Exited() { t.Fatal("exited while the cgroup is still populated (codex R2-5)") } ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() if err := p.Stop(ctx); !errors.Is(err, ErrStillRunning) { t.Fatalf("Stop = %v, want ErrStillRunning", err) } unit.set("failed", "inv1", false, false) waitFor(t, "the unit to exit", func() bool { return p.Exited() }) } func TestUnitProcStopsAForeignInvocation(t *testing.T) { t.Parallel() ctl := &fakeControl{inv: "inv1"} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) var stopServing func() ctl.onStart = func() { stopServing = serveOK(t, rt.Socket) } defer func() { if stopServing != nil { stopServing() } }() unit.set("active", "inv1", true, true) p, _, err := b.Start(context.Background(), rt) if err != nil { t.Fatal(err) } ctl.stopHook = func(string, instancectl.Role) { unit.set("inactive", "", false, false) } unit.set("active", "inv2", true, true) // someone restarted it outside the activator waitFor(t, "the unit to exit", func() bool { return p.Exited() }) if !strings.Contains(ctl.joined(), "stop h.example.com run") { t.Errorf("foreign invocation was not stopped: %s", ctl.joined()) } } func TestStartUnitRefusesWhenTheBoxIsNotReadyOrFenced(t *testing.T) { ctl := &fakeControl{inv: "inv1"} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) b.NestedOK = func() error { return errors.New("probe: pid1 fail") } if _, _, err := b.Start(context.Background(), rt); err == nil || !strings.Contains(err.Error(), "pid1") { t.Fatalf("not-ready wake = %v", err) } if strings.Contains(ctl.joined(), "start") { t.Error("a start was issued on a box that is not ready") } b.NestedOK = nil b.LayoutFacts.Journal = func(string) bool { return true } if _, _, err := b.Start(context.Background(), rt); err == nil || !strings.Contains(err.Error(), "migration in flight") { t.Fatalf("fenced wake = %v", err) } } func TestSeedUnitCleansUpOnlyItsOwnDBOnFailure(t *testing.T) { ctl := &fakeControl{seed: instancectl.SeedResponse{InvocationID: "s1", Result: "exit-code", Status: "3", Code: "exited"}} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) rt.Seed = `["seed","--fast"]` unit.set("inactive", "", false, false) err := b.Seed(context.Background(), rt) if err == nil || !strings.Contains(err.Error(), "exit 3") { t.Fatalf("Seed = %v", err) } raw, err := os.ReadFile(filepath.Join(provision.InstanceDir(b.DataDir, rt.Host), "launch.seed.json")) if err != nil { t.Fatal(err) } var l instancectl.Launch _ = json.Unmarshal(raw, &l) if len(l.Argv) != 5 || l.Argv[1] != "seed" || l.Argv[2] != "--fast" || l.Argv[3] != "--db" || l.Argv[4] != rt.DBPath { t.Errorf("seed argv = %v", l.Argv) } gotHost := false for _, kv := range l.Env { if strings.HasPrefix(kv, "CARLOS_CONTROL_SOCKET=") || strings.HasPrefix(kv, "CARLOS_PURGE=") { t.Errorf("seed env carries %s", kv) } gotHost = gotHost || kv == "CARLOS_HOST="+rt.Host } if !gotHost { t.Errorf("seed env lacks CARLOS_HOST=%s (#256): %v", rt.Host, l.Env) } if _, err := os.Stat(rt.DBPath); !os.IsNotExist(err) { t.Errorf("failed seed left its database: %v", err) } } // capturedEvent is one call eventCapture recorded. type capturedEvent struct{ level, msg string } // eventCapture is a minimal EventSink that records Event calls for // assertions; App is unused by these tests but required by the interface. type eventCapture struct { mu sync.Mutex events []capturedEvent } func (e *eventCapture) App(acct, app, host, line string) {} func (e *eventCapture) Event(acct, app, host, level, msg string) { e.mu.Lock() e.events = append(e.events, capturedEvent{level, msg}) e.mu.Unlock() } func (e *eventCapture) last() (capturedEvent, bool) { e.mu.Lock() defer e.mu.Unlock() if len(e.events) == 0 { return capturedEvent{}, false } return e.events[len(e.events)-1], true } // TestUnitProcExitLineBoundsAForgedExitRecord covers the exit record's // ruling: it is tenant-writable text, so a record with fields no real // trampoline would ever write must still produce a bounded, single-line, // member-safe event, and the record must be consumed (removed) exactly // once. func TestUnitProcExitLineBoundsAForgedExitRecord(t *testing.T) { t.Parallel() ctl := &fakeControl{inv: "inv1"} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) sink := &eventCapture{} b.Logs = sink var stopServing func() ctl.onStart = func() { stopServing = serveOK(t, rt.Socket) } defer func() { if stopServing != nil { stopServing() } }() unit.set("active", "inv1", true, true) p, _, err := b.Start(context.Background(), rt) if err != nil { t.Fatal(err) } up, ok := p.(*unitProc) if !ok { t.Fatalf("proc = %T", p) } rec := instancectl.ExitRecord{ Invocation: up.invocation, Result: "boom\nboom", // a newline: must never reach the event line raw Status: strings.Repeat("9", 200), // far past recordToken's 32-byte bound Code: "exi\x00ted", // a control character } raw, err := json.Marshal(rec) if err != nil { t.Fatal(err) } recordPath := filepath.Join(up.runDir, instancectl.ExitRecordName(instancectl.RoleRun, up.invocation)) if err := os.WriteFile(recordPath, raw, 0o600); err != nil { t.Fatal(err) } unit.set("inactive", "inv1", false, false) waitFor(t, "the unit to exit", func() bool { return p.Exited() }) ev, ok := sink.last() if !ok { t.Fatal("no event emitted") } if ev.level != "warn" { t.Errorf("level = %q, want warn", ev.level) } if ev.msg != "instance exited: exit ? (?)" { t.Errorf("msg = %q, want %q", ev.msg, "instance exited: exit ? (?)") } if strings.Contains(ev.msg, "\n") { t.Errorf("event line contains a newline: %q", ev.msg) } if _, err := os.Stat(recordPath); !os.IsNotExist(err) { t.Errorf("exit record was not removed after being consumed: %v", err) } } // TestStartUnitWithEmptyInvocationFailsCleanly covers the case the control // service reports no invocation at all: the unit already exited before its // start was observed (plan 2 final review). The wake must fail with a // diagnosis naming that, and, because nothing is running, no unitProc // is ever created to observe, so Control must never see a stop. func TestStartUnitWithEmptyInvocationFailsCleanly(t *testing.T) { ctl := &fakeControl{inv: ""} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) if _, _, err := b.Start(context.Background(), rt); err == nil || !strings.Contains(err.Error(), "exited before it was observed") { t.Fatalf("Start = %v", err) } if strings.Contains(ctl.joined(), "stop") { t.Errorf("nothing was ever running; nothing should have been stopped: %s", ctl.joined()) } } // TestStopKeepsListenersOpenUntilTerminationIsProven covers Stop's // ordering contract: an unproven stop (ErrStillRunning) must leave the // control and log listeners open, since a writer that may still exist must // still be able to reach them, and only a stop that observes done may // close them. func TestStopKeepsListenersOpenUntilTerminationIsProven(t *testing.T) { ctl := &fakeControl{inv: "inv1"} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) var stopServing func() ctl.onStart = func() { stopServing = serveOK(t, rt.Socket) } defer func() { if stopServing != nil { stopServing() } }() unit.set("active", "inv1", true, true) p, _, err := b.Start(context.Background(), rt) if err != nil { t.Fatal(err) } logSocket := b.logSocket(rt.Host, instancectl.RoleRun) ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() if err := p.Stop(ctx); !errors.Is(err, ErrStillRunning) { t.Fatalf("Stop = %v, want ErrStillRunning", err) } c, err := net.Dial("unix", logSocket) if err != nil { t.Fatalf("log socket refused a dial after an unproven stop: %v", err) } c.Close() unit.set("inactive", "inv1", false, false) waitFor(t, "the unit to exit", func() bool { return p.Exited() }) ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) defer cancel2() if err := p.Stop(ctx2); err != nil { t.Errorf("second Stop after exit: %v", err) } if c2, err := net.Dial("unix", logSocket); err == nil { c2.Close() t.Error("log socket still accepts a dial after Stop closed the listeners") } } // TestStopClosesListenersAfterDoneWithNoSecondStopCall pins the fix for the // leak Stop's ctx.Done branch used to have: several real callers (the // sidecar co-stop, a discarded sidecar, a failed health wait) never call // Stop a second time once it returns ErrStillRunning. The listeners must // still close once termination is proven, with nobody asking again. func TestStopClosesListenersAfterDoneWithNoSecondStopCall(t *testing.T) { t.Parallel() ctl := &fakeControl{inv: "inv1"} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) var stopServing func() ctl.onStart = func() { stopServing = serveOK(t, rt.Socket) } defer func() { if stopServing != nil { stopServing() } }() unit.set("active", "inv1", true, true) p, _, err := b.Start(context.Background(), rt) if err != nil { t.Fatal(err) } logSocket := b.logSocket(rt.Host, instancectl.RoleRun) ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() if err := p.Stop(ctx); !errors.Is(err, ErrStillRunning) { t.Fatalf("Stop = %v, want ErrStillRunning", err) } unit.set("inactive", "inv1", false, false) waitFor(t, "the unit to exit", func() bool { return p.Exited() }) deadline := time.Now().Add(2 * time.Second) for { c, err := net.Dial("unix", logSocket) if err != nil { break // the listener closed on its own; no second Stop call was made } c.Close() if time.Now().After(deadline) { t.Fatal("log socket still accepts a dial 2s after the unit exited, with no second Stop call") } time.Sleep(10 * time.Millisecond) } } // TestSeedUnitFailureStopsBeforeCleanupAndKeepsTheDBWhenProcessesRemain // covers the ordering rule: Control.Stop for the seed unit runs, and is // proven by an empty cgroup, BEFORE any database cleanup, and when the // cgroup never reports empty, nothing is removed at all. The wait is // bounded by roleEmptyBound (actStopTimeout in production), which this test // genuinely waits out, shrunk to a fraction of a second. func TestSeedUnitFailureStopsBeforeCleanupAndKeepsTheDBWhenProcessesRemain(t *testing.T) { if roleEmptyBound != actStopTimeout { t.Fatalf("roleEmptyBound = %v, want actStopTimeout (%v) outside a test", roleEmptyBound, actStopTimeout) } old := roleEmptyBound roleEmptyBound = 300 * time.Millisecond t.Cleanup(func() { roleEmptyBound = old }) ctl := &fakeControl{seed: instancectl.SeedResponse{InvocationID: "s1", Result: "exit-code", Status: "3", Code: "exited"}} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) rt.Seed = `["seed","--fast"]` // Stand in for a real seed process that wrote its half-built database // before dying, created only once Seed actually runs, so the PRE-seed // stat that decides seedOwnsDB still sees it absent (seedOwnsDB=true). ctl.onSeed = func() { if err := os.WriteFile(rt.DBPath, []byte("half-built"), 0o600); err != nil { t.Fatal(err) } } // The cgroup stays populated for the whole wait: a process from the // seed survived SIGKILL and is still there when the bound elapses. unit.set("active", "s1", true, true) err := b.Seed(context.Background(), rt) if err == nil || !strings.Contains(err.Error(), "processes remain") { t.Fatalf("Seed = %v, want a processes-remain error", err) } if !strings.Contains(ctl.joined(), "stop h.example.com seed") { t.Errorf("seed unit was never stopped before cleanup was attempted: %s", ctl.joined()) } if _, err := os.Stat(rt.DBPath); err != nil { t.Errorf("the database was removed despite its processes remaining: %v", err) } } // switchableEvents is a CgroupEvents seam a test can turn from "cannot // read" (an unproven cgroup, answered at once rather than after // actStopTimeout) to "gone". type switchableEvents struct { mu sync.Mutex err error reads []string } func (s *switchableEvents) set(err error) { s.mu.Lock(); s.err = err; s.mu.Unlock() } func (s *switchableEvents) events(slice, unit string) (bool, bool, error) { s.mu.Lock() defer s.mu.Unlock() s.reads = append(s.reads, unit) if s.err != nil { return false, false, s.err } return false, false, nil } func (s *switchableEvents) read() []string { s.mu.Lock() defer s.mu.Unlock() return append([]string(nil), s.reads...) } // Task 8 fix round 1, finding 2: a nested seed writes its obligation ahead // as a flat one does, before root is asked to run anything, and a seed // whose unit is proven empty removes it. func TestANestedSeedRecordsItsObligationBeforeItRuns(t *testing.T) { ctl := &fakeControl{seed: instancectl.SeedResponse{InvocationID: "s1", Result: "success"}} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) rt.Seed = `["seed"]` ob := rt.DBPath + seedUnprovenSuffix seen := false ctl.onSeed = func() { _, err := os.Stat(ob) seen = err == nil } unit.set("inactive", "", false, false) if err := b.Seed(context.Background(), rt); err != nil { t.Fatalf("Seed: %v", err) } if !seen { t.Fatal("root ran the seed with no obligation on disk") } if _, err := os.Stat(ob); !os.IsNotExist(err) { t.Fatalf("obligation still there (%v) after a proven seed", err) } } // Finding 2: a nested seed whose unit is not proven empty (failed or // successful) keeps its database AND leaves the obligation. The next wake // proves the nested seed's own unit empty (stopped, then its cgroup read), // never carlos-exec@, which a nested seed never ran in: while that // proof fails the wake is refused and the database is untouched; once it // holds, the database and the obligation go and the wake is fresh. func TestAnUnprovenNestedSeedLeavesItsObligationAndTheNextWakeProvesItsUnit(t *testing.T) { for _, result := range []string{"exit-code", "success"} { t.Run(result, func(t *testing.T) { ctl := &fakeControl{seed: instancectl.SeedResponse{InvocationID: "s1", Result: result, Status: "3"}} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) rt.Seed = `["seed"]` ev := &switchableEvents{err: errors.New("cgroup.events: unreadable")} b.CgroupEvents = ev.events var flat []string b.cgroupKill = func(_ context.Context, _ registry.Route, child string) error { flat = append(flat, child) return errors.New("a nested seed never ran in carlos-exec@") } swapRestoreReplica(t, func(context.Context, string, string, string, string) (bool, error) { return false, nil }) ctl.onSeed = func() { if err := os.WriteFile(rt.DBPath, []byte("half"), 0o600); err != nil { t.Fatal(err) } } ob := rt.DBPath + seedUnprovenSuffix if err := b.Seed(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) { t.Fatalf("Seed = %v, want ErrSeedUnproven", err) } if _, err := os.Stat(ob); err != nil { t.Fatalf("no obligation beside the nested seed's database: %v", err) } if fresh, err := b.Restore(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) || fresh { t.Fatalf("Restore = %v, %v; want a refusal while the seed unit is unproven", fresh, err) } if got, _ := os.ReadFile(rt.DBPath); string(got) != "half" { t.Fatalf("database = %q after a refused wake, want it untouched", got) } ev.set(nil) before := strings.Count(ctl.joined(), "stop h.example.com seed") if fresh, err := b.Restore(context.Background(), rt); err != nil || !fresh { t.Fatalf("Restore = %v, %v; want fresh once the seed unit is proven empty", fresh, err) } if strings.Count(ctl.joined(), "stop h.example.com seed") <= before { t.Fatalf("the wake did not stop the seed unit before its proof: %s", ctl.joined()) } if len(flat) != 0 { t.Fatalf("the nested seed's proof asked carlos-exec@ (%q)", flat) } if reads := ev.read(); len(reads) == 0 || reads[len(reads)-1] != instancectl.UnitName(instancectl.RoleSeed, rt.Host) { t.Fatalf("cgroup reads %q, want the seed unit's", reads) } for _, p := range []string{rt.DBPath, ob} { if _, err := os.Stat(p); !os.IsNotExist(err) { t.Fatalf("%s still there (%v) after the proof held", filepath.Base(p), err) } } }) } } // The obligation sits beside the database, on ground the instance can // write. A FIFO planted at its name must not hang the wake (the read never // waits for a writer), and it must not hold the row down either: it is an // obligation that does not say what the seed owned, so it is proven, the // database is kept, and the planted name is removed. func TestAPlantedFIFOAtTheObligationIsAnsweredNotWaitedOn(t *testing.T) { const host = "fifo-obligation.example.com" b, _ := delegatedBackend(t, host) b.cgroupKill = func(context.Context, registry.Route, string) error { return nil } rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`} db := b.pathsFor(rt).dbPath if err := os.MkdirAll(filepath.Dir(db), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(db, []byte("data"), 0o600); err != nil { t.Fatal(err) } if err := unix.Mkfifo(db+seedUnprovenSuffix, 0o600); err != nil { t.Fatalf("Mkfifo: %v", err) } type result struct { fresh bool err error } done := make(chan result, 1) go func() { fresh, err := b.Restore(context.Background(), rt) done <- result{fresh, err} }() select { case r := <-done: if r.err != nil || r.fresh { t.Fatalf("Restore = %v, %v; want the warm database once the proof holds", r.fresh, r.err) } case <-time.After(5 * time.Second): t.Fatal("Restore hung on a FIFO planted at the obligation's name") } if got, err := os.ReadFile(db); err != nil || string(got) != "data" { t.Fatalf("database = %q, %v; want it kept: the planted name says nothing the seed owned", got, err) } if _, err := os.Lstat(db + seedUnprovenSuffix); !os.IsNotExist(err) { t.Fatalf("the planted name is still there (%v)", err) } } // TestPrepareRemovesAStaleSocketFile covers prepare's stale-socket removal: // a file left behind at rt.Socket by an earlier, SIGKILLed invocation must // not survive into the next one, or every later wake's health check would // fail with nothing but a dial timeout. func TestPrepareRemovesAStaleSocketFile(t *testing.T) { ctl := &fakeControl{inv: "inv1"} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) if err := os.MkdirAll(filepath.Dir(rt.Socket), 0o750); err != nil { t.Fatal(err) } if err := os.WriteFile(rt.Socket, []byte("stale"), 0o600); err != nil { t.Fatal(err) } _, ctl2, logs, _, err := b.prepare(rt, instancectl.RoleRun) if err != nil { t.Fatalf("prepare: %v", err) } defer logs.Close() if ctl2 != nil { defer ctl2.Close() } if _, err := os.Stat(rt.Socket); !os.IsNotExist(err) { t.Errorf("stale socket file was not removed by prepare: %v", err) } } // TestPrepareRefusesAFIFOAtTheAdminTokenPath pins EnsureAdminTokenBeneath's // use from the nested prepare: the token file lives on the same // tenant-writable ground as everything else prepare touches, so a FIFO // planted at that name must fail prepare fast rather than hang it forever // behind the token mint's own read. func TestPrepareRefusesAFIFOAtTheAdminTokenPath(t *testing.T) { ctl := &fakeControl{inv: "inv1"} unit := &fakeUnit{} b, rt := nestedBackend(t, ctl, unit) instDir := provision.InstanceDir(b.DataDir, rt.Host) if err := os.MkdirAll(instDir, 0o700); err != nil { t.Fatal(err) } if err := unix.Mkfifo(filepath.Join(instDir, rt.Host+".env"), 0o600); err != nil { t.Fatalf("Mkfifo: %v", err) } done := make(chan error, 1) go func() { _, _, _, _, err := b.prepare(rt, instancectl.RoleRun) done <- err }() select { case err := <-done: if err == nil { t.Fatal("prepare against a FIFO'd admin token file succeeded, want an error") } case <-time.After(2 * time.Second): t.Fatal("prepare blocked on a FIFO'd admin token file instead of refusing it") } } // serveOK binds a unix socket at path and answers 200 to every request, so // waitHealthyCtx passes without a real unit. func serveOK(t *testing.T, path string) func() { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { t.Fatal(err) } ln, err := net.Listen("unix", path) if err != nil { t.Skipf("cannot bind here (%v); run with TMPDIR=/tmp/g outside the sandbox", err) } srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })} go srv.Serve(ln) //nolint:errcheck return func() { srv.Close(); ln.Close() } }