package caps import ( "bytes" "errors" "fmt" "os" "path/filepath" "strings" "testing" ) // fakeSystemd is a tiny in-memory systemd: it RECORDS every state-changing // verb and answers the query seam from the state those verbs left behind. The // state tracking is what lets "converge twice, the second pass changes // nothing" be asserted against a DAEMON route — the interesting case — rather // than only against a route with no unit state to repair. type fakeSystemd struct { calls []string active map[string]bool enabled map[string]bool fail map[string]error // "verb unit" -> error } func newFakeSystemd() *fakeSystemd { return &fakeSystemd{active: map[string]bool{}, enabled: map[string]bool{}, fail: map[string]error{}} } func (f *fakeSystemd) run(args ...string) error { call := strings.Join(args, " ") f.calls = append(f.calls, call) if err := f.fail[call]; err != nil { return err } if len(args) < 2 { return nil } switch args[0] { case "start", "restart": f.active[args[1]] = true case "stop": f.active[args[1]] = false case "enable": f.enabled[args[1]] = true case "disable": f.enabled[args[1]] = false } return nil } func (f *fakeSystemd) query(args ...string) bool { if len(args) < 2 { return false } switch args[0] { case "is-active": return f.active[args[1]] case "is-enabled": return f.enabled[args[1]] } return false } func (f *fakeSystemd) reset() { f.calls = nil } // testOps wires every seam at a temp dir so nothing here touches systemd, // /etc, /run or /var/lib. func testOps(t *testing.T) (Ops, *fakeSystemd, *[]string, *[]string) { t.Helper() f := newFakeSystemd() var logs []string var users []string o := Ops{ EtcDir: t.TempDir(), StateRoot: t.TempDir(), RunDir: t.TempDir(), Systemctl: f.run, Query: f.query, EnsureUser: func(name string) error { users = append(users, name) return nil }, Logf: func(format string, a ...any) { logs = append(logs, fmt.Sprintf(format, a...)) }, } return o, f, &logs, &users } func keymailPlan(t *testing.T, g Grants) Plan { t.Helper() return Plan{Host: "keymail.dev", App: "keymail", Set: keymailSet(t), Grants: g} } func readFile(t *testing.T, path string) []byte { t.Helper() b, err := os.ReadFile(path) if err != nil { t.Fatalf("read %s: %v", path, err) } return b } func mustNotExist(t *testing.T, path string) { t.Helper() if _, err := os.Lstat(path); !os.IsNotExist(err) { t.Fatalf("%s: want absent, got err=%v", path, err) } } func hasLog(logs []string, substr string) bool { for _, l := range logs { if strings.Contains(l, substr) { return true } } return false } // indexOf returns the position of the first call equal to want, or -1. func indexOf(calls []string, want string) int { for i, c := range calls { if c == want { return i } } return -1 } // TestGrantAddedWritesTheDropInAndRestarts pins the whole happy path: both // drop-ins are the golden bytes, exactly one daemon-reload, the socket // sequence in its pinned order (a socket restart under a running service // races into EADDRINUSE), and `enable` for BOTH units — bootstrap enables no // tenant socket unit, so without it the ListenStream binds nothing after a // reboot. func TestGrantAddedWritesTheDropInAndRestarts(t *testing.T) { o, f, _, users := testOps(t) p := keymailPlan(t, keymailGrants()) res, err := Converge(o, p) if err != nil { t.Fatalf("Converge: %v", err) } if !res.Changed { t.Fatal("Changed = false, want true on a first converge") } svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) if got, want := readFile(t, svcPath), readGolden(t, "keymail.service.conf"); !bytes.Equal(got, want) { t.Fatalf("service drop-in:\n got: %q\n want: %q", got, want) } if got, want := readFile(t, sockPath), readGolden(t, "keymail.socket.conf"); !bytes.Equal(got, want) { t.Fatalf("socket drop-in:\n got: %q\n want: %q", got, want) } if len(*users) != 1 || (*users)[0] != "carlos-keymail" { t.Fatalf("EnsureUser calls = %v, want [carlos-keymail]", *users) } reloads := 0 for _, c := range f.calls { if c == "daemon-reload" { reloads++ } } if reloads != 1 { t.Fatalf("daemon-reload x%d, want exactly 1 (calls: %v)", reloads, f.calls) } stop := indexOf(f.calls, "stop carlos-app@keymail.dev.service") sock := indexOf(f.calls, "restart carlos-app@keymail.dev.socket") start := indexOf(f.calls, "start carlos-app@keymail.dev.service") if stop < 0 || sock < 0 || start < 0 || !(stop < sock && sock < start) { t.Fatalf("socket ordering: stop=%d socket=%d start=%d, want stop < socket restart < start (calls: %v)", stop, sock, start, f.calls) } if indexOf(f.calls, "enable carlos-app@keymail.dev.service") < 0 { t.Fatalf("no enable for the service unit (calls: %v)", f.calls) } if indexOf(f.calls, "enable carlos-app@keymail.dev.socket") < 0 { t.Fatalf("no enable for the socket unit — the ListenStream would bind nothing after a reboot (calls: %v)", f.calls) } // The reload precedes EVERY unit verb: a restart against configuration // systemd has not re-read is a pass that looks like it worked and changed // nothing. reload := indexOf(f.calls, "daemon-reload") for i, c := range f.calls { verb, _, _ := strings.Cut(c, " ") switch verb { case "restart", "start", "stop", "enable": if i < reload { t.Fatalf("%q ran before the daemon-reload: %v", c, f.calls) } } } } // A failed daemon-reload must not leave the new bytes on disk: the next pass // would see no diff, conclude there is nothing to do, and the box would run // the old configuration behind a drop-in that reads "converged" forever. func TestAFailedReloadRollsTheBytesBackSoTheNextPassRetries(t *testing.T) { o, f, _, _ := testOps(t) p := keymailPlan(t, keymailGrants()) f.fail["daemon-reload"] = errors.New("systemd is not running") res, err := Converge(o, p) if err == nil { t.Fatal("Converge swallowed a failed daemon-reload") } if res.Changed { t.Fatal("Changed = true after the pass was rolled back") } svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) mustNotExist(t, svcPath) mustNotExist(t, sockPath) for _, c := range f.calls { verb, _, _ := strings.Cut(c, " ") if verb == "restart" || verb == "start" || verb == "stop" { t.Fatalf("unit state changed after a failed reload: %v", f.calls) } } // Pass N+1, with a working systemd: the whole change is retried. delete(f.fail, "daemon-reload") f.reset() res, err = Converge(o, p) if err != nil { t.Fatalf("Converge (retry): %v", err) } if !res.Changed { t.Fatal("the retry pass found nothing to do — the rollback did not restore the diff") } if got, want := readFile(t, svcPath), readGolden(t, "keymail.service.conf"); !bytes.Equal(got, want) { t.Fatalf("retry did not converge:\n got: %q\n want: %q", got, want) } if got, want := readFile(t, sockPath), readGolden(t, "keymail.socket.conf"); !bytes.Equal(got, want) { t.Fatalf("retry did not converge the socket half:\n got: %q\n want: %q", got, want) } } // A half-written pass is the same divergence in miniature: one unit // converged, the other not, and the disk claiming both are done. func TestAPartialWriteRollsBothHalvesBackToPrev(t *testing.T) { o, f, _, _ := testOps(t) p := keymailPlan(t, keymailGrants()) if _, err := Converge(o, p); err != nil { t.Fatalf("Converge (first): %v", err) } svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) svcA, sockA := readFile(t, svcPath), readFile(t, sockPath) f.reset() // Version B changes both halves, but the socket half cannot be written: // its temp file lands in a directory that has been replaced by... itself, // made unwritable. if err := os.Chmod(filepath.Dir(sockPath), 0o500); err != nil { t.Fatal(err) } t.Cleanup(func() { os.Chmod(filepath.Dir(sockPath), 0o755) }) var b Set if err := b.UnmarshalJSON([]byte(`{"v":1,"command":"forward","daemon":true,"runtime_dir":"keymail","host_role":true,"ports":[{"port":587,"name":"submission"}]}`)); err != nil { t.Fatal(err) } p.Set = &b p.Grants = Grants{Ports: map[int]bool{587: true}, HostRole: true} res, err := Converge(o, p) if err == nil { t.Skip("this environment allows writing into a 0500 directory (running as root?)") } if res.Changed { t.Fatal("Changed = true after both halves were rolled back") } if got := readFile(t, svcPath); !bytes.Equal(got, svcA) { t.Fatalf("the service half was left mid-change:\n got: %q\n want: %q", got, svcA) } if got := readFile(t, sockPath); !bytes.Equal(got, sockA) { t.Fatalf("the socket half moved:\n got: %q\n want: %q", got, sockA) } for _, c := range f.calls { verb, _, _ := strings.Cut(c, " ") if verb == "restart" || verb == "start" || verb == "stop" { t.Fatalf("unit state changed on a half-written pass: %v", f.calls) } } } // Deviation #4's consequence, made visible: the socket unit is enabled for // hibernating routes too (or the ListenStream binds nothing), and a granted // public port is then a wake path that does not go through the activator's // health gate. func TestAGrantedPortOnAHibernatingRouteIsAdvised(t *testing.T) { o, _, logs, _ := testOps(t) var s Set if err := s.UnmarshalJSON([]byte(`{"v":1,"command":"relay","ports":[{"port":25,"name":"mx"}]}`)); err != nil { t.Fatal(err) } p := Plan{Host: "mx.example.com", App: "keymail", Set: &s, Grants: Grants{Ports: map[int]bool{25: true}}, Hibernate: true} if _, err := Converge(o, p); err != nil { t.Fatalf("Converge: %v", err) } if !hasLog(*logs, "port:25 is granted on a hibernating route") { t.Fatalf("logs = %v, want the hibernating-port advisory", *logs) } if !hasLog(*logs, "bypassing the activator's health-gated wake") { t.Fatalf("logs = %v, want the advisory to say what the risk is", *logs) } // Not advised when the port is merely declared: nothing binds, so there is // no wake path to warn about. Advice that fires on a healthy fleet is // advice nobody reads. o2, _, logs2, _ := testOps(t) q := p q.Host = "mx2.example.com" q.Grants = Grants{} if _, err := Converge(o2, q); err != nil { t.Fatalf("Converge (ungranted): %v", err) } if hasLog(*logs2, "hibernating route") { t.Fatalf("logs = %v, want no advisory for an ungranted port", *logs2) } } // An address-form ListenStream is the same claim: a hand-written drop-in // cannot dodge the box-wide port check by spelling :25 differently. func TestPortClaimsSeeAddressFormListenStreams(t *testing.T) { for _, form := range []string{"25", "0.0.0.0:25", "[::]:25", "10.0.0.4:25"} { o, _, logs, _ := testOps(t) sockPath := filepath.Join(o.EtcDir, SocketUnit("hand.example.com")+".d", DropInName) if err := os.MkdirAll(filepath.Dir(sockPath), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(sockPath, []byte("[Socket]\nListenStream="+form+"\n"), 0o644); err != nil { t.Fatal(err) } if _, err := Converge(o, keymailPlan(t, keymailGrants())); err == nil { t.Fatalf("ListenStream=%s did not count as a claim on :25", form) } if !hasLog(*logs, "port 25 is already claimed by hand.example.com") { t.Fatalf("ListenStream=%s: logs = %v", form, *logs) } } // The template's own unix listener claims no TCP port at all. o, _, _, _ := testOps(t) sockPath := filepath.Join(o.EtcDir, SocketUnit("hand.example.com")+".d", DropInName) if err := os.MkdirAll(filepath.Dir(sockPath), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(sockPath, []byte("[Socket]\nListenStream=/run/carlos/apps/hand.example.com.sock\n"), 0o644); err != nil { t.Fatal(err) } if _, err := Converge(o, keymailPlan(t, keymailGrants())); err != nil { t.Fatalf("a unix ListenStream was read as a TCP claim: %v", err) } } // TestGrantRevokedRewritesTheDropIn: a grant is not a promise. Revoking // port:25 and host-role removes the socket file entirely and rewrites the // service file without its User= lines — and the removal alone is enough to // count as Changed. func TestGrantRevokedRewritesTheDropIn(t *testing.T) { o, f, _, _ := testOps(t) p := keymailPlan(t, keymailGrants()) if _, err := Converge(o, p); err != nil { t.Fatalf("Converge (granted): %v", err) } f.reset() p.Grants = Grants{} res, err := Converge(o, p) if err != nil { t.Fatalf("Converge (revoked): %v", err) } if !res.Changed { t.Fatal("Changed = false after a revoke, want true") } svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) mustNotExist(t, sockPath) svc := string(readFile(t, svcPath)) for _, gone := range []string{"DynamicUser=no", "User=carlos-keymail", "Group=carlos-keymail", "IPAddressAllow"} { if strings.Contains(svc, gone) { t.Fatalf("service drop-in still holds %q after the revoke:\n%s", gone, svc) } } for _, kept := range []string{"ExecStart=/opt/carlos/live/%i relay", "RuntimeDirectory=keymail", "TasksMax=1024"} { if !strings.Contains(svc, kept) { t.Fatalf("service drop-in lost the self-service %q:\n%s", kept, svc) } } reloads := 0 for _, c := range f.calls { if c == "daemon-reload" { reloads++ } } if reloads != 1 { t.Fatalf("daemon-reload x%d, want exactly 1 (calls: %v)", reloads, f.calls) } if res.Prev.Socket == nil || !res.Prev.SocketHad { t.Fatal("Prev did not capture the socket drop-in that was removed") } // A grant is not a promise (spec §7): the socket unit must be restarted // so systemd actually drops :25. Removing the file and reloading is not // enough — the running socket unit is still holding the listener. stop := indexOf(f.calls, "stop carlos-app@keymail.dev.service") sock := indexOf(f.calls, "restart carlos-app@keymail.dev.socket") start := indexOf(f.calls, "start carlos-app@keymail.dev.service") if stop < 0 || sock < 0 || start < 0 || !(stop < sock && sock < start) { t.Fatalf("a revoked port did not go through the socket sequence: %v", f.calls) } } // TestUnreadableManifestLeavesDropInsAlone is the three-state rule, and the // reason it exists: a bucket hiccup must not strip :25 off a running mail // server. Nil Set = leave-alone, not removal. func TestUnreadableManifestLeavesDropInsAlone(t *testing.T) { o, f, logs, _ := testOps(t) p := keymailPlan(t, keymailGrants()) if _, err := Converge(o, p); err != nil { t.Fatalf("Converge (staged): %v", err) } svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) svcBefore, sockBefore := readFile(t, svcPath), readFile(t, sockPath) f.reset() *logs = nil p.Set = nil res, err := Converge(o, p) if err != nil { t.Fatalf("Converge (nil Set) returned an error: %v — leave-alone is not a failure", err) } if res.Changed { t.Fatal("Changed = true on a nil Set") } if len(f.calls) != 0 { t.Fatalf("systemctl calls on a nil Set: %v", f.calls) } if !bytes.Equal(readFile(t, svcPath), svcBefore) || !bytes.Equal(readFile(t, sockPath), sockBefore) { t.Fatal("a nil Set changed the drop-ins") } if len(*logs) != 1 || !strings.Contains((*logs)[0], "leaving its drop-ins exactly as they are") { t.Fatalf("logs = %v, want exactly one leave-alone line", *logs) } } // TestInvalidStagedManifestRefusesTheRouteAndLeavesDropInsAlone: a // platform-binary rollback can meet an already-staged version whose // capability block this binary does not understand (ensureStaged short- // circuits on a staged artifact and never re-reads its manifest). That falls // the same way as an unreadable manifest — nothing removed — but it also // returns an error, so a version-change caller refuses the version rather // than pointing live at something it cannot materialize. func TestInvalidStagedManifestRefusesTheRouteAndLeavesDropInsAlone(t *testing.T) { o, f, logs, users := testOps(t) p := keymailPlan(t, keymailGrants()) if _, err := Converge(o, p); err != nil { t.Fatalf("Converge (valid): %v", err) } svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) svcBefore, sockBefore := readFile(t, svcPath), readFile(t, sockPath) f.reset() *logs = nil // A Set from a manifest a NEWER binary wrote: v2 is outside 1..Version. // It declares host_role, which is granted — so if the re-validation is // not the FIRST thing after the leave-alone branch, this refusal happens // only at render time, after the pass has already minted a system account // for a version it cannot materialize. p.Set = &Set{V: Version + 1, Command: "relay", Daemon: true, HostRole: true} res, err := Converge(o, p) if err == nil { t.Fatal("Converge accepted a capability block this binary does not speak") } if res.Changed { t.Fatal("Changed = true on a refused route") } if len(f.calls) != 0 { t.Fatalf("systemctl calls on a refused route: %v", f.calls) } if !bytes.Equal(readFile(t, svcPath), svcBefore) || !bytes.Equal(readFile(t, sockPath), sockBefore) { t.Fatal("a refused route's drop-ins were modified") } if len(*users) != 1 { // the first, valid converge — and nothing since t.Fatalf("EnsureUser calls = %v, want only the one from the valid pass", *users) } if !hasLog(*logs, "update the platform binary") { t.Fatalf("logs = %v, want the update-the-platform-binary line", *logs) } } // TestHandWrittenDropInIsNeverOverwritten: a file at a generated path without // the marker is never overwritten and never deleted — reported and skipped — // while the OTHER unit's half still converges. func TestHandWrittenDropInIsNeverOverwritten(t *testing.T) { o, _, logs, _ := testOps(t) p := keymailPlan(t, keymailGrants()) svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) hand := []byte("[Service]\nEnvironment=HAND=written\n") if err := os.MkdirAll(filepath.Dir(svcPath), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(svcPath, hand, 0o644); err != nil { t.Fatal(err) } if _, err := Converge(o, p); err != nil { t.Fatalf("Converge: %v", err) } if got := readFile(t, svcPath); !bytes.Equal(got, hand) { t.Fatalf("hand-written drop-in was modified:\n got: %q\nwant: %q", got, hand) } if !hasLog(*logs, "never overwritten, never removed") { t.Fatalf("logs = %v, want the hand-edited report", *logs) } if got, want := readFile(t, sockPath), readGolden(t, "keymail.socket.conf"); !bytes.Equal(got, want) { t.Fatalf("the other unit's half did not converge:\n got: %q\nwant: %q", got, want) } // And it is not restorable either: this pass never wrote it, so it is not // this pass's to put back — or to delete on a rollback. res, _ := Converge(o, p) if res.Prev.ServiceHad { t.Fatal("Snapshot claimed a hand-written file as its own") } if err := Restore(o, res.Prev); err != nil { t.Fatalf("Restore: %v", err) } if got := readFile(t, svcPath); !bytes.Equal(got, hand) { t.Fatal("Restore deleted a hand-written drop-in") } } // TestUnchangedPassWritesNothing is the determinism promise, asserted against // a DAEMON route with a socket — the case with the most unit state to // repair. Once the box is converged, the hourly pass changes nothing at all. func TestUnchangedPassWritesNothing(t *testing.T) { o, f, logs, _ := testOps(t) p := keymailPlan(t, keymailGrants()) if _, err := Converge(o, p); err != nil { t.Fatalf("Converge (first): %v", err) } svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) svcInfo, err := os.Stat(svcPath) if err != nil { t.Fatal(err) } f.reset() *logs = nil res, err := Converge(o, p) if err != nil { t.Fatalf("Converge (second): %v", err) } if res.Changed { t.Fatal("Changed = true on an unchanged pass") } if len(f.calls) != 0 { t.Fatalf("an unchanged pass made systemctl calls: %v", f.calls) } if len(*logs) != 0 { t.Fatalf("an unchanged pass logged: %v", *logs) } // Not even a rewrite with identical bytes: the mtime is untouched. after, err := os.Stat(svcPath) if err != nil { t.Fatal(err) } if !after.ModTime().Equal(svcInfo.ModTime()) { t.Fatal("the service drop-in was rewritten with identical bytes") } if got, want := readFile(t, sockPath), readGolden(t, "keymail.socket.conf"); !bytes.Equal(got, want) { t.Fatalf("socket drop-in drifted on a no-op pass:\n got: %q\nwant: %q", got, want) } // The mode is part of the contract systemd reads. if perm := svcInfo.Mode().Perm(); perm != 0o644 { t.Fatalf("service drop-in mode = %o, want 0644", perm) } } // TestDaemonPlusHibernateIsRefused: a daemon the platform is asked to keep // running and to stop when idle is a contradiction, and for these apps // hibernation means an MX that sleeps. Refused authoritatively here, with the // operator's exact fix in the line. func TestDaemonPlusHibernateIsRefused(t *testing.T) { o, f, logs, users := testOps(t) p := keymailPlan(t, keymailGrants()) p.Hibernate = true res, err := Converge(o, p) if err == nil { t.Fatal("Converge accepted daemon + hibernate") } if res.Changed { t.Fatal("Changed = true on a refusal") } if len(f.calls) != 0 { t.Fatalf("systemctl calls on a refusal: %v", f.calls) } if len(*users) != 0 { t.Fatalf("a refused plan created users: %v", *users) } svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) mustNotExist(t, svcPath) mustNotExist(t, sockPath) if !hasLog(*logs, "carlos route --host keymail.dev --hibernate=false") { t.Fatalf("logs = %v, want the --hibernate=false fix", *logs) } } // TestHostRoleFlipRefusedWhenTheStateDirIsTheOtherShape is spec §4b's pinned // sharp edge: with DynamicUser=yes the state dir is a SYMLINK into // /var/lib/private, and a static User= needs a real directory. The platform // refuses in place; the operator does the stop → move → start. func TestHostRoleFlipRefusedWhenTheStateDirIsTheOtherShape(t *testing.T) { o, f, logs, users := testOps(t) p := keymailPlan(t, keymailGrants()) if err := os.Symlink(filepath.Join(o.StateRoot, "private", "carlos-app", p.Host), filepath.Join(o.StateRoot, p.Host)); err != nil { t.Fatal(err) } res, err := Converge(o, p) if err == nil { t.Fatal("Converge flipped host_role on over a DynamicUser state dir") } if res.Changed { t.Fatal("Changed = true on a refusal") } if len(f.calls) != 0 { t.Fatalf("systemctl calls on a refusal: %v", f.calls) } if len(*users) != 0 { t.Fatalf("a refused plan created users: %v", *users) } svcPath, _ := DropInPaths(o.EtcDir, p.Host) mustNotExist(t, svcPath) if !hasLog(*logs, "refused in place") { t.Fatalf("logs = %v, want a refused-in-place line naming the operator's move", *logs) } } // The OFF direction of the same flip: once THIS package's drop-in has put the // unit on a static user, dropping host_role would hand the template's // DynamicUser=yes an existing real directory it cannot use. func TestHostRoleFlipOffRefusedWhenTheStateDirIsAlreadyStatic(t *testing.T) { o, _, logs, _ := testOps(t) p := keymailPlan(t, keymailGrants()) if _, err := Converge(o, p); err != nil { t.Fatalf("Converge (host_role on): %v", err) } if err := os.Mkdir(filepath.Join(o.StateRoot, p.Host), 0o700); err != nil { t.Fatal(err) } svcPath, _ := DropInPaths(o.EtcDir, p.Host) before := readFile(t, svcPath) *logs = nil p.Grants = Grants{Ports: map[int]bool{25: true}} // host-role revoked if _, err := Converge(o, p); err == nil { t.Fatal("Converge flipped host_role off over a static state dir") } if got := readFile(t, svcPath); !bytes.Equal(got, before) { t.Fatal("a refused flip rewrote the service drop-in") } if !hasLog(*logs, "refused in place") { t.Fatalf("logs = %v, want a refused-in-place line", *logs) } } // A real state directory that no drop-in of OURS created is not a flip — it // is a legacy hand-provisioned app — and must not freeze an unrelated route's // capabilities forever. func TestPreExistingStateDirDoesNotBlockARouteThatNeverHadHostRole(t *testing.T) { o, _, _, _ := testOps(t) var s Set if err := s.UnmarshalJSON([]byte(`{"v":1,"command":"relay","daemon":true}`)); err != nil { t.Fatal(err) } p := Plan{Host: "legacy.example.com", App: "legacy", Set: &s} if err := os.Mkdir(filepath.Join(o.StateRoot, p.Host), 0o700); err != nil { t.Fatal(err) } if _, err := Converge(o, p); err != nil { t.Fatalf("Converge refused a route over a state dir it never created: %v", err) } svcPath, _ := DropInPaths(o.EtcDir, p.Host) if !strings.Contains(string(readFile(t, svcPath)), "ExecStart=/opt/carlos/live/%i relay") { t.Fatal("the route did not converge") } } // TestRestorePutsThePreviousDropInsBack is the drop-in half of adoptHost's // restart-failure rollback (the symlink half is release.PointLive's). func TestRestorePutsThePreviousDropInsBack(t *testing.T) { o, f, _, _ := testOps(t) p := keymailPlan(t, keymailGrants()) if _, err := Converge(o, p); err != nil { t.Fatalf("Converge A: %v", err) } svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) svcA, sockA := readFile(t, svcPath), readFile(t, sockPath) // Version B: the app dropped `ports` and `children` from its manifest. var b Set if err := b.UnmarshalJSON([]byte(`{"v":1,"command":"relay","daemon":true,"runtime_dir":"keymail","host_role":true}`)); err != nil { t.Fatal(err) } p.Set = &b resB, err := Converge(o, p) if err != nil { t.Fatalf("Converge B: %v", err) } if !resB.Changed { t.Fatal("B did not change anything") } mustNotExist(t, sockPath) f.reset() if err := Restore(o, resB.Prev); err != nil { t.Fatalf("Restore: %v", err) } if got := readFile(t, svcPath); !bytes.Equal(got, svcA) { t.Fatalf("service drop-in after Restore:\n got: %q\n want: %q", got, svcA) } if got := readFile(t, sockPath); !bytes.Equal(got, sockA) { t.Fatalf("socket drop-in after Restore:\n got: %q\n want: %q", got, sockA) } reload := indexOf(f.calls, "daemon-reload") if reload < 0 { t.Fatalf("Restore did not daemon-reload (calls: %v)", f.calls) } // Bytes are not a binding. A restored ListenStream systemd never re-binds // restores nothing — the production MX would stay off :25 after a rollback // — so the socket half goes back through the pinned sequence, after the // reload. stop := indexOf(f.calls, "stop carlos-app@keymail.dev.service") sock := indexOf(f.calls, "restart carlos-app@keymail.dev.socket") start := indexOf(f.calls, "start carlos-app@keymail.dev.service") if stop < 0 || sock < 0 || start < 0 || !(reload < stop && stop < sock && sock < start) { t.Fatalf("Restore did not re-bind the restored listener in order: %v", f.calls) } } // The mirror case, and the one a bytes-only Restore gets exactly as wrong: // version A had NO socket drop-in, B added one and bound :25, the restart // failed, and the rollback must both delete the file and give the port back. // Left bound, it is a public listener held by a binary that never asked for // it and a claim that blocks the next route. func TestRestoreReleasesAPortThatTheRolledBackVersionAdded(t *testing.T) { o, f, _, _ := testOps(t) var a Set if err := a.UnmarshalJSON([]byte(`{"v":1,"command":"relay","daemon":true}`)); err != nil { t.Fatal(err) } p := Plan{Host: "keymail.dev", App: "keymail", Set: &a, Grants: keymailGrants()} if _, err := Converge(o, p); err != nil { t.Fatalf("Converge A: %v", err) } _, sockPath := DropInPaths(o.EtcDir, p.Host) mustNotExist(t, sockPath) p.Set = keymailSet(t) // B declares ports, and :25 is granted resB, err := Converge(o, p) if err != nil { t.Fatalf("Converge B: %v", err) } if resB.Prev.SocketHad { t.Fatal("the snapshot claims a socket drop-in that never existed") } readFile(t, sockPath) // B really did bind it f.reset() // The state a rollback actually happens in: the restart that triggered it // failed, so the service is DOWN. Whether it is supposed to be running is // not something the disk can answer, which is why the snapshot carries it // — without that, the sequence would skip the stop/start pair and leave a // daemon stopped on the version it was just rolled back to. f.active[ServiceUnit(p.Host)] = false if err := Restore(o, resB.Prev); err != nil { t.Fatalf("Restore: %v", err) } mustNotExist(t, sockPath) reload := indexOf(f.calls, "daemon-reload") sock := indexOf(f.calls, "restart carlos-app@keymail.dev.socket") if reload < 0 || sock < 0 || reload > sock { t.Fatalf("Restore removed the drop-in but never released the port: %v", f.calls) } if indexOf(f.calls, "start carlos-app@keymail.dev.service") < 0 || !f.active[ServiceUnit(p.Host)] { t.Fatalf("Restore left the daemon stopped: %v", f.calls) } // And the box no longer counts it as a claim, so the next route can have it. if ports, _ := scanClaims(o.EtcDir, "other.example.com"); ports[25] != "" { t.Fatalf("port 25 is still claimed by %q after the rollback", ports[25]) } } // TestCheckClaimsRefusesAPortHeldByTwoRoutesAndARuntimeDirCollision: box-wide // exclusivity, resolved before anything is written, and deterministic on ties // so a conflict does not flap between passes. func TestCheckClaimsRefusesAPortHeldByTwoRoutesAndARuntimeDirCollision(t *testing.T) { mx := func(host string) Plan { var s Set if err := s.UnmarshalJSON([]byte(`{"v":1,"ports":[{"port":25,"name":"mx"}]}`)); err != nil { t.Fatal(err) } return Plan{Host: host, App: "keymail", Set: &s, Grants: Grants{Ports: map[int]bool{25: true}}} } rd := func(host string) Plan { var s Set if err := s.UnmarshalJSON([]byte(`{"v":1,"runtime_dir":"keymail"}`)); err != nil { t.Fatal(err) } return Plan{Host: host, App: "keymail", Set: &s} } for i := 0; i < 5; i++ { // deterministic across repeated runs, both input orders plans := []Plan{mx("b.example.com"), mx("a.example.com"), rd("d.example.com"), rd("c.example.com")} if i%2 == 1 { plans = []Plan{plans[1], plans[0], plans[3], plans[2]} } refused := CheckClaims(plans) if len(refused) != 2 { t.Fatalf("refused %d routes, want 2: %v", len(refused), refused) } if refused["a.example.com"] != nil || refused["c.example.com"] != nil { t.Fatalf("the lexicographically-first host lost its claim: %v", refused) } if refused["b.example.com"] == nil || !strings.Contains(refused["b.example.com"].Error(), "a.example.com") { t.Fatalf("port conflict not refused against the holder: %v", refused["b.example.com"]) } if refused["d.example.com"] == nil || !strings.Contains(refused["d.example.com"].Error(), "c.example.com") { t.Fatalf("runtime_dir conflict not refused against the holder: %v", refused["d.example.com"]) } } } // An UNGRANTED port claims nothing: nothing materializes for it, so it cannot // lock another route out of the port it was actually granted. func TestCheckClaimsIgnoresUngrantedPortsAndUnresolvedManifests(t *testing.T) { var s Set if err := s.UnmarshalJSON([]byte(`{"v":1,"ports":[{"port":25,"name":"mx"}]}`)); err != nil { t.Fatal(err) } plans := []Plan{ {Host: "a.example.com", App: "keymail", Set: &s}, // declared, not granted {Host: "b.example.com", App: "keymail", Set: &s, Grants: Grants{Ports: map[int]bool{25: true}}}, // granted {Host: "c.example.com", App: "keymail", Set: nil}, // leave-alone claims nothing } if refused := CheckClaims(plans); len(refused) != 0 { t.Fatalf("refused %v, want none", refused) } } // TestPortClaimedByAnotherRoutesDropInIsRefused covers the single-route // version-change path, which never sees the other plans: the claim is read // off what is already written on the box, so a stale drop-in for a route that // has left the registry still holds its port. func TestPortClaimedByAnotherRoutesDropInIsRefused(t *testing.T) { o, f, logs, _ := testOps(t) first := keymailPlan(t, keymailGrants()) if _, err := Converge(o, first); err != nil { t.Fatalf("Converge (first): %v", err) } f.reset() *logs = nil // A second MX-shaped route with NO runtime_dir, so the port claim is the // only thing that can refuse it. var s Set if err := s.UnmarshalJSON([]byte(`{"v":1,"command":"relay","daemon":true,"ports":[{"port":25,"name":"mx"}]}`)); err != nil { t.Fatal(err) } second := Plan{Host: "mx2.example.com", App: "keymail", Set: &s, Grants: Grants{Ports: map[int]bool{25: true}}} res, err := Converge(o, second) if err == nil { t.Fatal("two routes were allowed to claim :25 on one box") } if res.Changed { t.Fatal("Changed = true on a refusal") } if len(f.calls) != 0 { t.Fatalf("systemctl calls on a refusal: %v", f.calls) } svcPath, sockPath := DropInPaths(o.EtcDir, second.Host) mustNotExist(t, svcPath) mustNotExist(t, sockPath) if !hasLog(*logs, "port 25 is already claimed by keymail.dev") { t.Fatalf("logs = %v, want the port-conflict line naming the port and the holder", *logs) } // The same route reconverging keeps its own claim: it is not in conflict // with itself. if _, err := Converge(o, first); err != nil { t.Fatalf("the holder refused its own claim: %v", err) } } // The same disk-read claim check for runtime_dir: the second app to claim // `keymail` is refused rather than handed a directory another app's sockets // live in (spec §7). func TestRuntimeDirClaimedByAnotherRoutesDropInIsRefused(t *testing.T) { o, f, logs, _ := testOps(t) if _, err := Converge(o, keymailPlan(t, keymailGrants())); err != nil { t.Fatalf("Converge (first): %v", err) } f.reset() *logs = nil var s Set if err := s.UnmarshalJSON([]byte(`{"v":1,"command":"serve","runtime_dir":"keymail"}`)); err != nil { t.Fatal(err) } second := Plan{Host: "other.example.com", App: "other", Set: &s} if _, err := Converge(o, second); err == nil { t.Fatal("two routes were allowed to claim /run/keymail on one box") } if len(f.calls) != 0 { t.Fatalf("systemctl calls on a refusal: %v", f.calls) } svcPath, _ := DropInPaths(o.EtcDir, second.Host) mustNotExist(t, svcPath) if !hasLog(*logs, `runtime_dir "keymail" is already claimed by keymail.dev`) { t.Fatalf("logs = %v, want the runtime_dir-conflict line naming the holder", *logs) } } // A hand-written drop-in at another host's generated path still holds its // port: this pass will never rewrite it, so systemd applies it, so the claim // is real. Fail closed — the alternative is two units racing for :25. func TestAHandWrittenDropInStillHoldsItsPortAgainstAnotherRoute(t *testing.T) { o, _, logs, _ := testOps(t) svcPath, _ := DropInPaths(o.EtcDir, "hand.example.com") if err := os.MkdirAll(filepath.Dir(svcPath), 0o755); err != nil { t.Fatal(err) } sockPath := filepath.Join(o.EtcDir, SocketUnit("hand.example.com")+".d", DropInName) if err := os.MkdirAll(filepath.Dir(sockPath), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(sockPath, []byte("[Socket]\nListenStream=25\n"), 0o644); err != nil { t.Fatal(err) } if _, err := Converge(o, keymailPlan(t, keymailGrants())); err == nil { t.Fatal("a route was granted a port a hand-written drop-in already binds") } if !hasLog(*logs, "port 25 is already claimed by hand.example.com") { t.Fatalf("logs = %v, want the port-conflict line", *logs) } } // Changed is true when only a REMOVAL happened: the caller's rollback and the // pass's own daemon-reload both hang off it, and "the box changed" is as true // of a deletion as of a write. func TestChangedIsTrueWhenOnlyARemovalHappened(t *testing.T) { o, f, _, _ := testOps(t) p := keymailPlan(t, keymailGrants()) if _, err := Converge(o, p); err != nil { t.Fatalf("Converge (first): %v", err) } svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) f.reset() // A new version declaring only self-service, directive-free capabilities: // both halves render nil, so both generated files go away. var s Set if err := s.UnmarshalJSON([]byte(`{"v":1,"daemon":true,"data":{"replicate":"platform","dbs":["registry.db"]}}`)); err != nil { t.Fatal(err) } p.Set = &s res, err := Converge(o, p) if err != nil { t.Fatalf("Converge (removal): %v", err) } if !res.Changed { t.Fatal("Changed = false when both drop-ins were removed") } mustNotExist(t, svcPath) mustNotExist(t, sockPath) if indexOf(f.calls, "daemon-reload") < 0 { t.Fatalf("a removal did not daemon-reload: %v", f.calls) } } // The /run existence check is the SECOND runtime_dir guard: the static // reserved list is finite, and a box can have anything under /run that no // list will ever enumerate. func TestRuntimeDirRefusedWhenRunPathAlreadyExists(t *testing.T) { o, f, logs, _ := testOps(t) p := keymailPlan(t, keymailGrants()) if err := os.Mkdir(filepath.Join(o.RunDir, "keymail"), 0o755); err != nil { t.Fatal(err) } if _, err := Converge(o, p); err == nil { t.Fatal("Converge accepted a runtime_dir whose /run path already exists") } if len(f.calls) != 0 { t.Fatalf("systemctl calls on a refusal: %v", f.calls) } svcPath, _ := DropInPaths(o.EtcDir, p.Host) mustNotExist(t, svcPath) if !hasLog(*logs, "is not this route's own claim") { t.Fatalf("logs = %v, want the /run existence refusal", *logs) } } // ...and it must not refuse a route its OWN previous pass created the // directory for, or a converged box would refuse itself every hour. func TestRuntimeDirAcceptsThisRoutesOwnPriorClaim(t *testing.T) { o, _, _, _ := testOps(t) p := keymailPlan(t, keymailGrants()) if _, err := Converge(o, p); err != nil { t.Fatalf("Converge (first): %v", err) } // systemd creates /run/keymail once the unit starts. if err := os.Mkdir(filepath.Join(o.RunDir, "keymail"), 0o755); err != nil { t.Fatal(err) } if _, err := Converge(o, p); err != nil { t.Fatalf("Converge refused the route its own RuntimeDirectory belongs to: %v", err) } } // TestHostWithASeparatorIsRefused: a registry row is trusted at the same // level as live/, but this one becomes a filename under // /etc/systemd/system, written as root (appconfig.syncHost's guard, same // direction). func TestHostWithASeparatorIsRefused(t *testing.T) { o, f, _, _ := testOps(t) for _, host := range []string{"", ".", "..", "../../etc/passwd", "a/b", "sub/../keymail.dev"} { p := keymailPlan(t, keymailGrants()) p.Host = host if _, err := Converge(o, p); err == nil { t.Fatalf("Converge accepted host %q", host) } if err := Restore(o, Snapshot{Host: host}); err == nil { t.Fatalf("Restore accepted host %q", host) } if err := Restart(o, p); err == nil { t.Fatalf("Restart accepted host %q", host) } } if len(f.calls) != 0 { t.Fatalf("a refused host reached systemctl: %v", f.calls) } ents, err := os.ReadDir(o.EtcDir) if err != nil { t.Fatal(err) } if len(ents) != 0 { t.Fatalf("a refused host wrote into EtcDir: %v", ents) } } // TestSocketRestartFailureStillStartsTheService: the service must never be // left stopped because the socket failed to come back — and the error must // still be returned, so the caller's rollback fires. func TestSocketRestartFailureStillStartsTheService(t *testing.T) { o, f, _, _ := testOps(t) p := keymailPlan(t, keymailGrants()) f.fail["restart carlos-app@keymail.dev.socket"] = errors.New("Address already in use") res, err := Converge(o, p) if err == nil { t.Fatal("Converge swallowed a failed socket restart") } if !strings.Contains(err.Error(), "Address already in use") { t.Fatalf("error = %v, want the socket failure", err) } // systemd's listener no longer matches the bytes, so the bytes go back: // otherwise the next pass finds no diff and never retries, and the box // sits on a drop-in that reads "converged" forever. if res.Changed { t.Fatal("Changed = true, but the failed pass was rolled back") } svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) mustNotExist(t, svcPath) mustNotExist(t, sockPath) stop := indexOf(f.calls, "stop carlos-app@keymail.dev.service") start := indexOf(f.calls, "start carlos-app@keymail.dev.service") if stop < 0 || start < 0 || start < stop { t.Fatalf("the service was left stopped after a failed socket restart: %v", f.calls) } if !f.active[ServiceUnit(p.Host)] { t.Fatalf("the service is not running after the sequence: %v", f.calls) } // The same failure on a NON-daemon route, where no enable/start repair // exists to paper over it: the start can only have come from the sequence // itself. This is the assertion that pins "the start is attempted even // when the socket verb failed" rather than merely observing a start. o2, f2, _, _ := testOps(t) var s Set if err := s.UnmarshalJSON([]byte(`{"v":1,"command":"relay","ports":[{"port":25,"name":"mx"}]}`)); err != nil { t.Fatal(err) } q := Plan{Host: "mx.example.com", App: "keymail", Set: &s, Grants: Grants{Ports: map[int]bool{25: true}}} f2.active[ServiceUnit(q.Host)] = true // running, so the sequence stops it f2.fail["restart carlos-app@mx.example.com.socket"] = errors.New("Address already in use") if _, err := Converge(o2, q); err == nil { t.Fatal("Converge swallowed a failed socket restart on a non-daemon route") } stop = indexOf(f2.calls, "stop carlos-app@mx.example.com.service") start = indexOf(f2.calls, "start carlos-app@mx.example.com.service") if stop < 0 || start < 0 || start < stop { t.Fatalf("a non-daemon service was left stopped after a failed socket restart: %v", f2.calls) } if !f2.active[ServiceUnit(q.Host)] { t.Fatalf("the service is not running after the sequence: %v", f2.calls) } } // EnsureUser failing refuses the WHOLE plan: writing User=carlos-keymail for // a user that does not exist gives systemd a unit that fails to start, which // Restart=always then spins. func TestEnsureUserFailureRefusesThePlan(t *testing.T) { o, f, logs, _ := testOps(t) o.EnsureUser = func(string) error { return errors.New("useradd: no free uid") } p := keymailPlan(t, keymailGrants()) if _, err := Converge(o, p); err == nil { t.Fatal("Converge wrote a User= line for a user it could not create") } svcPath, sockPath := DropInPaths(o.EtcDir, p.Host) mustNotExist(t, svcPath) mustNotExist(t, sockPath) if len(f.calls) != 0 { t.Fatalf("systemctl calls after a user-creation failure: %v", f.calls) } if !hasLog(*logs, "refusing the whole plan") { t.Fatalf("logs = %v, want the refusal", *logs) } } // The user is created only when host_role is BOTH declared and granted — an // ungranted declaration must not mint a system account. func TestUngrantedHostRoleCreatesNoUser(t *testing.T) { o, _, _, users := testOps(t) p := keymailPlan(t, Grants{}) if _, err := Converge(o, p); err != nil { t.Fatalf("Converge: %v", err) } if len(*users) != 0 { t.Fatalf("EnsureUser called for an ungranted host_role: %v", *users) } } // A .d path that is a FILE rather than a directory is an honest error, not a // panic and not a silent skip. func TestDropInDirectoryThatIsAFileIsAnError(t *testing.T) { o, _, _, _ := testOps(t) p := keymailPlan(t, keymailGrants()) svcPath, _ := DropInPaths(o.EtcDir, p.Host) if err := os.WriteFile(filepath.Dir(svcPath), []byte("not a directory"), 0o644); err != nil { t.Fatal(err) } if _, err := Converge(o, p); err == nil { t.Fatal("Converge succeeded with a file where the .d directory belongs") } } // A non-daemon route's restart stays the caller's try-restart, byte- // identically to today: converge writes the unit text and starts nothing. func TestNonDaemonRouteIsNeitherEnabledNorStarted(t *testing.T) { o, f, _, _ := testOps(t) var s Set if err := s.UnmarshalJSON([]byte(`{"v":1,"command":"serve","children":true}`)); err != nil { t.Fatal(err) } p := Plan{Host: "app.example.com", App: "app", Set: &s, Hibernate: true} if _, err := Converge(o, p); err != nil { t.Fatalf("Converge: %v", err) } for _, c := range f.calls { if c != "daemon-reload" { t.Fatalf("a hibernating non-daemon route had its unit state changed: %v", f.calls) } } svcPath, _ := DropInPaths(o.EtcDir, p.Host) if !strings.Contains(string(readFile(t, svcPath)), "ExecStart=/opt/carlos/live/%i serve") { t.Fatal("the drop-in was not written for a hibernating route") } } // Drift repair, the other half of the determinism promise: a converged box // whose daemon has been stopped and whose units have been disabled by hand is // put back — this is what the hourly pass is for (spec §4d). func TestStoppedDaemonAndDisabledUnitsAreRepaired(t *testing.T) { o, f, _, _ := testOps(t) p := keymailPlan(t, keymailGrants()) if _, err := Converge(o, p); err != nil { t.Fatalf("Converge (first): %v", err) } f.active[ServiceUnit(p.Host)] = false f.enabled[ServiceUnit(p.Host)] = false f.enabled[SocketUnit(p.Host)] = false f.reset() res, err := Converge(o, p) if err != nil { t.Fatalf("Converge (repair): %v", err) } if res.Changed { t.Fatal("Changed = true, but no byte moved — Changed is about the FILES") } if indexOf(f.calls, "daemon-reload") >= 0 { t.Fatalf("a unit-state repair reloaded the daemon: %v", f.calls) } for _, want := range []string{ "enable carlos-app@keymail.dev.socket", "enable carlos-app@keymail.dev.service", "start carlos-app@keymail.dev.service", } { if indexOf(f.calls, want) < 0 { t.Fatalf("drift not repaired: %q missing from %v", want, f.calls) } } } // Restart's error is REAL — it is what makes adoptHost's rollback fire. func TestRestartReportsItsFailure(t *testing.T) { o, f, _, _ := testOps(t) p := keymailPlan(t, keymailGrants()) f.fail["restart carlos-app@keymail.dev.service"] = errors.New("job failed") if err := Restart(o, p); err == nil { t.Fatal("Restart swallowed a failed restart") } if f.calls[len(f.calls)-1] != "restart carlos-app@keymail.dev.service" { t.Fatalf("Restart ran %v", f.calls) } } // Two plans, one box, no conflict: the second route's drop-ins are written // and the first route's are untouched. func TestASecondRouteOnTheBoxConvergesIndependently(t *testing.T) { o, _, _, _ := testOps(t) first := keymailPlan(t, keymailGrants()) if _, err := Converge(o, first); err != nil { t.Fatalf("Converge (first): %v", err) } svcA, sockA := DropInPaths(o.EtcDir, first.Host) beforeSvc, beforeSock := readFile(t, svcA), readFile(t, sockA) second := Plan{Host: "platform.carlosframework.com", App: "console", Set: consoleSet(t), Grants: consoleGrants()} if _, err := Converge(o, second); err != nil { t.Fatalf("Converge (second): %v", err) } svcB, sockB := DropInPaths(o.EtcDir, second.Host) if got, want := readFile(t, svcB), readGolden(t, "console.service.conf"); !bytes.Equal(got, want) { t.Fatalf("console service drop-in:\n got: %q\n want: %q", got, want) } mustNotExist(t, sockB) // console declares no ports if !bytes.Equal(readFile(t, svcA), beforeSvc) || !bytes.Equal(readFile(t, sockA), beforeSock) { t.Fatal("converging one route changed another's drop-ins") } }