package main import ( "database/sql" "flag" "fmt" "net" "net/http" "os" "path/filepath" "reflect" "strings" "syscall" "testing" "time" ) func TestFlagsAfterPositional(t *testing.T) { fs := flag.NewFlagSet("t", flag.ContinueOnError) socket := fs.String("socket", "", "") all := fs.Bool("all", false, "") // host BEFORE the flags — the natural order that the stdlib parser drops. if err := fs.Parse(reorderFlagsFirst(fs, []string{"chat.example.com", "--socket", "/x.sock", "--all"})); err != nil { t.Fatalf("parse: %v", err) } if *socket != "/x.sock" { t.Errorf("socket = %q, want /x.sock", *socket) } if !*all { t.Error("all = false, want true") } if fs.Arg(0) != "chat.example.com" { t.Errorf("Arg(0) = %q, want chat.example.com", fs.Arg(0)) } } func TestReorderDoesNotPullFlagAsValue(t *testing.T) { fs := flag.NewFlagSet("t", flag.ContinueOnError) fs.String("socket", "", "") fs.String("router-unit", "", "") // --socket is given no value (next token is another flag); reorder must not // swallow --router-unit as --socket's value. out := reorderFlagsFirst(fs, []string{"host", "--socket", "--router-unit", "foo"}) if !strings.Contains(strings.Join(out, " "), "--router-unit foo") { t.Errorf("reorder paired a flag as a value: %v", out) } } func TestValidHost(t *testing.T) { ok := []string{"chat.example.com", "rtr-test.example.com", "a.b", "x1.y2.example.com"} bad := []string{"", "nodots", "-bad.example.com", "bad-.example.com", "has space.com", "Chat.Example.com", "a/b.com", "a..b"} for _, h := range ok { if !validHost(h) { t.Errorf("validHost(%q) = false, want true", h) } } for _, h := range bad { if validHost(h) { t.Errorf("validHost(%q) = true, want false", h) } } } func TestCheckSocketLen(t *testing.T) { if err := checkSocketLen("/run/lchat/chat.example.com.sock"); err != nil { t.Errorf("short path: unexpected error %v", err) } long := "/run/lchat/" + strings.Repeat("a", 120) + ".sock" if err := checkSocketLen(long); err == nil { t.Error("long path: expected error, got nil") } } func TestEnvFileRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "chat.example.com.env") in := map[string]string{"LCHAT_ADMIN_TOKEN": "tok-123", "LCHAT_HOME_ORIGIN": "https://home.example.com"} if err := writeEnvFile(path, in); err != nil { t.Fatalf("write: %v", err) } fi, err := os.Stat(path) if err != nil { t.Fatal(err) } if fi.Mode().Perm() != 0o600 { t.Errorf("perms = %v, want -rw-------", fi.Mode().Perm()) } out, err := readEnvFile(path) if err != nil { t.Fatalf("read: %v", err) } for k, v := range in { if out[k] != v { t.Errorf("readEnvFile[%q] = %q, want %q", k, out[k], v) } } } func TestRegistryGet(t *testing.T) { reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } if got, err := reg.get("missing.example.com"); err != nil || got != nil { t.Fatalf("get(missing) = %v, %v; want nil, nil", got, err) } if err := reg.upsert(Route{Host: "chat.example.com", Kind: "instance", Socket: "/run/lchat/chat.example.com.sock", Status: "ready"}); err != nil { t.Fatal(err) } got, err := reg.get("chat.example.com") if err != nil || got == nil { t.Fatalf("get(existing) = %v, %v", got, err) } if got.Socket != "/run/lchat/chat.example.com.sock" || got.Status != "ready" || got.Kind != "instance" { t.Errorf("get returned %+v", *got) } // delete stops routing the host. if err := reg.delete("chat.example.com"); err != nil { t.Fatalf("delete: %v", err) } if got, err := reg.get("chat.example.com"); err != nil || got != nil { t.Fatalf("after delete, get = %v, %v; want nil, nil", got, err) } } // The register/materialize seam must be pure refactor: running the two named // steps by hand ends in exactly the state `lchat add-instance` produces. func TestRegisterThenMaterializeMatchesAddInstance(t *testing.T) { reg := controlTestEnv(t) composed, err := addInstance(reg, addInstanceOpts{Host: "composed.example.com"}) if err != nil { t.Fatalf("addInstance: %v", err) } route, token, err := registerInstance(reg, addInstanceOpts{Host: "split.example.com"}) if err != nil { t.Fatalf("register: %v", err) } // After register alone the space exists but nothing runs: a "creating" row, // no env file (the token's only persistent home is written by materialize). if rt, err := reg.get("split.example.com"); err != nil || rt == nil || rt.Status != "creating" { t.Fatalf("after register, route = %+v, err %v; want status creating", rt, err) } if _, err := os.Stat(filepath.Join(instanceDataDir, "split.example.com.env")); !os.IsNotExist(err) { t.Error("register wrote the env file — that's materialize's job") } split, err := materializeInstance(reg, route, token, addInstanceOpts{Host: "split.example.com"}) if err != nil { t.Fatalf("materialize: %v", err) } // Both paths yield a ready route with the same host-independent shape… for _, host := range []string{"composed.example.com", "split.example.com"} { res := composed if host == "split.example.com" { res = split } rt, err := reg.get(host) if err != nil || rt == nil { t.Fatalf("get(%s): %+v, %v", host, rt, err) } if rt.Status != "ready" || rt.Kind != "instance" || rt.Version == "" { t.Errorf("%s: route = %+v, want ready instance with a version", host, *rt) } // …an env file holding the admin token the result reports… env, err := readEnvFile(filepath.Join(instanceDataDir, host+".env")) if err != nil { t.Fatalf("read env %s: %v", host, err) } if env["LCHAT_ADMIN_TOKEN"] != res.AdminToken { t.Errorf("%s: env token != result token", host) } // …and a setup link minted against the started instance. if !strings.HasPrefix(res.SetupLink, "https://"+host+"/join/setup#") { t.Errorf("%s: setup link = %q", host, res.SetupLink) } } // Field-for-field, the two routes differ only in what derives from the host // (paths, sockets) and the create timestamp. a, _ := reg.get("composed.example.com") b, _ := reg.get("split.example.com") for _, rt := range []*Route{a, b} { rt.Host, rt.Socket, rt.DBPath, rt.CreatedAt = "", "", "", 0 } if !reflect.DeepEqual(*a, *b) { t.Errorf("routes diverge:\n composed %+v\n split %+v", *a, *b) } } // register() carries the geo/co-op placement seams onto the row, and // materialize preserves them through its ready upsert. Nothing else may key on // them yet — this test pins only the plumbing. // `lchat add-instance --home-region ` records the region on the new route // (multi-edge Phase 1): the flag flows through addInstanceOpts.HomeRegion into // the registry row the mesh later places by, with surrounding whitespace // trimmed. Exercises the real CLI entry point (runAddInstance) over the shared // controlTestEnv fakes. func TestAddInstanceHomeRegionFlag(t *testing.T) { _ = controlTestEnv(t) // fakes: systemd (binds the socket), self-stage, dirs, public-ready regPath := filepath.Join(t.TempDir(), "registry.db") runAddInstance([]string{"region.example.com", "--home-region", " ie ", "--registry", regPath}) reg, err := openRegistry(regPath) if err != nil { t.Fatal(err) } rt, err := reg.get("region.example.com") if err != nil || rt == nil { t.Fatalf("route = %+v, err %v", rt, err) } if rt.HomeRegion != "ie" { t.Errorf("home_region = %q, want ie (flag value, trimmed)", rt.HomeRegion) } } func TestRegisterAcceptsRegionProvider(t *testing.T) { reg := controlTestEnv(t) opts := addInstanceOpts{Host: "geo.example.com", HomeRegion: "eu-central-1", Provider: "hetzner"} route, token, err := registerInstance(reg, opts) if err != nil { t.Fatalf("register: %v", err) } if rt, _ := reg.get("geo.example.com"); rt == nil || rt.HomeRegion != "eu-central-1" || rt.Provider != "hetzner" { t.Fatalf("after register, route = %+v, want home_region/provider recorded", rt) } if _, err := materializeInstance(reg, route, token, opts); err != nil { t.Fatalf("materialize: %v", err) } rt, _ := reg.get("geo.example.com") if rt == nil || rt.Status != "ready" || rt.HomeRegion != "eu-central-1" || rt.Provider != "hetzner" { t.Errorf("after materialize, route = %+v, want placement preserved on the ready row", rt) } } // A registry created before the geo/co-op columns existed must gain them on // open, with existing rows reading as the implicit ""/"" placement — live boxes // carry such DBs, so the migration has to be safe over real data. func TestRegistryMigratesGeoProviderColumns(t *testing.T) { path := filepath.Join(t.TempDir(), "reg.db") // Build the pre-seam schema by hand and seed a row, as a live box would have. old, err := sql.Open("sqlite", path) if err != nil { t.Fatal(err) } if _, err := old.Exec(` CREATE TABLE routes ( host TEXT PRIMARY KEY, kind TEXT NOT NULL, socket TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'ready', last_error TEXT NOT NULL DEFAULT '', db_path TEXT NOT NULL DEFAULT '', version TEXT NOT NULL DEFAULT '', prev_version TEXT NOT NULL DEFAULT '', app_store_url TEXT NOT NULL DEFAULT '', home_origin TEXT NOT NULL DEFAULT '', push_relay_url TEXT NOT NULL DEFAULT '', push_relay_token TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL ); INSERT INTO routes (host, kind, socket, created_at) VALUES ('old.example.com', 'instance', '/run/lchat/old.sock', 1700000000); `); err != nil { t.Fatal(err) } if err := old.Close(); err != nil { t.Fatal(err) } reg, err := openRegistry(path) if err != nil { t.Fatalf("open over pre-seam registry: %v", err) } rt, err := reg.get("old.example.com") if err != nil || rt == nil { t.Fatalf("get pre-existing row: %+v, %v", rt, err) } if rt.HomeRegion != "" || rt.Provider != "" { t.Errorf("pre-existing row: home_region=%q provider=%q, want the implicit \"\"/\"\"", rt.HomeRegion, rt.Provider) } // The pre-seam schema predates next_due too — the same open must add it. if rt.NextDue != 0 { t.Errorf("pre-existing row: next_due=%d, want 0", rt.NextDue) } // New values round-trip through the migrated table. rt.HomeRegion, rt.Provider = "us-east-1", "aws" if err := reg.upsert(*rt); err != nil { t.Fatalf("upsert with placement: %v", err) } got, err := reg.get("old.example.com") if err != nil || got == nil || got.HomeRegion != "us-east-1" || got.Provider != "aws" { t.Errorf("round-trip = %+v, %v", got, err) } // migrate is idempotent — reopening must not trip on the added columns. if _, err := openRegistry(path); err != nil { t.Errorf("reopen after migration: %v", err) } } // The due-time index's registry half (activator.go's actDueStore): setNextDue // is a narrow write, dueInstances returns exactly the ready instances whose // time has arrived, and a route rewrite (deploy's read-modify-write upsert) // must not clobber a sleeping host's due time. func TestRegistryDueTimeIndex(t *testing.T) { reg, err := openRegistry(filepath.Join(t.TempDir(), "reg.db")) if err != nil { t.Fatal(err) } seed := []Route{ {Host: "sleepy.example.com", Kind: "instance", Socket: "/run/a.sock", Hibernate: true}, {Host: "quiet.example.com", Kind: "instance", Socket: "/run/b.sock", Hibernate: true}, {Host: "svc.example.com", Kind: "service", Socket: "/run/c.sock"}, {Host: "broken.example.com", Kind: "instance", Socket: "/run/d.sock", Status: "failed", Hibernate: true}, {Host: "manual.example.com", Kind: "instance", Socket: "/run/e.sock"}, // systemd-managed (hibernate off) } for _, rt := range seed { if err := reg.upsert(rt); err != nil { t.Fatalf("upsert %s: %v", rt.Host, err) } } const now = int64(1_700_000_000) for host, due := range map[string]int64{ "sleepy.example.com": now - 5, // due "svc.example.com": now - 5, // due but a service — never woken "broken.example.com": now - 5, // due but not ready — never woken "manual.example.com": now - 5, // due but NOT hibernating — always on, never due-woken // quiet: next_due stays 0 — nothing scheduled, never due-woken } { if err := reg.setNextDue(host, due); err != nil { t.Fatalf("setNextDue %s: %v", host, err) } } due, err := reg.dueInstances(now) if err != nil { t.Fatalf("dueInstances: %v", err) } if len(due) != 1 || due[0].Host != "sleepy.example.com" { t.Fatalf("dueInstances = %+v, want just sleepy.example.com", due) } // Not yet due: a future timestamp stays out of the set. if err := reg.setNextDue("sleepy.example.com", now+60); err != nil { t.Fatal(err) } if due, _ := reg.dueInstances(now); len(due) != 0 { t.Fatalf("future due time returned early: %+v", due) } // Read-modify-write (how deploy/rollback update a route) carries next_due. rt, err := reg.get("sleepy.example.com") if err != nil || rt == nil { t.Fatalf("get: %+v, %v", rt, err) } rt.Version = "v2" if err := reg.upsert(*rt); err != nil { t.Fatalf("upsert rewrite: %v", err) } if got, _ := reg.get("sleepy.example.com"); got.NextDue != now+60 { t.Fatalf("route rewrite clobbered next_due: %d, want %d", got.NextDue, now+60) } // The narrow setters stay orthogonal: the rewrite above carried the // hibernate flag, and setHibernate must not touch next_due. if got, _ := reg.get("sleepy.example.com"); !got.Hibernate { t.Fatal("route rewrite dropped the hibernate flag") } if err := reg.setHibernate("sleepy.example.com", false); err != nil { t.Fatalf("setHibernate: %v", err) } if got, _ := reg.get("sleepy.example.com"); got.Hibernate || got.NextDue != now+60 { t.Fatalf("setHibernate clobbered the route: hibernate=%v next_due=%d", got.Hibernate, got.NextDue) } // An unknown host is a quiet no-op — it can't be woken anyway. if err := reg.setNextDue("ghost.example.com", now); err != nil { t.Fatalf("setNextDue unknown host: %v", err) } } // A registry created before the hibernate column existed must gain it on open, // with existing rows reading as hibernate=off — the live fleet's registry is // exactly such a DB, so the migration must default every real instance to the // classic systemd shape (the whole point of the per-host opt-in). func TestRegistryMigratesHibernateColumn(t *testing.T) { path := filepath.Join(t.TempDir(), "reg.db") // The schema as of the due-time index — everything current EXCEPT hibernate. old, err := sql.Open("sqlite", path) if err != nil { t.Fatal(err) } if _, err := old.Exec(` CREATE TABLE routes ( host TEXT PRIMARY KEY, kind TEXT NOT NULL, socket TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'ready', last_error TEXT NOT NULL DEFAULT '', db_path TEXT NOT NULL DEFAULT '', version TEXT NOT NULL DEFAULT '', prev_version TEXT NOT NULL DEFAULT '', app_store_url TEXT NOT NULL DEFAULT '', home_origin TEXT NOT NULL DEFAULT '', push_relay_url TEXT NOT NULL DEFAULT '', push_relay_token TEXT NOT NULL DEFAULT '', home_region TEXT NOT NULL DEFAULT '', provider TEXT NOT NULL DEFAULT '', next_due INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL ); INSERT INTO routes (host, kind, socket, created_at) VALUES ('old.example.com', 'instance', '/run/lchat/old.sock', 1700000000); `); err != nil { t.Fatal(err) } if err := old.Close(); err != nil { t.Fatal(err) } reg, err := openRegistry(path) if err != nil { t.Fatalf("open over pre-hibernate registry: %v", err) } rt, err := reg.get("old.example.com") if err != nil || rt == nil { t.Fatalf("get pre-existing row: %+v, %v", rt, err) } if rt.Hibernate { t.Error("pre-existing row: hibernate=true, want the systemd-managed default (false)") } // The flag round-trips through both the narrow setter and a full upsert. if err := reg.setHibernate("old.example.com", true); err != nil { t.Fatalf("setHibernate: %v", err) } if got, _ := reg.get("old.example.com"); got == nil || !got.Hibernate { t.Fatalf("after setHibernate, route = %+v, want hibernate on", got) } rt, _ = reg.get("old.example.com") rt.Hibernate = false if err := reg.upsert(*rt); err != nil { t.Fatalf("upsert with hibernate: %v", err) } if got, _ := reg.get("old.example.com"); got == nil || got.Hibernate { t.Fatalf("after upsert, route = %+v, want hibernate off", got) } // migrate is idempotent — reopening must not trip on the added column. if _, err := openRegistry(path); err != nil { t.Errorf("reopen after migration: %v", err) } } // A registry created before the replication column existed must gain it on // open, with existing rows reading as "" — the stream class, i.e. exactly the // pre-ladder behavior, so a deploy over the live fleet's registry changes no // host's durability until an operator (or billing) says so. func TestRegistryMigratesReplicationColumn(t *testing.T) { path := filepath.Join(t.TempDir(), "reg.db") // The schema as of the hibernate flag — everything current EXCEPT replication. old, err := sql.Open("sqlite", path) if err != nil { t.Fatal(err) } if _, err := old.Exec(` CREATE TABLE routes ( host TEXT PRIMARY KEY, kind TEXT NOT NULL, socket TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'ready', last_error TEXT NOT NULL DEFAULT '', db_path TEXT NOT NULL DEFAULT '', version TEXT NOT NULL DEFAULT '', prev_version TEXT NOT NULL DEFAULT '', app_store_url TEXT NOT NULL DEFAULT '', home_origin TEXT NOT NULL DEFAULT '', push_relay_url TEXT NOT NULL DEFAULT '', push_relay_token TEXT NOT NULL DEFAULT '', home_region TEXT NOT NULL DEFAULT '', provider TEXT NOT NULL DEFAULT '', hibernate INTEGER NOT NULL DEFAULT 0, next_due INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL ); INSERT INTO routes (host, kind, socket, hibernate, created_at) VALUES ('old.example.com', 'instance', '/run/lchat/old.sock', 1, 1700000000); `); err != nil { t.Fatal(err) } if err := old.Close(); err != nil { t.Fatal(err) } reg, err := openRegistry(path) if err != nil { t.Fatalf("open over pre-replication registry: %v", err) } rt, err := reg.get("old.example.com") if err != nil || rt == nil { t.Fatalf("get pre-existing row: %+v, %v", rt, err) } if rt.Replication != "" { t.Errorf("pre-existing row: replication=%q, want \"\" (the stream default)", rt.Replication) } // The class round-trips through the narrow setter, an upsert carries it, // and setReplication stays orthogonal to the rest of the route. if err := reg.setReplication("old.example.com", replicationDaily); err != nil { t.Fatalf("setReplication: %v", err) } if got, _ := reg.get("old.example.com"); got == nil || got.Replication != replicationDaily || !got.Hibernate { t.Fatalf("after setReplication, route = %+v, want daily + hibernate intact", got) } rt, _ = reg.get("old.example.com") rt.Version = "v2" if err := reg.upsert(*rt); err != nil { t.Fatalf("upsert rewrite: %v", err) } if got, _ := reg.get("old.example.com"); got == nil || got.Replication != replicationDaily { t.Fatalf("route rewrite dropped the class: %+v", got) } // Garbage never lands: no write path may store a class the wake path would // silently read as stream. if err := reg.setReplication("old.example.com", "hourly"); err == nil { t.Error("setReplication accepted an unknown class") } if got, _ := reg.get("old.example.com"); got == nil || got.Replication != replicationDaily { t.Fatalf("rejected set still changed the row: %+v", got) } // An unknown host is a quiet no-op, matching setHibernate/setNextDue. if err := reg.setReplication("ghost.example.com", replicationBatched); err != nil { t.Fatalf("setReplication unknown host: %v", err) } // migrate is idempotent — reopening must not trip on the added column. if _, err := openRegistry(path); err != nil { t.Errorf("reopen after migration: %v", err) } } // The class→interval mapping is the ladder's contract: stream (and the unset // "" every pre-ladder registry row reads as) keeps litestream's default, and // the coarser classes stretch it — never the other way around. func TestReplicationSyncInterval(t *testing.T) { for class, want := range map[string]time.Duration{ "": 0, replicationStream: 0, replicationBatched: 5 * time.Minute, replicationDaily: 24 * time.Hour, } { if got := replicationSyncInterval(class); got != want { t.Errorf("replicationSyncInterval(%q) = %v, want %v", class, got, want) } } for _, ok := range []string{"", "stream", "batched", "daily"} { if !validReplication(ok) { t.Errorf("validReplication(%q) = false, want true", ok) } } for _, bad := range []string{"hourly", "Stream", "off"} { if validReplication(bad) { t.Errorf("validReplication(%q) = true, want false", bad) } } } // `add-instance --hibernate` produces a registered, unmaterialized-but- // spawnable route: everything the activator's spawn needs at first wake (the // env file with the admin token, a staged binary behind the live symlink, a // ready+flagged route with the router reloaded) and nothing systemd (no unit // enabled, no add-time health check, no DB — the first wake boots fresh). The // public-readiness probe doubles as that first wake — simulated here by // binding the socket when it fires — and the setup-invite mint rides the // woken socket, so the space is claimable the moment add-instance returns. func TestAddInstanceHibernate(t *testing.T) { reg := controlTestEnv(t) const host = "hib.example.com" // Replace the harness's systemd fake with a pure recorder: the hibernating // path must never enable a unit, so nothing here binds the socket for it. var calls []string systemctl = func(args ...string) error { calls = append(calls, strings.Join(args, " ")) return nil } // Play the router+activator: the readiness probe is the first request, and // the wake it triggers binds the instance socket — answering the health // check and the invite mint like a freshly spawned child would. confirmPublicReady = func(h, _ string) { calls = append(calls, "confirm "+h) ln, err := net.Listen("unix", filepath.Join(instanceSocketDir, h+".sock")) if err != nil { t.Errorf("bind woken socket: %v", err) return } mux := http.NewServeMux() mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) }) mux.HandleFunc("POST /api/admin/key-invites", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{"ok": true}) }) srv := &http.Server{Handler: mux} go srv.Serve(ln) t.Cleanup(func() { srv.Close() }) } res, err := addInstance(reg, addInstanceOpts{Host: host, Hibernate: true}) if err != nil { t.Fatalf("addInstance --hibernate: %v", err) } // Registered: ready, flagged, under version management. rt, err := reg.get(host) if err != nil || rt == nil { t.Fatalf("route = %+v, %v", rt, err) } if rt.Status != "ready" || !rt.Hibernate || rt.Version == "" { t.Errorf("route = %+v, want a ready hibernating instance with a version", *rt) } // Spawnable: the env file holds the admin token (the spawn's env layering // and every saved admin link depend on it)… env, err := readEnvFile(filepath.Join(instanceDataDir, host+".env")) if err != nil { t.Fatalf("read env: %v", err) } if env["LCHAT_ADMIN_TOKEN"] != res.AdminToken { t.Error("env token != result token") } // …and the live symlink resolves to a staged binary (the spawn execs it). if ver, err := liveVersion(host); err != nil || ver == "" { t.Errorf("live symlink not staged: ver=%q err=%v", ver, err) } // Unmaterialized: no unit, no DB; the router WAS reloaded (the route must // serve before any request can wake it) and only then probed. reloadIdx, confirmIdx := -1, -1 for i, c := range calls { if strings.HasPrefix(c, "enable ") { t.Errorf("a systemd unit was touched: %q", c) } if c == "reload lchat-router" { reloadIdx = i } if c == "confirm "+host { confirmIdx = i } } if reloadIdx == -1 || confirmIdx == -1 || confirmIdx < reloadIdx { t.Errorf("want router reload before the wake probe, got calls %v", calls) } if _, err := os.Stat(filepath.Join(instanceDataDir, host+".db")); !os.IsNotExist(err) { t.Error("a DB was created — the first wake should boot fresh") } // Claimable: the setup link was minted through the woken socket. if !strings.HasPrefix(res.SetupLink, "https://"+host+"/join/setup#") { t.Errorf("setup link = %q (err %v)", res.SetupLink, res.SetupErr) } } // The flip-on ordering (hibernateOn): flag first — it's inert until a reload — // then the unit stops BEFORE the router reloads (a wake must be able to bind // the socket), then the warm request closes the gap. A crash at any seam // leaves the host serveable. func TestHibernateOnOrdering(t *testing.T) { reg := controlTestEnv(t) const host = "flip-on.example.com" if _, err := addInstance(reg, addInstanceOpts{Host: host}); err != nil { t.Fatalf("addInstance: %v", err) } var calls []string flagAtDisable, flagAtReload := false, false systemctl = func(args ...string) error { call := strings.Join(args, " ") calls = append(calls, call) rt, _ := reg.get(host) switch call { case "disable --now lchat@" + host: flagAtDisable = rt.Hibernate case "reload lchat-router": flagAtReload = rt.Hibernate } return nil } confirmPublicReady = func(h, _ string) { calls = append(calls, "confirm "+h) } rt, _ := reg.get(host) if err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { t.Fatalf("hibernateOn: %v", err) } want := []string{"disable --now lchat@" + host, "reload lchat-router", "confirm " + host} if !reflect.DeepEqual(calls, want) { t.Errorf("flip-on sequence = %v, want %v", calls, want) } if !flagAtDisable || !flagAtReload { t.Errorf("flag not set before unit stop/reload (disable=%v reload=%v) — a crash mid-flip would strand the ordering", flagAtDisable, flagAtReload) } if got, _ := reg.get(host); got == nil || !got.Hibernate { t.Error("flag not set after flip-on") } } // The flip-off ordering (hibernateOff): flag cleared, router reloaded (which // makes the activator release the host), and the unit started ONLY after the // activator's child has let the socket go — verified by a fake child that // holds the socket until the reload lands, releasing it a beat later like the // real (asynchronous) reconcile. func TestHibernateOffOrdering(t *testing.T) { reg := controlTestEnv(t) const host = "flip-off.example.com" sock := filepath.Join(instanceSocketDir, host+".sock") if err := reg.upsert(Route{Host: host, Kind: "instance", Socket: sock, Status: "ready", Hibernate: true}); err != nil { t.Fatal(err) } // The fake activator child, awake and holding the socket. child, err := net.Listen("unix", sock) if err != nil { t.Fatal(err) } childSrv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })} go childSrv.Serve(child) t.Cleanup(func() { childSrv.Close() }) var calls []string systemctl = func(args ...string) error { call := strings.Join(args, " ") calls = append(calls, call) switch { case call == "reload lchat-router": // The reconcile's release is asynchronous inside the router // process — model that: the child lets go a beat after the reload. go func() { time.Sleep(300 * time.Millisecond) childSrv.Close() }() case call == "enable --now lchat@"+host: // The ordering crux: by unit-start time the child must be gone… if c, err := net.DialTimeout("unix", sock, 100*time.Millisecond); err == nil { c.Close() t.Error("unit started while the activator's child still held the socket") } // …and the flag off, so systemd unambiguously owns the host. if rt, _ := reg.get(host); rt.Hibernate { t.Error("unit started with the hibernate flag still set") } // Bind like a started unit would (clearing the stale socket file // first, as the instance's listen() does), so waitHealthy passes. _ = os.Remove(sock) ln, err := net.Listen("unix", sock) if err != nil { return err } srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })} go srv.Serve(ln) t.Cleanup(func() { srv.Close() }) } return nil } rt, _ := reg.get(host) if err := hibernateOff(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { t.Fatalf("hibernateOff: %v", err) } want := []string{"reload lchat-router", "enable --now lchat@" + host} if !reflect.DeepEqual(calls, want) { t.Errorf("flip-off sequence = %v, want %v", calls, want) } if got, _ := reg.get(host); got == nil || got.Hibernate { t.Error("flag still set after flip-off") } } // A host with a bot sidecar staged (its -bot.env exists) hands the sidecar // over at the hibernate flips: --on disables the always-on unit BEFORE the // router reload (the activator co-spawns from then on; a systemd sidecar and // a co-spawned one would double every bot reply), and --off re-enables it // LAST — only after the instance unit is healthy, add-bot's "instance first" // bootstrap order. A host without one (the two ordering tests above) never // touches the bot unit. func TestHibernateFlipHandsSidecarUnitOver(t *testing.T) { reg := controlTestEnv(t) const host = "flip-bots.example.com" sock := filepath.Join(instanceSocketDir, host+".sock") if err := reg.upsert(Route{Host: host, Kind: "instance", Socket: sock, Status: "ready"}); err != nil { t.Fatal(err) } // The instance env (hibernateOn chowns it) and the sidecar staging — // exactly what add-instance + add-bot leave on disk. if err := os.WriteFile(filepath.Join(instanceDataDir, host+".env"), []byte("LCHAT_ADMIN_TOKEN=t\n"), 0o600); err != nil { t.Fatal(err) } if err := os.WriteFile(botEnvPath(host), []byte("LCHAT_BASE=https://"+host+"\n"), 0o600); err != nil { t.Fatal(err) } if err := os.MkdirAll(botSessionDir(host), 0o750); err != nil { t.Fatal(err) } var calls []string systemctl = func(args ...string) error { call := strings.Join(args, " ") calls = append(calls, call) if call == "enable --now lchat@"+host { // Bind like a started unit so hibernateOff's health check passes. ln, err := net.Listen("unix", sock) if err != nil { return err } srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })} go srv.Serve(ln) t.Cleanup(func() { srv.Close() }) } return nil } rt, _ := reg.get(host) if err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { t.Fatalf("hibernateOn: %v", err) } wantOn := []string{"disable --now lchat@" + host, "disable --now lchat-bot@" + host, "reload lchat-router"} if !reflect.DeepEqual(calls, wantOn) { t.Errorf("flip-on sequence = %v, want %v", calls, wantOn) } calls = nil rt, _ = reg.get(host) if err := hibernateOff(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { t.Fatalf("hibernateOff: %v", err) } wantOff := []string{"reload lchat-router", "enable --now lchat@" + host, "enable --now lchat-bot@" + host} if !reflect.DeepEqual(calls, wantOff) { t.Errorf("flip-off sequence = %v, want %v", calls, wantOff) } } // waitSocketQuiet is the flip-off's release observation: quiet means no // listener (dial failure), whether or not the socket file lingers. func TestWaitSocketQuiet(t *testing.T) { sock := tmpSock(t) // No listener: quiet immediately. if err := waitSocketQuiet(sock, time.Second); err != nil { t.Errorf("no listener: %v, want quiet", err) } // A held listener: not quiet — the bounded wait must report it. ln, err := net.Listen("unix", sock) if err != nil { t.Fatal(err) } defer ln.Close() if err := waitSocketQuiet(sock, 400*time.Millisecond); err == nil { t.Error("live listener reported quiet — the flip-off would start a unit against a live child") } } // matchOwner hands a root-written env file to the service user (the owner of // /data) so the router's activator can read it at wake time — the bug the // production canary's first wake hit. Unprivileged test: same-owner is a // clean no-op and the owners really do match afterwards. func TestMatchOwner(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "x.env") if err := os.WriteFile(path, []byte("A=b\n"), 0o600); err != nil { t.Fatal(err) } if err := matchOwner(path, dir); err != nil { t.Fatalf("matchOwner: %v", err) } ds, _ := os.Stat(dir) fs, _ := os.Stat(path) if ds.Sys().(*syscall.Stat_t).Uid != fs.Sys().(*syscall.Stat_t).Uid { t.Fatalf("owner mismatch after matchOwner") } } // The flip-window guard: while the box-wide litestream config still tracks a // host's DB, flipping it on must stop before touching the unit — two // replicators share one meta dir and corrupt it (hit live on the first real // flip). The flag must already be SET when the guard fires, because the // operator's fix (re-running the backup script, which excludes hibernate=1 // hosts) depends on it — and the flag is inert until the reload the guard // prevents. func TestHibernateOnRefusesWhileBoxLitestreamTracks(t *testing.T) { reg := controlTestEnv(t) const host = "flip-guard.example.com" if _, err := addInstance(reg, addInstanceOpts{Host: host}); err != nil { t.Fatalf("addInstance: %v", err) } conf := filepath.Join(t.TempDir(), "litestream.yml") body := "dbs:\n - path: " + filepath.Join(instanceDataDir, host+".db") + "\n" if err := os.WriteFile(conf, []byte(body), 0o644); err != nil { t.Fatal(err) } t.Setenv("LCHAT_LITESTREAM_CONFIG", conf) var calls []string systemctl = func(args ...string) error { calls = append(calls, strings.Join(args, " ")) return nil } confirmPublicReady = func(string, string) { t.Fatal("must not warm while guarded") } rt, _ := reg.get(host) err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router") if err == nil || !strings.Contains(err.Error(), "fleet-backup.sh") { t.Fatalf("want the re-run-fleet-backup refusal, got %v", err) } if len(calls) != 0 { t.Fatalf("guarded flip touched systemd: %v", calls) } if got, _ := reg.get(host); !got.Hibernate { t.Fatalf("flag must be set when the guard fires (the operator's fix depends on it)") } // The exact two-step flow the error prescribes: the config regen drops the // host (the flag now reads true), and the re-run flip proceeds. if err := os.WriteFile(conf, []byte("dbs: []\n"), 0o644); err != nil { t.Fatal(err) } confirmPublicReady = func(string, string) {} rt, _ = reg.get(host) if err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { t.Fatalf("re-run flip after exclusion: %v", err) } if len(calls) == 0 || calls[0] != "disable --now lchat@"+host { t.Fatalf("re-run flip sequence = %v", calls) } } // The litestream guard's two-step leaves flag-set-but-unit-running behind; the // promised re-run must FINISH the handover, not early-exit on the flag (hit // live on the first bulk flip: "already hibernating" with the unit still up). func TestHibernateOnRerunFinishesHandover(t *testing.T) { reg := controlTestEnv(t) const host = "flip-rerun.example.com" if _, err := addInstance(reg, addInstanceOpts{Host: host}); err != nil { t.Fatal(err) } if err := reg.setHibernate(host, true); err != nil { // the guard's leftover state t.Fatal(err) } oldQ := systemctlQuery systemctlQuery = func(args ...string) error { return nil } // unit reads active defer func() { systemctlQuery = oldQ }() var calls []string systemctl = func(args ...string) error { calls = append(calls, strings.Join(args, " ")); return nil } confirmPublicReady = func(string, string) { calls = append(calls, "confirm") } rt, _ := reg.get(host) if err := hibernateOn(reg, rt, "lchat@", "lchat-bot@", "lchat-router"); err != nil { t.Fatalf("re-run: %v", err) } if len(calls) == 0 || calls[0] != "disable --now lchat@"+host { t.Fatalf("re-run must stop the unit first, got %v", calls) } } func TestRegistryMigratesCanaryColumn(t *testing.T) { path := filepath.Join(t.TempDir(), "reg.db") // The schema as of the replication column — everything current EXCEPT canary. old, err := sql.Open("sqlite", path) if err != nil { t.Fatal(err) } if _, err := old.Exec(` CREATE TABLE routes ( host TEXT PRIMARY KEY, kind TEXT NOT NULL, socket TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'ready', last_error TEXT NOT NULL DEFAULT '', db_path TEXT NOT NULL DEFAULT '', version TEXT NOT NULL DEFAULT '', prev_version TEXT NOT NULL DEFAULT '', app_store_url TEXT NOT NULL DEFAULT '', home_origin TEXT NOT NULL DEFAULT '', push_relay_url TEXT NOT NULL DEFAULT '', push_relay_token TEXT NOT NULL DEFAULT '', home_region TEXT NOT NULL DEFAULT '', provider TEXT NOT NULL DEFAULT '', hibernate INTEGER NOT NULL DEFAULT 0, replication TEXT NOT NULL DEFAULT '', next_due INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL ); INSERT INTO routes (host, kind, socket, hibernate, created_at) VALUES ('old.example.com', 'instance', '/run/lchat/old.sock', 1, 1700000000); `); err != nil { t.Fatal(err) } if err := old.Close(); err != nil { t.Fatal(err) } reg, err := openRegistry(path) if err != nil { t.Fatalf("open over pre-canary registry: %v", err) } // Existing rows read as canary=false — a live instance is never a throwaway // by default; the flag is only ever set deliberately. rt, err := reg.get("old.example.com") if err != nil || rt == nil { t.Fatalf("get pre-existing row: %+v, %v", rt, err) } if rt.Canary { t.Error("pre-existing row: canary=true, want the never-inferred default (false)") } // The flag round-trips through the narrow setter, an upsert carries it, and // setCanary stays orthogonal to the rest of the route (hibernate intact). if err := reg.setCanary("old.example.com", true); err != nil { t.Fatalf("setCanary: %v", err) } if got, _ := reg.get("old.example.com"); got == nil || !got.Canary || !got.Hibernate { t.Fatalf("after setCanary, route = %+v, want canary + hibernate intact", got) } rt, _ = reg.get("old.example.com") rt.Version = "v2" if err := reg.upsert(*rt); err != nil { t.Fatalf("upsert rewrite: %v", err) } if got, _ := reg.get("old.example.com"); got == nil || !got.Canary { t.Fatalf("route rewrite dropped the canary flag: %+v", got) } // Clearing it round-trips too, and an unknown host is a quiet no-op. if err := reg.setCanary("old.example.com", false); err != nil { t.Fatalf("setCanary off: %v", err) } if got, _ := reg.get("old.example.com"); got == nil || got.Canary { t.Fatalf("setCanary off left it flagged: %+v", got) } if err := reg.setCanary("ghost.example.com", true); err != nil { t.Fatalf("setCanary unknown host should no-op: %v", err) } } func TestCanResetGatesOnTheCanaryFlag(t *testing.T) { // The whole safety model: reset refuses anything not flagged a throwaway. if err := canReset(nil); err == nil { t.Error("canReset(nil) should refuse an unknown host") } if err := canReset(&Route{Host: "push.example.com", Kind: "service", Canary: true}); err == nil { t.Error("canReset should refuse a non-instance route even if flagged") } real := &Route{Host: "family.example.com", Kind: "instance", Canary: false} if err := canReset(real); err == nil { t.Fatal("canReset MUST refuse a real (unflagged) instance — this is the data-safety gate") } canary := &Route{Host: "test-x.11ch.at", Kind: "instance", Canary: true} if err := canReset(canary); err != nil { t.Fatalf("canReset should allow a flagged canary instance, got: %v", err) } } func TestArchiveInstanceDBMovesDBKeepsEnv(t *testing.T) { dir := t.TempDir() oldData := instanceDataDir instanceDataDir = dir defer func() { instanceDataDir = oldData }() host := "test-x.11ch.at" // Seed the DB set + a blob dir + the env/vapid that must SURVIVE. for _, f := range []string{host + ".db", host + ".db-wal", host + ".db-shm", host + ".env", host + ".vapid"} { if err := os.WriteFile(filepath.Join(dir, f), []byte("x"), 0o600); err != nil { t.Fatal(err) } } if err := os.MkdirAll(filepath.Join(dir, host+"-blobs"), 0o700); err != nil { t.Fatal(err) } if err := archiveInstanceDB(host); err != nil { t.Fatalf("archiveInstanceDB: %v", err) } // DB + blobs gone from /data; env + vapid kept in place. for _, gone := range []string{host + ".db", host + ".db-wal", host + ".db-shm", host + "-blobs"} { if _, err := os.Stat(filepath.Join(dir, gone)); !os.IsNotExist(err) { t.Errorf("%s should have been archived out of the data dir", gone) } } for _, kept := range []string{host + ".env", host + ".vapid"} { if _, err := os.Stat(filepath.Join(dir, kept)); err != nil { t.Errorf("%s must survive a reset (admin token + config): %v", kept, err) } } // And the DB is recoverable under /data/trash, not destroyed. trashes, _ := filepath.Glob(filepath.Join(dir, "trash", host+"-reset-*", host+".db")) if len(trashes) != 1 { t.Errorf("archived DB should be recoverable in /data/trash, found %d", len(trashes)) } } func TestSanitizeGroupName(t *testing.T) { long := strings.Repeat("é", 40) // 80 bytes — the cap must cut on a rune boundary cases := []struct{ in, want string }{ {" The Campbells ", "The Campbells"}, {"Line\nBREAK=1\ninjection", "LineBREAK=1injection"}, // env-line injection defused {"tabs\tand\rreturns", "tabsandreturns"}, {long, strings.Repeat("é", 30)}, {"\n\t ", ""}, } for _, c := range cases { if got := sanitizeGroupName(c.in); got != c.want { t.Errorf("sanitizeGroupName(%q) = %q, want %q", c.in, got, c.want) } } } // A seeded display name lands in the instance env (LCHAT_GROUP_NAME) already // sanitized — the instance names itself from it at first boot. func TestAddInstanceSeedsGroupName(t *testing.T) { reg := controlTestEnv(t) host := "named.example.com" if _, err := addInstance(reg, addInstanceOpts{Host: host, GroupName: " The Campbells\n "}); err != nil { t.Fatalf("addInstance: %v", err) } env, err := readEnvFile(filepath.Join(instanceDataDir, host+".env")) if err != nil { t.Fatalf("read env: %v", err) } if env["LCHAT_GROUP_NAME"] != "The Campbells" { t.Errorf("LCHAT_GROUP_NAME = %q, want %q", env["LCHAT_GROUP_NAME"], "The Campbells") } // No name → no env line at all (the instance keeps its default behaviour). host2 := "unnamed.example.com" if _, err := addInstance(reg, addInstanceOpts{Host: host2}); err != nil { t.Fatalf("addInstance: %v", err) } env2, err := readEnvFile(filepath.Join(instanceDataDir, host2+".env")) if err != nil { t.Fatalf("read env: %v", err) } if _, present := env2["LCHAT_GROUP_NAME"]; present { t.Error("empty group name still wrote LCHAT_GROUP_NAME") } } // applySeededName consumes LCHAT_GROUP_NAME exactly once: it names a fresh // space and never overrides a name an admin has since chosen. func TestApplySeededName(t *testing.T) { db := newTestDB(t) t.Setenv("LCHAT_GROUP_NAME", "Book Club") if err := applySeededName(db); err != nil { t.Fatal(err) } if got := db.getGroupName(); got != "Book Club" { t.Errorf("seeded name = %q, want Book Club", got) } // A later boot with a different env value must not clobber the set name. t.Setenv("LCHAT_GROUP_NAME", "Usurper") if err := applySeededName(db); err != nil { t.Fatal(err) } if got := db.getGroupName(); got != "Book Club" { t.Errorf("after re-seed, name = %q, want Book Club (seed-only)", got) } } // The readiness probe's status vocabulary: an instance's own answers (2xx-4xx // bar 404) mean ready; 404 is the router's "unknown host" (a route still // replicating to its owner box) and must keep the probe waiting, like any 5xx. func TestPublicReadyStatus(t *testing.T) { for code, want := range map[int]bool{200: true, 302: true, 401: true, 404: false, 500: false, 502: false} { if got := publicReadyStatus(code); got != want { t.Errorf("publicReadyStatus(%d) = %v, want %v", code, got, want) } } } // A hibernating instance the mesh placed on a peer box never listens on the // local socket — the setup link then mints through the public path (the same // router hop members use, which follows the mesh to wherever the instance // woke). Both paths failing surfaces both errors and no link. func TestAddInstanceSetupLinkPublicFallback(t *testing.T) { reg := controlTestEnv(t) systemctl = func(args ...string) error { return nil } oldMint := mintPublicSetupInvite defer func() { mintPublicSetupInvite = oldMint }() confirmPublicReady = func(string, string) { /* woke on the peer — nothing local */ } var mintedHost, mintedToken string mintPublicSetupInvite = func(host, token string) (string, error) { mintedHost, mintedToken = host, token return "https://" + host + "/join/setup#PUB", nil } res, err := addInstance(reg, addInstanceOpts{Host: "peerowned.example.com", Hibernate: true}) if err != nil { t.Fatalf("addInstance: %v", err) } if res.SetupErr != nil || res.SetupLink != "https://peerowned.example.com/join/setup#PUB" { t.Fatalf("setup link = %q (err %v), want the public-path mint", res.SetupLink, res.SetupErr) } if mintedHost != "peerowned.example.com" || mintedToken != res.AdminToken { t.Errorf("public mint saw host %q, token match %v", mintedHost, mintedToken == res.AdminToken) } mintPublicSetupInvite = func(string, string) (string, error) { return "", fmt.Errorf("no route") } res2, err := addInstance(reg, addInstanceOpts{Host: "dark.example.com", Hibernate: true}) if err != nil { t.Fatalf("addInstance: %v", err) } if res2.SetupLink != "" || res2.SetupErr == nil { t.Fatalf("double failure: link %q, err %v — want no link + a surfaced error", res2.SetupLink, res2.SetupErr) } if !strings.Contains(res2.SetupErr.Error(), "public") { t.Errorf("error should name both paths: %v", res2.SetupErr) } }