package main // `carlos route --replication` — the door the replication ladder did not // have. The ladder was built and honoured and unreachable: three named // rungs, one of them reachable, and it was the most expensive one // (discussion #10). import ( "os/exec" "path/filepath" "strings" "testing" "github.com/carlosframework/platform/internal/registry" ) // TestReplicationRailRefusesNonHibernatingRoutes pins the first of this // verb's two refusals, as a pure function for the reason unitRail and // backingRail are: the class is read in exactly one place — the activator's // wake path — and /etc/litestream.yml covers everything the box-wide // replicator is not told to leave alone, at its own interval. On a route that // does not hibernate the column would be written, reported as set, and read // by nothing, which is the same silent no-op the missing door was. func TestReplicationRailRefusesNonHibernatingRoutes(t *testing.T) { awake := registry.Route{ Host: "console.carloku.com", Kind: "instance", App: "console", Addr: "127.0.0.1:8090", DBPath: "/data/console.carloku.com.db", } err := replicationRail(awake) if err == nil { t.Fatal("replicationRail(non-hibernating route) accepted; want a refusal") } for _, want := range []string{"hibernate", "litestream"} { if !strings.Contains(err.Error(), want) { t.Errorf("refusal %q does not mention %q — the operator has to be told which replicator owns this database instead", err, want) } } hibernating := awake hibernating.Hibernate = true if err := replicationRail(hibernating); err != nil { t.Fatalf("replicationRail(hibernating route) = %v, want accepted", err) } } // TestReplicationRailRefusesAHibernatingRouteWithNoDatabase pins the second // half of the same premise, which the first version of this rail missed. // Box-wide litestream excludes a route on `r.Hibernate && r.DBPath != ""` // (ops.FleetDBsWithSkipped) — hibernation ALONE excludes nothing. A // hibernating route whose row names no database is claimed by the dataDir // glob like any other file and replicated at /etc/litestream.yml's interval, // so a rung set on it would be written, reported by `carlos routes`, and // honoured by nobody: the exact failure the rail exists to refuse, reached // through the rail. // // Not hypothetical: `carlos add` defaults db_path only for a single-upstream // non-unit `instance`, so a static or service route, or an instance record // carrying an upstream set (provision.Record.DerivesPaths is false there), // hibernates with the column empty. func TestReplicationRailRefusesAHibernatingRouteWithNoDatabase(t *testing.T) { pathless := registry.Route{ Host: "static.example.com", Kind: "static", App: "site", Hibernate: true, } err := replicationRail(pathless) if err == nil { t.Fatal("replicationRail(hibernating route with no db_path) accepted; want a refusal") } for _, want := range []string{"db_path", "--db"} { if !strings.Contains(err.Error(), want) { t.Errorf("refusal %q does not mention %q — the operator has to be told what is missing and which door supplies it", err, want) } } } // TestReplicationWordNamesTheDefault: "" is this column's value on every // route that predates the door, it behaves exactly like stream, and printing // it as an empty string reads as a bug rather than as the answer it is. func TestReplicationWordNamesTheDefault(t *testing.T) { if got := replicationWord(""); !strings.Contains(got, "default") { t.Errorf(`replicationWord("") = %q, want it to name the default`, got) } for _, class := range registry.ReplicationClasses() { got := replicationWord(class) if !strings.Contains(got, class) { t.Errorf("replicationWord(%q) = %q, want it to name the rung", class, got) } if !strings.Contains(got, "loss window") { t.Errorf("replicationWord(%q) = %q, want the loss window spelled out — that is the whole trade", class, got) } } } // TestReplicationCellNamesTheDefault: the console-side listing has the same // duty. Unlike ENVIRONMENT, whose empty value is dashed, this one is named: // it is the column where "no answer" cost one tenant 97% of a day's replica // traffic. func TestReplicationCellNamesTheDefault(t *testing.T) { if got := replicationCell(""); got != "default" { t.Errorf(`replicationCell("") = %q, want "default"`, got) } if got := replicationCell(registry.ReplicationDaily); got != "daily" { t.Errorf("replicationCell(daily) = %q", got) } } // TestRouteReplicationSetsAndClearsTheRung drives the whole door as a // subprocess, the shape it ships in: the routes that need a cheaper rung // already exist on live boxes, so this stamps a LIVE registry row rather than // needing the route recreated. // // It pins the failure a careless door would cause as much as the one the // missing door did: replicationSyncInterval answers an unknown class with // litestream's default, so an accepted typo is a route that quietly never // left the most expensive rung. func TestRouteReplicationSetsAndClearsTheRung(t *testing.T) { bin := buildCarlos(t) tmp := t.TempDir() scratch := t.TempDir() regPath := filepath.Join(tmp, "registry.db") run := func(args ...string) (string, error) { t.Helper() cmd := exec.Command(bin, args...) cmd.Env = carlosTestEnv(t) // pick() walks cwd ancestors for .carlos/config, so the subprocess // must not run from the real checkout. cmd.Dir = scratch out, err := cmd.CombinedOutput() return string(out), err } replicationOf := func(host string) string { t.Helper() reg, err := registry.Open(regPath) if err != nil { t.Fatalf("open registry: %v", err) } defer reg.Close() rt, err := reg.Get(host) if err != nil || rt == nil { t.Fatalf("get %s: %+v, %v", host, rt, err) } return rt.Replication } // A hibernating instance as one exists on a box today: a large database, // low write volume, on the every-second default because nothing could say // otherwise. if out, err := run("add", "-registry", regPath, "-app", "seapointish", "-socket", filepath.Join(tmp, "s.sock"), "-hibernate", "seapointish.example.com"); err != nil { t.Fatalf("add route: %v\n%s", err, out) } if got := replicationOf("seapointish.example.com"); got != "" { t.Fatalf("a fresh route already carries a rung: %q", got) } // A typo must cost nothing and reach nothing. out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", "-replication", "dayly") if err == nil { t.Fatalf("route --replication dayly: want non-zero exit, got 0\n%s", out) } if got := replicationOf("seapointish.example.com"); got != "" { t.Fatalf("a refused --replication still wrote %q", got) } // A bare "" is refused rather than silently treated as a clear: the doors // that clear say so out loud. if out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", "-replication", ""); err == nil { t.Fatalf("route --replication \"\": want non-zero exit, got 0\n%s", out) } // -dry-run runs the rail and prints the plan, writing nothing. out, err = run("route", "-registry", regPath, "-host", "seapointish.example.com", "-replication", "daily", "-dry-run") if err != nil { t.Fatalf("route --replication --dry-run: %v\n%s", err, out) } if !strings.Contains(out, "daily") { t.Errorf("dry run output = %q, want it to name the rung", out) } if got := replicationOf("seapointish.example.com"); got != "" { t.Fatalf("--dry-run wrote %q", got) } // The real write. if out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", "-replication", "daily"); err != nil { t.Fatalf("route --replication daily: %v\n%s", err, out) } if got := replicationOf("seapointish.example.com"); got != registry.ReplicationDaily { t.Fatalf("replication = %q, want %q", got, registry.ReplicationDaily) } // And the way back, which is the sentinel rather than a name to type. if out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", "-replication", "-"); err != nil { t.Fatalf("route --replication -: %v\n%s", err, out) } if got := replicationOf("seapointish.example.com"); got != "" { t.Fatalf("--replication - left %q, want the default", got) } // The listing has to answer the question the setter just decided, or an // operator cannot see what they set — which was half of #10's complaint. if out, err := run("route", "-registry", regPath, "-host", "seapointish.example.com", "-replication", "batched"); err != nil { t.Fatalf("route --replication batched: %v\n%s", err, out) } out, err = run("instances", "-registry", regPath) if err != nil { t.Fatalf("instances: %v\n%s", err, out) } if !strings.Contains(out, "REPL") || !strings.Contains(out, "batched") { t.Errorf("`carlos instances` does not show the rung:\n%s", out) } } // TestRouteReplicationRefusesCombinationWithOtherAxes: the dispatch runs one // arm, so a command naming two axes would write one and drop the other on the // floor without a word — the silent partial --backing's exclusions were added // for. --app is refused outright: a bulk durability change is precisely the // operation nobody should be able to type by accident. func TestRouteReplicationRefusesCombinationWithOtherAxes(t *testing.T) { bin := buildCarlos(t) tmp := t.TempDir() scratch := t.TempDir() regPath := filepath.Join(tmp, "registry.db") run := func(args ...string) (string, error) { t.Helper() cmd := exec.Command(bin, args...) cmd.Env = carlosTestEnv(t) cmd.Dir = scratch out, err := cmd.CombinedOutput() return string(out), err } if out, err := run("add", "-registry", regPath, "-app", "a", "-socket", filepath.Join(tmp, "a.sock"), "-hibernate", "a.example.com"); err != nil { t.Fatalf("add: %v\n%s", err, out) } for _, extra := range [][]string{ {"-hibernate=false"}, {"-channel", "edge"}, {"-unit", "a.service"}, } { args := append([]string{"route", "-registry", regPath, "-host", "a.example.com", "-replication", "daily"}, extra...) if out, err := run(args...); err == nil { t.Errorf("route %v: want a refusal, got 0\n%s", extra, out) } } if out, err := run("route", "-registry", regPath, "-app", "a", "-replication", "daily"); err == nil { t.Errorf("route --app --replication: want a refusal, got 0\n%s", out) } } // TestRouteHibernateOffClearsTheRung: the rail is point-in-time, and until // this landed nothing held it afterwards. A route set to `daily` could stop // hibernating — by this verb, by `carlos add` with no --hibernate, or by a // converge pass reading an instance record — and `carlos routes` went on // printing `daily` while box-wide /etc/litestream.yml replicated the database // at its own interval. A column that lies is the failure the door was built // to end, so it must not survive the flip that makes it false. // // The clearing itself lives in registry.SetHibernate, where every door that // turns hibernation off passes; what is pinned here is the operator-facing // half — the flip works through the real binary, and it SAYS what it spent, // because the rung is the one operator-typed value this command drops // without being asked to. func TestRouteHibernateOffClearsTheRung(t *testing.T) { bin := buildCarlos(t) tmp := t.TempDir() scratch := t.TempDir() regPath := filepath.Join(tmp, "registry.db") run := func(args ...string) (string, error) { t.Helper() cmd := exec.Command(bin, args...) cmd.Env = carlosTestEnv(t) cmd.Dir = scratch out, err := cmd.CombinedOutput() return string(out), err } rowOf := func(host string) (string, bool) { t.Helper() reg, err := registry.Open(regPath) if err != nil { t.Fatalf("open registry: %v", err) } defer reg.Close() rt, err := reg.Get(host) if err != nil || rt == nil { t.Fatalf("get %s: %+v, %v", host, rt, err) } return rt.Replication, rt.Hibernate } host := "seapointish.example.com" if out, err := run("add", "-registry", regPath, "-app", "seapointish", "-socket", filepath.Join(tmp, "s.sock"), "-hibernate", host); err != nil { t.Fatalf("add route: %v\n%s", err, out) } if out, err := run("route", "-registry", regPath, "-host", host, "-replication", "daily"); err != nil { t.Fatalf("route --replication daily: %v\n%s", err, out) } if class, _ := rowOf(host); class != registry.ReplicationDaily { t.Fatalf("setup: replication = %q, want %q", class, registry.ReplicationDaily) } out, err := run("route", "-registry", regPath, "-host", host, "-hibernate=false") if err != nil { t.Fatalf("route --hibernate=false: %v\n%s", err, out) } class, hib := rowOf(host) if hib { t.Fatalf("setup: the route still hibernates\n%s", out) } if class != "" { t.Fatalf("replication = %q on a route that no longer hibernates — `carlos routes` would print a rung the box replicator does not honour", class) } // Told, not discovered. for _, want := range []string{"replication", "cleared"} { if !strings.Contains(out, want) { t.Errorf("--hibernate=false output does not mention %q:\n%s", want, out) } } // And the door is shut behind it: the rung cannot be re-set while the // route is awake, which is what makes the clear an honest state rather // than a gap between two commands. if out, err := run("route", "-registry", regPath, "-host", host, "-replication", "daily"); err == nil { t.Errorf("route --replication on a woken route: want a refusal, got 0\n%s", out) } }