package ops import ( "context" "errors" "os" "path/filepath" "slices" "strings" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/dbverify" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/registry" ) func at(s string) time.Time { t, err := time.Parse(time.RFC3339, s) if err != nil { panic(err) } return t } // The two passes must partition the fleet: every database is owned by exactly // one of them. This test is the mirror of TestConfigGenSkipsHibernatingRoutes // — what that one proves is EXCLUDED here must be INCLUDED, or it is drilled // by nobody. func TestHibernatingHostsIsTheComplementOfFleetDBs(t *testing.T) { dir := t.TempDir() reg, err := registry.Open(filepath.Join(dir, "registry.db")) if err != nil { t.Fatal(err) } unitDir := t.TempDir() awake := filepath.Join(dir, "a.example.com.db") asleep := filepath.Join(dir, "b.example.com.db") unitTenant := filepath.Join(unitDir, "state.db") for _, p := range []string{awake, asleep, unitTenant} { if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { t.Fatal(err) } } if err := reg.Upsert(registry.Route{Host: "a.example.com", DBPath: awake}); err != nil { t.Fatal(err) } if err := reg.Upsert(registry.Route{Host: "b.example.com", DBPath: asleep, Hibernate: true}); err != nil { t.Fatal(err) } if err := reg.Upsert(registry.Route{Host: "c.example.com", DBPath: unitTenant, Hibernate: true, Backing: registry.BackingUnit}); err != nil { t.Fatal(err) } hosts := hostsOf(mustHibernating(t, reg)) if !slices.Contains(hosts, "b.example.com") { t.Fatalf("the hibernating exec route is drilled by nobody: %v", hosts) } if slices.Contains(hosts, "a.example.com") { t.Fatalf("an awake route is claimed by both passes: %v", hosts) } if slices.Contains(hosts, "c.example.com") { t.Fatalf("a unit tenant (box-wide replicator's job) is claimed by both passes: %v", hosts) } // And nothing this pass claims may appear in the box-wide config. dbs, err := FleetDBs(reg, dir) if err != nil { t.Fatal(err) } cfg := LitestreamConfig(dbs, "b", "p", "eu-west-1") for _, h := range hosts { if strings.Contains(cfg, h+".db") { t.Fatalf("%s is replicated by BOTH passes — two replicators on one WAL:\n%s", h, cfg) } } } // routes builds the drill's input from bare host names. The DBPath is left // empty on purpose: these tests drive the RESTORE outcomes, and an absent // source is reported as its own verdict, which TestVerifyHibernatingStale // covers explicitly. func routes(hosts ...string) []HibernatingRoute { out := make([]HibernatingRoute, 0, len(hosts)) for _, h := range hosts { out = append(out, HibernatingRoute{Host: h}) } return out } func hostsOf(rs []HibernatingRoute) []string { out := make([]string, 0, len(rs)) for _, r := range rs { out = append(out, r.Host) } return out } func mustHibernating(t *testing.T, reg *registry.Registry) []HibernatingRoute { t.Helper() got, err := HibernatingHosts(reg) if err != nil { t.Fatal(err) } return got } // swapVerifyOne points the restore seam at a stub. func swapVerifyOne(t *testing.T, fn func(ctx context.Context, bucket, prefix, region, dir string) (uint32, error)) { t.Helper() prev := verifyOne verifyOne = fn t.Cleanup(func() { verifyOne = prev }) } // The three verdicts, and — most importantly — that an S3 failure is // "unverified", not a corruption alarm, and a never-woken instance is neither. func TestVerifyHibernatingVerdicts(t *testing.T) { now := at("2026-08-15T09:00:00Z") swapVerifyOne(t, func(_ context.Context, _, prefix, _, _ string) (uint32, error) { switch { case strings.Contains(prefix, "good"): return 1, nil case strings.Contains(prefix, "damaged"): return 0, errors.New("integrity_check: page 4 corrupt") case strings.Contains(prefix, "new"): return 0, lease.ErrNoReplica } return 0, errors.New("unexpected host") }) cfg := &activator.Config{Bucket: "instances", Prefix: "hib", Region: "eu-west-1"} got := VerifyHibernating(context.Background(), cfg, routes("good.example.com", "damaged.example.com", "new.example.com"), 0, now, t.TempDir()) if v := got["good.example.com"]; !v.OK || v.Unverified { t.Fatalf("clean replica verdict = %+v", v) } if v := got["damaged.example.com"]; v.OK || v.Unverified { t.Fatalf("damaged replica must be a failure, got %+v", v) } if v := got["new.example.com"]; v.OK || !v.Unverified { t.Fatalf("never-woken instance must be unverified (nothing was proven), got %+v", v) } rec := dbverify.Merge(nil, "box-a", now, []string{"good.example.com", "damaged.example.com", "new.example.com"}, got) if rec.Failures() != 1 { t.Fatalf("Failures() = %d, want 1 — only the damaged replica", rec.Failures()) } if rec.Unverified() != 1 { t.Fatalf("Unverified() = %d, want 1", rec.Unverified()) } } // The per-host prefix must be the one the activator actually writes to, or // every drill reports "no replica" over perfectly good backups. func TestVerifyHibernatingUsesTheActivatorsPrefix(t *testing.T) { var seen string swapVerifyOne(t, func(_ context.Context, _, prefix, _, _ string) (uint32, error) { seen = prefix return 1, nil }) cfg := &activator.Config{Bucket: "instances", Prefix: "hib"} VerifyHibernating(context.Background(), cfg, routes("x.example.com"), 0, time.Now(), t.TempDir()) want := activator.HibernatePrefix("hib", "x.example.com") if seen != want { t.Fatalf("drilled prefix %q, activator writes %q", seen, want) } } // A capped run visits the cap and no more — the rest keep their old verdict // via Merge, which is what stops a cap from reading as full coverage. func TestVerifyHibernatingHonoursTheCap(t *testing.T) { calls := 0 swapVerifyOne(t, func(context.Context, string, string, string, string) (uint32, error) { calls++ return 1, nil }) cfg := &activator.Config{Bucket: "instances"} got := VerifyHibernating(context.Background(), cfg, routes("a.example.com", "b.example.com", "c.example.com"), 2, time.Now(), t.TempDir()) if calls != 2 { t.Fatalf("restored %d replicas under --max 2", calls) } if len(got) != 2 { t.Fatalf("returned %d verdicts under --max 2: %v", len(got), got) } } // Oldest-first is what makes a capped drill cover the whole fleet over time // instead of re-drilling the same alphabetical head every night. func TestOldestFirstPutsNeverVerifiedThenStalest(t *testing.T) { prior := &dbverify.Record{Hosts: map[string]dbverify.Verdict{ "recent.example.com": {OK: true, VerifiedAt: at("2026-08-14T00:00:00Z")}, "stale.example.com": {OK: true, VerifiedAt: at("2026-08-01T00:00:00Z")}, }} got := hostsOf(OldestFirst(routes("recent.example.com", "stale.example.com", "never.example.com"), prior)) want := []string{"never.example.com", "stale.example.com", "recent.example.com"} if !slices.Equal(got, want) { t.Fatalf("OldestFirst = %v, want %v", got, want) } // With no prior record at all, the order must still be deterministic. if got := hostsOf(OldestFirst(routes("b.example.com", "a.example.com"), nil)); !slices.Equal(got, []string{"a.example.com", "b.example.com"}) { t.Fatalf("OldestFirst with no prior record = %v", got) } } // A hibernating instance's replica has a live owner — the activator's embedded // replicator, under the lease — and an owner that has stopped writing leaves a // replica that still restores perfectly. Same blind spot as the box-wide // drill, same signal: the replica's change counter stopped moving while the // database it mirrors did not. func TestVerifyHibernatingCatchesAStaleReplica(t *testing.T) { dir := t.TempDir() live := filepath.Join(dir, "live.db") writeCounterDB(t, live, 30) // Quiet for longer than the grace window: unambiguously "replication // stopped", not "still catching up". old := time.Now().Add(-2 * time.Hour) if err := os.Chtimes(live, old, old); err != nil { t.Fatal(err) } // The replica restores clean, but at a change counter from long ago. swapVerifyOne(t, func(context.Context, string, string, string, string) (uint32, error) { return 2, nil }) now := at("2026-08-15T09:00:00Z") got := VerifyHibernating(context.Background(), &activator.Config{Bucket: "instances"}, []HibernatingRoute{{Host: "frozen.example.com", DBPath: live}}, 0, now, t.TempDir()) v := got["frozen.example.com"] if v.OK { t.Fatalf("a replica frozen while its instance kept writing was reported OK: %+v", v) } if v.Unverified { t.Fatalf("a stale replica must be a FAILURE, not merely unverified: %+v", v) } if !strings.Contains(v.Detail, "replication has stopped") { t.Fatalf("verdict does not say what is wrong: %q", v.Detail) } // And it must reach the published record as a failure, so the operator // sees it without a session on the box. rec := dbverify.Merge(nil, "box-a", now, []string{"frozen.example.com"}, got) if rec.Failures() != 1 { t.Fatalf("Failures() = %d, want 1", rec.Failures()) } }