package ops // The hibernation half of the restore drill. // // FleetDBs deliberately EXCLUDES hibernating exec routes from // /etc/litestream.yml — the activator's embedded litestream owns their // replication under the lease, and a second replicator on the same WAL would // corrupt the replica (Eleven's lesson). The unintended consequence was that // the fleet's LARGEST set of app databases was the one set nobody ever // restore-drilled: their replicas live in the instances bucket under // //, and the first thing to find out whether // one restores clean was a real user's wake. // // This pass closes that. It is a READER: it restores each replica into a // throwaway file and integrity-checks it (see lease.VerifyReplica for why // that needs no lease and cannot shadow the true holder's state), so it can // run against instances that are asleep, awake and actively replicating, or // anywhere in between. import ( "context" "errors" "fmt" "sort" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/dbverify" "github.com/carlosframework/platform/internal/lease" "github.com/carlosframework/platform/internal/registry" ) // HibernatingRoute is one instance this drill owns: the host names its replica // prefix, and DBPath is the live database the replica is supposed to be // tracking — carried because "restores clean" and "is current" are two // different questions and the second one needs the source. type HibernatingRoute struct { Host string DBPath string } // HibernatingHosts lists the routes whose replication the activator owns — // exactly the set FleetDBs filters out, derived with the SAME conditions so // the two cannot drift into a gap that belongs to neither pass. func HibernatingHosts(reg *registry.Registry) ([]HibernatingRoute, error) { routes, err := reg.List() if err != nil { return nil, err } var hosts []HibernatingRoute for _, r := range routes { // BackingUnit tenants are the box-wide replicator's job (their wake is // systemctl and the activator never replicates them) — FleetDBs adds // them, so this pass must not claim them. if r.Backing == registry.BackingUnit { continue } if r.Hibernate && r.DBPath != "" { hosts = append(hosts, HibernatingRoute{Host: r.Host, DBPath: r.DBPath}) } } sort.Slice(hosts, func(i, j int) bool { return hosts[i].Host < hosts[j].Host }) return hosts, nil } // verifyOne is the seam the tests drive; production points it at S3. var verifyOne = lease.VerifyReplica // VerifyHibernating restore-drills up to max of the given hosts and returns a // verdict per host it VISITED. Hosts beyond the cap are simply absent from // the result — Merge keeps their previous verdict and its old timestamp // rather than inventing coverage for them. // // Order is the caller's: pass hosts oldest-verified-first so a fleet larger // than the cap is still covered completely, just over several days. // // The cap is not squeamishness — it is the same request-cost discipline that // set LeaseTTL. Each verification is a full restore: in-region S3→EC2 // transfer is free, but the GETs and the restore CPU are not, and an // unbounded nightly drill over a growing fleet is a bill that grows with it. func VerifyHibernating(ctx context.Context, cfg *activator.Config, hosts []HibernatingRoute, max int, now time.Time, tmpDir string) map[string]dbverify.Verdict { out := map[string]dbverify.Verdict{} for i, rt := range hosts { if max > 0 && i >= max { break } host := rt.Host prefix := activator.HibernatePrefix(cfg.Prefix, host) counter, err := verifyOne(ctx, cfg.Bucket, prefix, cfg.Region, tmpDir) switch { case err == nil: // Restores clean — now the second question: is it CURRENT? A // hibernating instance's replica has a live owner (the activator's // embedded replicator, under the lease), and an owner that has // stopped writing leaves a replica that still restores perfectly. if f, detail := hibernateFreshness(rt.DBPath, counter); f == freshnessStale || f == freshnessNoSource { out[host] = dbverify.Verdict{VerifiedAt: now.UTC(), Detail: detail} fmt.Printf("STALE %s: %s\n", host, detail) continue } out[host] = dbverify.Verdict{OK: true, VerifiedAt: now.UTC()} fmt.Printf("ok %s\n", host) case errors.Is(err, lease.ErrNoReplica): // Never replicated: a registered instance that has not woken yet. // Not a failure — but not a pass either, because nothing was // proven about anything. out[host] = dbverify.Verdict{Unverified: true, VerifiedAt: now.UTC(), Detail: "no replica yet"} fmt.Printf("unverified %s: no replica yet (never woken)\n", host) case ctx.Err() != nil: // The run is being torn down; what we could not reach is // unverified, never failed. out[host] = dbverify.Verdict{Unverified: true, VerifiedAt: now.UTC(), Detail: err.Error()} fmt.Printf("unverified %s: %v\n", host, err) default: out[host] = dbverify.Verdict{VerifiedAt: now.UTC(), Detail: err.Error()} fmt.Printf("FAIL %s: %v\n", host, err) } } if max > 0 && len(hosts) > max { // A silent cap reads as full coverage — say what was left. fmt.Printf("hibernate-verify: %d of %d hosts this run (--max %d); the rest keep their previous verdict\n", max, len(hosts), max) } return out } // hibernateFreshness asks whether this replica is still tracking its source. // // A route with no recorded DBPath yields "ok": there is nothing to compare // against, and mistaking "we do not know where the source lives" for "the // source is gone" would raise a false alarm on a healthy instance. // HibernatingHosts only yields routes with a DBPath, so this is a guard rather // than a path production takes. func hibernateFreshness(dbPath string, restoredCounter uint32) (freshness, string) { if dbPath == "" { return freshnessOK, "" } return replicaFreshnessCounter(dbPath, restoredCounter, sourceModTime(dbPath)) } // OldestFirst orders hosts by how long ago they were last verified, most // stale first, so a capped run works through the fleet instead of drilling // the same alphabetical prefix every night. Hosts with no verdict at all sort // first — never verified is staler than any timestamp. func OldestFirst(hosts []HibernatingRoute, prior *dbverify.Record) []HibernatingRoute { stamp := map[string]time.Time{} if prior != nil { for h, v := range prior.Hosts { stamp[h] = v.VerifiedAt } } ordered := append([]HibernatingRoute(nil), hosts...) sort.SliceStable(ordered, func(i, j int) bool { a, aok := stamp[ordered[i].Host] b, bok := stamp[ordered[j].Host] if aok != bok { return !aok // never-verified first } if a.Equal(b) { return ordered[i].Host < ordered[j].Host // stable, deterministic in tests } return a.Before(b) }) return ordered }