package host import ( "errors" "os" "path/filepath" "testing" "time" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) type fakeLister struct { routes []registry.Route err error } func (f fakeLister) List() ([]registry.Route, error) { return f.routes, f.err } func stageSweepVersion(t *testing.T, optDir, acct, ver, app string, age time.Duration) { t.Helper() p := release.VersionPath(optDir, acct, ver, app) if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(p, []byte("binary"), 0o755); err != nil { t.Fatal(err) } when := time.Now().Add(-age) if err := os.Chtimes(p, when, when); err != nil { t.Fatal(err) } } // noGenerations is a generations file path with neither file present: no // generations, every box before its agent first published. func noGenerations(t *testing.T) string { return release.GenerationsPath(t.TempDir()) } func staged(t *testing.T, optDir, ver string) bool { t.Helper() _, err := os.Stat(filepath.Join(optDir, "versions", ver)) return err == nil } // PrevVersion is a pin with no grace: a rollback is wanted precisely when the // new build has been running long enough for the grace to have expired. func TestSweepStagedVersionsPinsPrevVersion(t *testing.T) { opt := t.TempDir() stageSweepVersion(t, opt, "acct1", "v3", "blog", 48*time.Hour) stageSweepVersion(t, opt, "acct1", "v2", "blog", 48*time.Hour) stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour) reg := fakeLister{routes: []registry.Route{{Host: "blog.example.com", Version: "v3", PrevVersion: "v2"}}} sweepStagedVersions(reg, opt, noGenerations(t), 6*time.Hour, time.Now()) if !staged(t, opt, "v3") { t.Error("v3 (Version) was removed") } if !staged(t, opt, "v2") { t.Error("v2 (PrevVersion) was removed — the rollback target is a pin") } if staged(t, opt, "v1") { t.Error("v1 survived; it is pinned by nothing and older than the grace") } } // A route with no version at all must not pin the empty string into // something the sweep then matches. func TestSweepStagedVersionsIgnoresEmptyVersions(t *testing.T) { opt := t.TempDir() stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour) reg := fakeLister{routes: []registry.Route{{Host: "pending.example.com"}}} sweepStagedVersions(reg, opt, noGenerations(t), 6*time.Hour, time.Now()) if staged(t, opt, "v1") { t.Error("v1 survived a pin set that should be empty") } } // An unlistable registry means an incomplete pin set, and an incomplete pin // set is how a running app loses its binary. Delete nothing. func TestSweepStagedVersionsDeletesNothingWhenRegistryFails(t *testing.T) { opt := t.TempDir() stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour) sweepStagedVersions(fakeLister{err: errors.New("database is locked")}, opt, noGenerations(t), 6*time.Hour, time.Now()) if !staged(t, opt, "v1") { t.Error("swept with an unreadable registry") } } // Likewise for an unreadable live dir. func TestSweepStagedVersionsDeletesNothingWhenLivePinsFail(t *testing.T) { opt := t.TempDir() stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour) if err := os.MkdirAll(filepath.Join(opt, "live"), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(opt, "live", "stray"), []byte("not a link"), 0o644); err != nil { t.Fatal(err) } sweepStagedVersions(fakeLister{}, opt, noGenerations(t), 6*time.Hour, time.Now()) if !staged(t, opt, "v1") { t.Error("swept with an unreadable live dir") } } // The live link is believed over the registry: PointLive succeeding and // SetVersions then failing on a full /data is the incident's own state, and // the link is the only record of what is running. func TestSweepStagedVersionsBelievesLiveLinkOverRegistry(t *testing.T) { opt := t.TempDir() stageSweepVersion(t, opt, release.OpsAccount, "v2", "edge", 48*time.Hour) if err := release.PointLive(opt, "edge.example.com", release.OpsAccount, "v2", "edge"); err != nil { t.Fatal(err) } // The registry still says v1 — the version write never landed. reg := fakeLister{routes: []registry.Route{{Host: "edge.example.com", Version: "v1"}}} sweepStagedVersions(reg, opt, noGenerations(t), 6*time.Hour, time.Now()) if !staged(t, opt, "v2") { t.Fatal("the running version was removed because the registry had not caught up") } } // Ruling R19 item 1 (Astra r3 F1(a)): a worker's captured build that only // the generations file still names (by version, or by staged path alone) // survives the sweep past its grace. What nothing names is still removed. func TestSweepStagedVersionsKeepsBuildsTheGenerationsFileNames(t *testing.T) { opt := t.TempDir() for _, v := range []string{"v0", "v1", "v2", "v3"} { stageSweepVersion(t, opt, "acct1", v, "bot", 48*time.Hour) } gen := noGenerations(t) if err := release.WriteGenerations(gen, release.Generations{ Hosts: map[string][]string{"bot.example.com": {"v1"}}, Paths: map[string][]string{"chained.example.com": {release.VersionPath(opt, "acct1", "v2", "bot")}}, }); err != nil { t.Fatal(err) } reg := fakeLister{routes: []registry.Route{{Host: "bot.example.com", Version: "v3"}}} sweepStagedVersions(reg, opt, gen, 6*time.Hour, time.Now()) if !staged(t, opt, "v1") { t.Error("v1 was removed; the generations file names it as a running worker's build") } if !staged(t, opt, "v2") { t.Error("v2 was removed; the generations file names its staged path") } if !staged(t, opt, "v3") { t.Error("v3 (Version) was removed") } if staged(t, opt, "v0") { t.Error("v0 survived; nothing names it and it is older than the grace") } } // Ruling R19 item 1: a generations file that does not read, or whose // publication did not finish (the intent ahead of the file), is an incomplete // pin set, and the sweep removes nothing that pass. func TestSweepStagedVersionsDeletesNothingWhenTheGenerationsFileIsIncomplete(t *testing.T) { for name, files := range map[string][2]string{ "unreadable": {"", `{"schema":1,"hosts":{"bot.exa`}, "intent ahead": {"2", `{"schema":1,"seq":1,"complete":true,"hosts":{}}`}, } { t.Run(name, func(t *testing.T) { opt := t.TempDir() stageSweepVersion(t, opt, "acct1", "v1", "bot", 48*time.Hour) gen := noGenerations(t) if err := os.MkdirAll(filepath.Dir(gen), 0o755); err != nil { t.Fatal(err) } if files[0] != "" { if err := os.WriteFile(release.GenerationsSeqPath(gen), []byte(files[0]), 0o644); err != nil { t.Fatal(err) } } if err := os.WriteFile(gen, []byte(files[1]), 0o644); err != nil { t.Fatal(err) } sweepStagedVersions(fakeLister{}, opt, gen, 6*time.Hour, time.Now()) if !staged(t, opt, "v1") { t.Error("swept with an incomplete generations file") } }) } } func TestVersionSweepGrace(t *testing.T) { env := func(v string) func(string) string { return func(k string) string { if k == versionSweepGraceEnv { return v } return "" } } if got := versionSweepGrace(env("")); got != release.DefaultSweepGrace { t.Errorf("unset = %s, want %s", got, release.DefaultSweepGrace) } if got := versionSweepGrace(env("30m")); got != 30*time.Minute { t.Errorf("30m = %s", got) } // Garbage and negatives fall back rather than sweeping with a zero grace. for _, bad := range []string{"soon", "-1h"} { if got := versionSweepGrace(env(bad)); got != release.DefaultSweepGrace { t.Errorf("%q = %s, want the default %s", bad, got, release.DefaultSweepGrace) } } } // The pass lock is what makes the sweep safe against a concurrent // carlos-stage.service, so it has to actually exclude. func TestLockAdoptPassExcludes(t *testing.T) { t.Parallel() dd := t.TempDir() unlock, err := lockAdoptPass(dd) if err != nil { t.Fatal(err) } held := make(chan struct{}) go func() { second, err := lockAdoptPass(dd) if err == nil { second() } close(held) }() select { case <-held: t.Fatal("a second pass took the lock while the first held it") case <-time.After(100 * time.Millisecond): } unlock() select { case <-held: case <-time.After(5 * time.Second): t.Fatal("the second pass never got the lock after the first released it") } } // The adopt lock and the restart marks lock are different files: RunAdopt // holds the first for the whole pass and restartPass takes the second inside // it, so sharing an inode would deadlock every pass that restarts anything. func TestAdoptAndRestartLocksAreDistinct(t *testing.T) { dd := t.TempDir() unlock, err := lockAdoptPass(dd) if err != nil { t.Fatal(err) } defer unlock() inner, err := lockRestartMarks(restartMarksPath(dd)) if err != nil { t.Fatalf("restart marks lock blocked by the pass lock: %v", err) } inner() } // Spec §8.5: the good build is a pin, with no grace, like PrevVersion: it is // the fallback a failed first wake boots, however long ago it last served. func TestSweepStagedVersionsKeepsTheGoodBuild(t *testing.T) { opt := t.TempDir() for _, v := range []string{"good", "gone"} { stageSweepVersion(t, opt, "acct1", v, "km", 48*time.Hour) } reg := fakeLister{routes: []registry.Route{{Host: "a.example.com", Kind: "instance", App: "km", Account: "acct1", Version: "x", GoodVersion: "good"}}} sweepStagedVersions(reg, opt, noGenerations(t), 6*time.Hour, time.Now()) if !staged(t, opt, "good") { t.Error("the good build was removed") } if staged(t, opt, "gone") { t.Error("gone survived; nothing names it and it is older than the grace") } } // settingsRow is an activator-managed instance with the given build-state // copy and update-settings columns. func settingsRow(host, buildState, settings string) registry.Route { return registry.Route{Host: host, Kind: "instance", App: "blog", Account: "acct1", Hibernate: true, Socket: "/run/carlos/" + host + ".sock", Version: "v9", BuildState: buildState, UpdateSettings: settings} } // Ruling R-T9-STORE: the box-local sweep keeps every build a build state // names (last_executed and a resolved request's T) and every pin, so a held // instance's staged recovery copy (EvidenceStaged) is never evicted. func TestSweepStagedVersionsKeepsBuildStateBuildsAndPins(t *testing.T) { opt := t.TempDir() for _, v := range []string{"v1", "v2", "v3", "v4", "v9"} { stageSweepVersion(t, opt, "acct1", v, "blog", 48*time.Hour) } copyCol := release.BuildStateCopy{Known: true, Replica: true, State: &release.BuildState{LastExecuted: "v1", Resolved: &release.ResolvedRequest{ID: 2, Target: "v2"}}}.Column() reg := fakeLister{routes: []registry.Route{ settingsRow("a.example.com", copyCol, release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable"}.Column()), settingsRow("b.example.com", release.BuildStateCopy{Known: true}.Column(), release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable", Pin: "v3"}.Column()), }} sweepStagedVersions(reg, opt, noGenerations(t), 6*time.Hour, time.Now()) for _, v := range []string{"v1", "v2", "v3", "v9"} { if !staged(t, opt, v) { t.Errorf("%s was removed", v) } } if staged(t, opt, "v4") { t.Error("v4 survived; nothing names it") } } // A copy that keeps its last known state after a failed refresh still pins // it: the build it names did not stop being the instance's. func TestSweepStagedVersionsKeepsALastKnownState(t *testing.T) { opt := t.TempDir() stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour) col := release.BuildStateCopy{Known: true, ReadErr: "timeout", State: &release.BuildState{LastExecuted: "v1"}}.Column() sweepStagedVersions(fakeLister{routes: []registry.Route{settingsRow("a.example.com", col, "")}}, opt, noGenerations(t), 6*time.Hour, time.Now()) if !staged(t, opt, "v1") { t.Error("the last known last_executed was removed after a failed refresh") } } // Astra round 1 finding 6: a held instance's copy that kept A after a failed // refresh is not a complete pin set. The durable state may name B since (a // recovery onto B on another box, then a return), and B's staged copy here is // its only recovery copy, named by no live link, column or generation. The // sweep deletes nothing, and so keeps A too. A default-settings row's failed // refresh holds nothing on its last_executed, so it does not stop the sweep. func TestSweepStagedVersionsRefusesAHeldCopyWithAReadError(t *testing.T) { opt := t.TempDir() for _, v := range []string{"A", "B", "v9"} { stageSweepVersion(t, opt, "acct1", v, "blog", 48*time.Hour) } manual := release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable"}.Column() col := release.BuildStateCopy{Known: true, Replica: true, ReadErr: "s3: timeout", State: &release.BuildState{LastExecuted: "A"}}.Column() sweepStagedVersions(fakeLister{routes: []registry.Route{settingsRow("a.example.com", col, manual)}}, opt, noGenerations(t), 6*time.Hour, time.Now()) for _, v := range []string{"A", "B"} { if !staged(t, opt, v) { t.Errorf("%s was removed: the sweep ran on a held copy whose last refresh failed", v) } } opt = t.TempDir() for _, v := range []string{"A", "B", "v9"} { stageSweepVersion(t, opt, "acct1", v, "blog", 48*time.Hour) } sweepStagedVersions(fakeLister{routes: []registry.Route{settingsRow("a.example.com", col, "")}}, opt, noGenerations(t), 6*time.Hour, time.Now()) if !staged(t, opt, "A") || staged(t, opt, "B") { t.Errorf("default settings: A staged %v, B staged %v; want A kept (named) and B swept", staged(t, opt, "A"), staged(t, opt, "B")) } } // Any column the sweep cannot read is an incomplete pin set: an unparseable // build-state copy or update-settings column, or a held instance whose build // state is not read yet (its last_executed is unknown). Delete nothing. func TestSweepStagedVersionsDeletesNothingWhenSettingsPinsAreUnknown(t *testing.T) { manual := release.UpdateSettings{Updates: release.UpdatesManual, Channel: "stable"}.Column() cases := map[string]registry.Route{ "unparseable build state": settingsRow("a.example.com", "{", ""), "unparseable update settings": settingsRow("a.example.com", "", "{"), "held, build state unread": settingsRow("a.example.com", "", manual), "pinned, build state unread": settingsRow("a.example.com", "", release.UpdateSettings{Updates: release.UpdatesAuto, Channel: "stable", Pin: "v1"}.Column()), } for name, rt := range cases { t.Run(name, func(t *testing.T) { opt := t.TempDir() stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour) sweepStagedVersions(fakeLister{routes: []registry.Route{rt}}, opt, noGenerations(t), 6*time.Hour, time.Now()) if !staged(t, opt, "v1") { t.Error("swept with a pin set it could not complete") } }) } // Rows whose unread copy says nothing the sweep needs: default settings // (nothing is held on its last_executed), and rows keepsBuildState never // refreshes, a named unit or the platform's own row, whatever settings // an app policy gives them (their copy stays unread forever). The sweep // goes on. unit := settingsRow("u.example.com", "", manual) unit.Unit = "blog.service" platform := settingsRow("p.example.com", "", manual) platform.Platform = true worker := settingsRow("w.example.com", "", manual) worker.Kind, worker.Hibernate = "sidecar", false goesOn := map[string]registry.Route{ "worker, settings set": worker, "default settings": settingsRow("a.example.com", "", ""), "named unit, held": unit, "platform row, held": platform, } for name, rt := range goesOn { t.Run(name, func(t *testing.T) { opt := t.TempDir() stageSweepVersion(t, opt, "acct1", "v1", "blog", 48*time.Hour) sweepStagedVersions(fakeLister{routes: []registry.Route{rt}}, opt, noGenerations(t), 6*time.Hour, time.Now()) if staged(t, opt, "v1") { t.Error("a row whose unread copy pins nothing stopped the sweep") } }) } }