package host import ( "bytes" "context" "log" "net/http" "net/http/httptest" "reflect" "runtime" "strings" "sync" "testing" "time" "github.com/carlosframework/platform/internal/activator" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) // The stamp is the whole licence for skipping the sweep's grace: it is written // only after the adopting pass is finished with the host. A drift whose route // is still stamped with the old version is an adoption in flight, and one with // no row at all is not a route this box adopts. func TestConvergeAdoptedCyclesOnlyWhenTheStampAgrees(t *testing.T) { rows := map[string]*registry.Route{ "done.test": {Host: "done.test", Version: "v2"}, "inflight.test": {Host: "inflight.test", Version: "v1"}, "named.test": {Host: "named.test", Version: "v2", Unit: "console.service"}, "unitback.test": {Host: "unitback.test", Version: "v2", Backing: registry.BackingUnit}, } drifted := func() map[string]string { return map[string]string{ "done.test": "v2", "inflight.test": "v2", "named.test": "v2", "unitback.test": "v2", "norow.test": "v2", } } var cycled []string n := convergeAdopted(drifted, func(h string) (*registry.Route, error) { return rows[h], nil }, func(h, ver string) bool { cycled = append(cycled, h+"@"+ver); return true }, 10) if want := []string{"done.test@v2"}; !reflect.DeepEqual(cycled, want) { t.Fatalf("cycled = %v, want %v", cycled, want) } if n != 1 { t.Fatalf("convergeAdopted = %d, want 1", n) } } // The ordinary tick, where nothing has shipped, must cost no registry read: // this leg runs every two seconds on every box. func TestConvergeAdoptedReadsNoRouteWhenNothingDrifted(t *testing.T) { n := convergeAdopted(func() map[string]string { return nil }, func(h string) (*registry.Route, error) { t.Fatalf("read the route for %s with nothing drifted", h) return nil, nil }, func(h, ver string) bool { t.Fatalf("cycled %s with nothing drifted", h) return false }, 10) if n != 0 { t.Fatalf("convergeAdopted = %d, want 0", n) } } // The leg runs on its own loop, never waiting on the fast tick: a tick that // takes a full reconcile and adopt pass can run tens of seconds, and a // stamped adoption must not wait behind it. Here the loop keeps calling the // leg on its own cadence, and stops when told. func TestConvergeLoopRunsTheLegOnItsOwnCadence(t *testing.T) { t.Parallel() calls := make(chan struct{}, 100) stop := make(chan struct{}) done := make(chan struct{}) go func() { runConvergeLoop(func() { calls <- struct{}{} }, 5*time.Millisecond, stop) close(done) }() for i := 0; i < 3; i++ { select { case <-calls: case <-time.After(10 * time.Second): t.Fatalf("the loop ran the leg %d times, want it to keep running", i) } } close(stop) select { case <-done: case <-time.After(10 * time.Second): t.Fatal("the loop did not stop") } } // The production sequence behind discussion #231, with the real pass on both // sides. The agent cannot write the version store, so the ROOT pass adopts a // new release for a warm exec instance: it stages, repoints and stamps, and — // having no handle on the agent's children — cycles nothing. The agent's own // pass then finds the host up to date and never reaches its restart callback. // Before the converge leg, that left the child on the old build until the // activator sweep's 90s grace ran out. func TestRootAdoptionOfAWarmExecInstanceIsFinishedByTheAgent(t *testing.T) { env := newFastFixture(t) ctx := context.Background() plat := runtime.GOOS + "-" + runtime.GOARCH host := "warm.test" ship := func(ver string) { t.Helper() m := release.Manifest{ App: "fastapp", Version: ver, Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}, } if err := release.Ship(ctx, env.store, "bab", m, map[string][]byte{plat: []byte("build " + ver)}); err != nil { t.Fatalf("Ship %s: %v", ver, err) } if err := release.Promote(ctx, env.store, env.priv, "bab", "fastapp", "edge", ver); err != nil { t.Fatalf("Promote %s: %v", ver, err) } } // rootRestartRoute with systemctl stubbed out: same dispatch, bounce nil. rootPass := func() { t.Helper() restart := makeRestartRoute(env.reg, func(string) {}, unitOnly(func(string) error { return nil }), nil) if err := release.Pass(ctx, env.reg, env.store, env.pub, env.optDir, restart, release.BakeDir(env.dataDir)); err != nil { t.Fatalf("root pass: %v", err) } } if err := env.reg.Upsert(registry.Route{ Host: host, Kind: "instance", App: "fastapp", Channel: "edge", Status: "ready", Hibernate: true, }); err != nil { t.Fatalf("upsert: %v", err) } ship("v1") rootPass() spawned := "v1" // the awake child's build: frozen at exec, as the activator records it ship("v2") rootPass() if v, err := release.LiveVersion(env.optDir, host); err != nil || v != "v2" { t.Fatalf("live = %q, %v after the root pass; want v2", v, err) } if rt, err := env.reg.Get(host); err != nil || rt == nil || rt.Version != "v2" { t.Fatalf("route = %+v, %v after the root pass; want it stamped v2", rt, err) } // The agent's tick. Its pass has a bounce to offer and never gets to use // it: the host is already up to date. var bounced, cycled []string restart := makeRestartRoute(env.reg, func(string) {}, unitOnly(func(string) error { return nil }), func(h string) bool { bounced = append(bounced, h); return true }) d := env.deps(nil) d.restart = restart d.passOpts = fastPassOpts(env.dataDir, func(string, string, error) {}, restart, nil) d.fullPass = func(ctx context.Context) { if err := release.Pass(ctx, d.reg, d.store, d.pub, d.optDir, d.restart, d.passOpts...); err != nil { t.Fatalf("agent pass: %v", err) } } leg := func() { convergeAdopted( func() map[string]string { live, _ := release.LiveVersion(env.optDir, host) if live == "" || live == spawned { return nil } return map[string]string{host: live} }, env.reg.Get, func(h, ver string) bool { cycled = append(cycled, h+"@"+ver); spawned = ver; return true }, adoptedCyclesPerTick) } now := time.Unix(1754640000, 0) fastTick(ctx, d, now) // seeds the serial if err := release.BumpSerial(ctx, env.store, now); err != nil { t.Fatalf("BumpSerial: %v", err) } fastTick(ctx, d, now.Add(10*time.Second)) // the serial move: the agent's full pass leg() // the converge loop's next run if len(bounced) != 0 { t.Fatalf("the agent's pass bounced %v — it found the host already adopted, so this test no longer models the gap", bounced) } if want := []string{host + "@v2"}; !reflect.DeepEqual(cycled, want) { t.Fatalf("cycled = %v, want %v: the warm child is still on v1 under a symlink and a stamp that both say v2", cycled, want) } leg() if len(cycled) != 1 { t.Fatalf("cycled again on the next tick: %v", cycled) } } // One pass can adopt every warm instance of an app inside a few seconds. // Stopping them all on one tick would blip every host of the app at the same // moment; the leg spends its budget in host order and leaves the rest for the // ticks that follow. A drifted host it has to wait on costs the others nothing. func TestConvergeAdoptedCyclesOnePerTickInHostOrder(t *testing.T) { spawned := map[string]string{"a.test": "v1", "b.test": "v1", "c.test": "v1", "wait.test": "v1"} stamp := map[string]string{"a.test": "v2", "b.test": "v2", "c.test": "v2", "wait.test": "v1"} drifted := func() map[string]string { out := map[string]string{} for h, v := range spawned { if v != "v2" { out[h] = "v2" } } return out } get := func(h string) (*registry.Route, error) { return ®istry.Route{Host: h, Version: stamp[h]}, nil } var cycled []string converge := func(h, ver string) bool { cycled = append(cycled, h); spawned[h] = ver; return true } for tick := 1; tick <= 4; tick++ { if n := convergeAdopted(drifted, get, converge, adoptedCyclesPerTick); tick <= 3 && n != 1 { t.Fatalf("tick %d cycled %d, want 1", tick, n) } } if want := []string{"a.test", "b.test", "c.test"}; !reflect.DeepEqual(cycled, want) { t.Fatalf("cycled = %v, want %v: one per tick, in host order, never the host whose stamp does not agree", cycled, want) } } // versionedBackend is activator.Backend with the one thing convergence needs // and fakeWakeBackend does not model: a child that knows which build it was // exec'd from, read off the SAME live symlink the release pass writes. It // binds no socket — the caller supplies the proxy — so it runs anywhere. type versionedBackend struct { fakeWakeBackend optDir string mu sync.Mutex starts []string // the build each spawn exec'd, in order } type flagProc struct { mu sync.Mutex stopped bool } func (p *flagProc) Stop(context.Context) error { p.mu.Lock() p.stopped = true p.mu.Unlock() return nil } func (p *flagProc) Exited() bool { p.mu.Lock() defer p.mu.Unlock() return p.stopped } func (b *versionedBackend) LiveVersion(host string) (string, error) { return release.LiveVersion(b.optDir, host) } func (b *versionedBackend) Start(_ context.Context, rt registry.Route) (activator.Proc, string, error) { ver, err := release.LiveVersion(b.optDir, rt.Host) if err != nil { return nil, "", err } b.mu.Lock() b.starts = append(b.starts, ver) b.mu.Unlock() return &flagProc{}, ver, nil } func (b *versionedBackend) started() []string { b.mu.Lock() defer b.mu.Unlock() return append([]string(nil), b.starts...) } // The whole of discussion #231 with nothing stood in for: the real release // pass as root runs it, a real activator owning the child, and the leg exactly // as fastConvergeLoop builds it (convergeLegFor). The sweeper is never // started, so the ONLY thing that can cycle the instance is the leg — a 90s // grace cannot rescue this test. func TestConvergeLegCyclesARootAdoptedInstanceOnARealActivator(t *testing.T) { t.Parallel() env := newFastFixture(t) ctx := context.Background() plat := runtime.GOOS + "-" + runtime.GOARCH host := "warm.test" rt := registry.Route{ Host: host, Kind: "instance", App: "fastapp", Channel: "edge", Status: "ready", Hibernate: true, } if err := env.reg.Upsert(rt); err != nil { t.Fatalf("upsert: %v", err) } adopt := func(ver string) { t.Helper() m := release.Manifest{ App: "fastapp", Version: ver, Kind: "binary", Artifacts: map[string]release.Artifact{plat: {}}, } if err := release.Ship(ctx, env.store, "bab", m, map[string][]byte{plat: []byte("build " + ver)}); err != nil { t.Fatalf("Ship %s: %v", ver, err) } if err := release.Promote(ctx, env.store, env.priv, "bab", "fastapp", "edge", ver); err != nil { t.Fatalf("Promote %s: %v", ver, err) } // The root pass: rootRestartRoute's dispatch (bounce nil), systemctl stubbed. restart := makeRestartRoute(env.reg, func(string) {}, unitOnly(func(string) error { return nil }), nil) if err := release.Pass(ctx, env.reg, env.store, env.pub, env.optDir, restart, release.BakeDir(env.dataDir)); err != nil { t.Fatalf("root pass for %s: %v", ver, err) } } backend := &versionedBackend{optDir: env.optDir} svc := activator.New(activator.Config{Bucket: "test-bucket", Idle: time.Hour}, backend, env.reg) t.Cleanup(func() { sctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() svc.Shutdown(sctx) }) request := func() { t.Helper() rec := httptest.NewRecorder() proxy := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) if !svc.ServeHTTP(rec, httptest.NewRequest("GET", "http://"+host+"/", nil), rt, proxy) { t.Fatal("the activator did not take the request") } if rec.Code != http.StatusNoContent { t.Fatalf("request = %d, want 204", rec.Code) } } leg := convergeLegFor(svc, env.reg) if leg == nil { t.Fatal("convergeLegFor returned no leg for a box with an activator") } // "Was it stopped?" is asked of the spawn count, after a request: Owns // stays true while an entry drains, so it cannot tell a spared child from // one being stopped. A request against a stopped instance spawns again. adopt("v1") request() // wakes the instance onto v1 leg() request() if got, want := backend.started(), []string{"v1"}; !reflect.DeepEqual(got, want) { t.Fatalf("spawned builds = %v, want %v: the leg stopped an instance whose build is the adopted one", got, want) } adopt("v2") // root: staged, repointed, stamped — and nothing cycled if !svc.Owns(host) { t.Fatal("the root pass stopped the agent's child — this test no longer models the gap") } leg() deadline := time.Now().Add(10 * time.Second) for svc.Owns(host) { if time.Now().After(deadline) { t.Fatal("the warm instance is still running v1 after the leg: it was adopted onto v2 and nothing cycled it") } time.Sleep(5 * time.Millisecond) } request() // the next request wakes it onto v2 if got, want := backend.started(), []string{"v1", "v2"}; !reflect.DeepEqual(got, want) { t.Fatalf("spawned builds = %v, want %v", got, want) } leg() request() if got, want := backend.started(), []string{"v1", "v2"}; !reflect.DeepEqual(got, want) { t.Fatalf("spawned builds = %v, want %v: the leg cycled the instance again after it had converged", got, want) } } func TestConvergeLegForABoxWithNoActivator(t *testing.T) { if convergeLegFor(nil, nil) != nil { t.Fatal("convergeLegFor built a leg with no activator to ask") } } // The adopt passes' restart dispatch must be built on adoptBounceFn, not // bounceFn. With the converge leg in place an instance can already be on the // new build when a slower pass reaches its own restart; the plain Bounce would // stop that replacement for nothing. A stale child still gets its cycle. func TestAdoptRestartDispatchSparesAConvergedChild(t *testing.T) { env := newFastFixture(t) host := "warm.test" rt := registry.Route{ Host: host, Kind: "instance", App: "fastapp", Channel: "edge", Status: "ready", Hibernate: true, } if err := env.reg.Upsert(rt); err != nil { t.Fatalf("upsert: %v", err) } env.stageLive(t, host, "v1") backend := &versionedBackend{optDir: env.optDir} svc := activator.New(activator.Config{Bucket: "test-bucket", Idle: time.Hour}, backend, env.reg) t.Cleanup(func() { sctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() svc.Shutdown(sctx) }) wake := func() { t.Helper() rec := httptest.NewRecorder() proxy := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) if !svc.ServeHTTP(rec, httptest.NewRequest("GET", "http://"+host+"/", nil), rt, proxy) || rec.Code != http.StatusNoContent { t.Fatalf("wake request = %d", rec.Code) } } restart := makeRestartRoute(env.reg, func(string) {}, unitOnly(func(u string) error { t.Fatalf("named-unit arm fired for %q on a route that names no unit", u) return nil }), adoptBounceFn(svc)) // Asked of the spawn count, after a request — see the note in // TestConvergeLegCyclesARootAdoptedInstanceOnARealActivator. wake() // exec'd from v1, and live/ names v1 if err := restart(host); err != nil { t.Fatalf("restart: %v", err) } wake() if got, want := backend.started(), []string{"v1"}; !reflect.DeepEqual(got, want) { t.Fatalf("spawned builds = %v, want %v: the adopt dispatch stopped a child already running the build live/ names", got, want) } env.stageLive(t, host, "v2") // the symlink moves under the running child if err := restart(host); err != nil { t.Fatalf("restart: %v", err) } wake() if got, want := backend.started(), []string{"v1", "v2"}; !reflect.DeepEqual(got, want) { t.Fatalf("spawned builds = %v, want %v: the adopt dispatch left a stale child running", got, want) } } // A leg past slowLegAfter says so in the journal, with what it was doing; a // quick one says nothing, because the fast loop runs every two seconds. func TestLogSlowSpeaksOnlyWhenSlow(t *testing.T) { var buf bytes.Buffer prev, flags := log.Writer(), log.Flags() log.SetOutput(&buf) log.SetFlags(0) t.Cleanup(func() { log.SetOutput(prev); log.SetFlags(flags) }) logSlow("fast sweep", time.Now(), "3 routes") if buf.Len() != 0 { t.Fatalf("a quick leg logged %q", buf.String()) } logSlow("serial pass stages", time.Now().Add(-2*slowLegAfter), "reconcile 9.0s, adopt 1.0s") got := buf.String() for _, want := range []string{"serial pass stages took 10", "(reconcile 9.0s, adopt 1.0s)"} { if !strings.Contains(got, want) { t.Fatalf("slow-leg line %q lacks %q", got, want) } } } // The point of the leg's own loop: it acts while the fast tick is stuck in a // long full pass. The pass is held open here until the leg has cycled the // drifted instance, which it could never do if it still rode the tick. func TestConvergeLoopActsWhileTheFullPassIsBusy(t *testing.T) { env := newFastFixture(t) d := env.deps(nil) inPass, unblock := make(chan struct{}), make(chan struct{}) d.fullPass = func(context.Context) { close(inPass); <-unblock } cycled := make(chan string, 1) leg := func() { convergeAdopted( func() map[string]string { return map[string]string{"warm.test": "v2"} }, func(h string) (*registry.Route, error) { return ®istry.Route{Host: h, Version: "v2"}, nil }, func(h, ver string) bool { select { case cycled <- h + "@" + ver: default: } return true }, adoptedCyclesPerTick) } ctx := context.Background() now := time.Unix(1754640000, 0) fastTick(ctx, d, now) // seeds the serial if err := release.BumpSerial(ctx, env.store, now); err != nil { t.Fatalf("BumpSerial: %v", err) } tickDone := make(chan struct{}) go func() { fastTick(ctx, d, now.Add(10*time.Second)); close(tickDone) }() <-inPass // the tick is now stuck in its full pass // Started only now, so anything it cycles was cycled DURING the pass. stop := make(chan struct{}) defer close(stop) go runConvergeLoop(leg, 5*time.Millisecond, stop) select { case got := <-cycled: if got != "warm.test@v2" { t.Fatalf("cycled %q", got) } case <-time.After(10 * time.Second): t.Fatal("the converge leg waited for the full pass") } close(unblock) <-tickDone }