package activator import ( "context" "errors" "strings" "testing" "time" ) func TestObserveSettle(t *testing.T) { t.Parallel() if r := observeSettle(context.Background(), &fakeProc{}, 20*time.Millisecond); r != settleOK { t.Fatalf("a process that stayed up: %v", r) } // A process that dies inside the window: alive at the first check, dead // at the observation's next poll (no sleep places the death). p := &fakeProc{} p.exitOnPoll(2) if r := observeSettle(context.Background(), p, time.Second); r != settleExited { t.Fatalf("a process that died: %v", r) } // An observation cancelled inside the window: the cancel happens at its // first poll after the opening check. ctx, cancel := context.WithCancel(context.Background()) defer cancel() if r := observeSettle(ctx, &cancelOnPoll{at: 2, cancel: cancel}, time.Second); r != settleCancelled { t.Fatalf("a cancelled observation: %v", r) } // Cancellation wins over an exit seen at the same moment (spec §6.4). dead := &fakeProc{} dead.exit() gone, stop := context.WithCancel(context.Background()) stop() if r := observeSettle(gone, dead, time.Second); r != settleCancelled { t.Fatalf("cancelled and exited: %v, want cancelled", r) } } // cancelOnPoll is a live process whose at-th Exited() call cancels the // observation watching it. type cancelOnPoll struct { polls int at int cancel context.CancelFunc } func (p *cancelOnPoll) Stop(context.Context) error { return nil } func (p *cancelOnPoll) Exited() bool { p.polls++ if p.polls == p.at { p.cancel() } return false } func TestSwapCanarySettle(t *testing.T) { old := swapSettle t.Cleanup(func() { swapSettle = old }) t.Run("exit during the settle fails the build", func(t *testing.T) { swapSettle = 2 * time.Second f := newSwapFixture(t) f.a.genHealth = func(context.Context, Gen) error { // Alive at the settle's opening check, dead at its next poll. _, p := f.b.lastGen() p.exitOnPoll(2) return nil } at := f.attempt(ModeOverlap) at.Canary = true res := f.a.Swap(context.Background(), at, f.newest) if res.Outcome != SwapFailed || !strings.Contains(res.Reason, errSettleExit.Error()) { t.Fatalf("Swap = %+v", res) } if got := f.appSocket(t); got != f.incumbent.Socket { t.Fatalf("AppSocket = %q, want the incumbent", got) } }) t.Run("a canary waits out the settle", func(t *testing.T) { swapSettle = 100 * time.Millisecond f := newSwapFixture(t) at := f.attempt(ModeOverlap) at.Canary = true start := time.Now() if res := f.a.Swap(context.Background(), at, f.newest); res.Outcome != SwapDone { t.Fatalf("Swap = %+v", res) } if took := time.Since(start); took < swapSettle { t.Fatalf("switched after %s, inside the %s settle", took, swapSettle) } }) t.Run("a swap after the canary does not settle", func(t *testing.T) { swapSettle = time.Hour f := newSwapFixture(t) done := make(chan SwapResult, 1) go func() { done <- f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest) }() select { case res := <-done: if res.Outcome != SwapDone { t.Fatalf("Swap = %+v", res) } case <-time.After(5 * time.Second): t.Fatal("a non-canary swap waited for a settle") } }) } // An intentional stop of the entry during the settle cancels the // observation and is not a failure (spec §6.4). The successor is stopped // before the incumbent, through the barrier. func TestSwapIntentionalStopDuringSettleIsNotAFailure(t *testing.T) { old := swapSettle swapSettle = 5 * time.Second t.Cleanup(func() { swapSettle = old }) for _, tc := range []struct { name string stop func(f *swapFixture) }{ {"restart", func(f *swapFixture) { f.a.Bounce(f.host) }}, {"hibernate", func(f *swapFixture) { f.clk.advance(2 * time.Hour); f.a.sweepOnce() }}, {"lease lost", func(f *swapFixture) { f.b.lastRep().setErr(errors.New("renewal failed")); f.a.sweepOnce() }}, {"maintenance fence", func(f *swapFixture) { go func() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() f.a.FenceAndStop(ctx, f.host, 1) //nolint:errcheck }() }}, {"move or park retire", func(f *swapFixture) { f.a.RetireExcept(map[string]bool{}) }}, {"shutdown", func(f *swapFixture) { go f.a.Shutdown(context.Background()) }}, } { t.Run(tc.name, func(t *testing.T) { f := newSwapFixture(t) healthy := make(chan struct{}) f.a.genHealth = func(context.Context, Gen) error { close(healthy); return nil } rep := f.b.lastRep() at := f.attempt(ModeOverlap) at.Canary = true results := make(chan SwapResult, 1) go func() { results <- f.a.Swap(context.Background(), at, f.newest) }() <-healthy // The settle has begun once it has polled the successor (nothing // else asks it between the health check and the settle). _, succ := f.b.lastGen() waitFor(t, "the settle's first poll", func() bool { return succ.pollCount() > 0 }) tc.stop(f) var res SwapResult select { case res = <-results: case <-time.After(10 * time.Second): t.Fatal("Swap did not return after the stop") } if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapStopped.Error()) { t.Fatalf("Swap = %+v, want abandoned by the stop", res) } waitFor(t, "the lease release", rep.isClosed) ev := f.b.events() if i, j := indexPrefix(ev, "stop:gen:"), indexPrefix(ev, "stop:instance"); i < 0 || j < 0 || i > j { t.Fatalf("order %v: the successor must stop before the incumbent", ev) } }) } }