package activator import ( "errors" "fmt" "testing" "time" ) // TestFenceAndStopAsMeetsOnlyItsOwnFence: an owned fence is the caller's // again only when it holds at the same generation with the same owner. // Any other fence (a plain one at a lower, equal or higher generation, or // another owner's at the same generation) refuses with ErrFenceHeld and // stays exactly as it was, with nothing stopped. func TestFenceAndStopAsMeetsOnlyItsOwnFence(t *testing.T) { t.Parallel() t.Run("own fence, re-collected", func(t *testing.T) { a, e, _, _, _ := maintenanceAwake(t) proof, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1") if err != nil || !proof.Matches(e.host, 12) { t.Fatalf("first install: %v", err) } again, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1") if err != nil || !again.Matches(e.host, 12) { t.Fatalf("the owner's repeat: %v", err) } if g := a.FenceGeneration(e.host); g != 12 { t.Fatalf("generation %d, want 12", g) } }) type held struct { gen uint64 owner string } for _, h := range []held{{5, ""}, {12, ""}, {20, ""}, {12, "owner-2"}, {5, "owner-1"}} { t.Run(fmt.Sprintf("held %d by %q", h.gen, h.owner), func(t *testing.T) { a, e, main, _, _ := maintenanceAwake(t) a.mu.Lock() a.fences = map[string]uint64{e.host: h.gen} if h.owner != "" { a.fenceOwners = map[string]string{e.host: h.owner} } a.mu.Unlock() if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1"); !errors.Is(err, ErrFenceHeld) { t.Fatalf("err = %v, want ErrFenceHeld", err) } a.mu.Lock() gen, owner := a.fences[e.host], a.fenceOwners[e.host] a.mu.Unlock() if gen != h.gen || owner != h.owner { t.Fatalf("held fence is %d by %q, want %d by %q", gen, owner, h.gen, h.owner) } if main.calls.Load() != 0 || !a.Owns(e.host) { t.Fatal("a refused owned fence stopped the host") } }) } t.Run("an empty owner", func(t *testing.T) { a, e, _, _, _ := maintenanceAwake(t) if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, ""); !errors.Is(err, ErrFenceGeneration) || a.Fenced(e.host) { t.Fatalf("err = %v, fenced %v", err, a.Fenced(e.host)) } }) } // TestAPlainFenceTakesOwnershipAway: FenceAndStop at the same generation // over an owned fence leaves a fence that is no longer the owner's, so // the owner can neither re-collect under it nor lift it. func TestAPlainFenceTakesOwnershipAway(t *testing.T) { a, e, _, _, _ := maintenanceAwake(t) if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1"); err != nil { t.Fatal(err) } if _, err := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 12); !errors.Is(err, ErrStopUnproven) && err != nil { t.Fatal(err) } if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1"); !errors.Is(err, ErrFenceHeld) { t.Fatalf("err = %v, want ErrFenceHeld", err) } if a.LiftFenceAs(e.host, 12, "owner-1") { t.Fatal("the owner lifted a fence it no longer owns") } if !a.Fenced(e.host) { t.Fatal("the plain fence was lifted") } } // TestLiftFenceAsLiftsOnlyTheOwnersFence: the lift names the generation // and the owner, and a plain LiftFence also clears the owner. func TestLiftFenceAsLiftsOnlyTheOwnersFence(t *testing.T) { a, e, _, _, _ := maintenanceAwake(t) if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1"); err != nil { t.Fatal(err) } if a.LiftFenceAs(e.host, 12, "owner-2") || a.LiftFenceAs(e.host, 11, "owner-1") || a.LiftFenceAs(e.host, 12, "") { t.Fatal("lifted with the wrong owner or generation") } if !a.LiftFenceAs(e.host, 12, "owner-1") || a.Fenced(e.host) { t.Fatal("the owner's lift did not lift its fence") } if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 13, "owner-1"); err != nil && !errors.Is(err, ErrStopUnproven) { t.Fatal(err) } if !a.LiftFence(e.host, 13) { t.Fatal("plain lift") } a.mu.Lock() _, left := a.fenceOwners[e.host] a.mu.Unlock() if left { t.Fatal("a plain lift left the owner behind") } } // TestAnOwnedCallNeverTakesBackAFenceReplacedWhileItWaited (fix round 1, // M1; ruling 75): FenceAndStopAs installs its owned fence and waits for // the stop. Meanwhile a plain FenceAndStop installs at the same generation, // or a plain LiftFence lifts it. When the stop completes the owned call // answers ErrStopUnproven and leaves the fence as the other caller left // it: still plain and unowned, or still lifted, never the owner's again. func TestAnOwnedCallNeverTakesBackAFenceReplacedWhileItWaited(t *testing.T) { t.Parallel() for _, meanwhile := range []string{"a plain fence", "a plain lift"} { t.Run(meanwhile, func(t *testing.T) { a, e, _, _, _ := maintenanceAwake(t) stopped := make(chan struct{}) a.mu.Lock() e.state, e.stopped = actStopping, stopped a.mu.Unlock() type answer struct { proof *StopEvidence err error } done := make(chan answer, 1) go func() { proof, err := a.FenceAndStopAs(maintenanceContext(t, 30*time.Second), e.host, 12, "owner-1") done <- answer{proof, err} }() for deadline := time.Now().Add(5 * time.Second); ; { a.mu.Lock() installed := a.fences[e.host] == 12 && a.fenceOwners[e.host] == "owner-1" a.mu.Unlock() if installed { break } if time.Now().After(deadline) { t.Fatal("the owned call never installed its fence") } time.Sleep(time.Millisecond) } switch meanwhile { case "a plain fence": // The stop is still in progress, so the plain call installs and // then waits out its own deadline: ErrStopUnproven. if _, err := a.FenceAndStop(maintenanceContext(t, 500*time.Millisecond), e.host, 12); !errors.Is(err, ErrStopUnproven) { t.Fatalf("the plain fence: %v", err) } case "a plain lift": if !a.LiftFence(e.host, 12) { t.Fatal("the plain lift lifted nothing") } } a.mu.Lock() delete(a.entries, e.host) a.mu.Unlock() close(stopped) var got answer select { case got = <-done: case <-time.After(30 * time.Second): t.Fatal("the owned call never answered after the stop completed") } if !errors.Is(got.err, ErrStopUnproven) || got.proof != nil { t.Fatalf("the owned call after its fence was replaced: %v %v", got.proof, got.err) } a.mu.Lock() gen := a.fences[e.host] owner, owned := a.fenceOwners[e.host] a.mu.Unlock() want := uint64(12) if meanwhile == "a plain lift" { want = 0 } if gen != want || owned { t.Fatalf("the fence is %d owned by %q (%v), want %d unowned", gen, owner, owned, want) } }) } }