package release import ( "context" "crypto/ed25519" "errors" "testing" "time" "github.com/carlosframework/platform/internal/registry" ) // Settings plan 4, Astra round 2 finding 2 (ruling R-CARRY-FENCE): a root // pass already running when a graceful cut begins publishes its request // record before the cut's carry, and is carried, or not at all. A late // record is never left behind on the source box. // // The root pass is the real one (Pass with the update settings, root's // resolve). It is paused by the test hook at one of the two points of its // fenced publication. The cut's half is the agent's carry as // buildStates.carryRequest runs it: the row held moving first, then the // fence (FenceCarry), the consumed file read and merged (CarryRequest), the // fence released. const fenceHost = "a.example.com" // fenceCutResult is what the cut sealed: the build state after its carry. type fenceCutResult struct { state BuildState err error } // fenceCut runs the cut's half: marked is closed once the row is held. func fenceCut(reg *registry.Registry, opt string, state BuildState, marked chan<- struct{}) fenceCutResult { if err := reg.SetStatus(fenceHost, "moving"); err != nil { return fenceCutResult{err: err} } close(marked) unlock, err := FenceCarry(context.Background(), opt, fenceHost) if err != nil { return fenceCutResult{err: err} } defer unlock() c, err := ReadConsumed(opt, fenceHost) if err != nil { return fenceCutResult{err: err} } return fenceCutResult{state: CarryRequest(&state, c)} } // fenceScenario is one root pass over one route: setup prepares the store, // registry and consumed file and returns the build state the cut would // seal and the pass options. type fenceScenario struct { name string setup func(t *testing.T, s Store, priv ed25519.PrivateKey, opt string, reg *registry.Registry) (BuildState, []PassOption) // published reports whether the consumed file holds the pass's record. published func(c *Consumed) bool } func fenceScenarios() []fenceScenario { ctx := context.Background() return []fenceScenario{ { // Update now, resolved by the pass to the head before staging. name: "a resolution", setup: func(t *testing.T, s Store, priv ed25519.PrivateKey, opt string, reg *registry.Registry) (BuildState, []PassOption) { shipRelease(t, s, "bab", "km", "v1", adoptPlatform, []byte("v1")) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", "v1"); err != nil { t.Fatal(err) } st := BuildState{LastExecuted: "v0"} settingsRoute(t, reg, fenceHost, "km", "canary/x", UpdateSettings{Updates: UpdatesManual, Channel: "canary/x", InstallRequest: 1}, BuildStateCopy{Known: true, Replica: true, State: &st}) return st, nil }, published: func(c *Consumed) bool { return c != nil && c.Resolved != nil && c.Resolved.ID == 1 }, }, { // R1 resolved to v2, which the failure book holds: the pass // records R1 completed, failed. name: "a failed completion", setup: func(t *testing.T, s Store, priv ed25519.PrivateKey, opt string, reg *registry.Registry) (BuildState, []PassOption) { for _, v := range []string{"v1", "v2"} { shipRelease(t, s, "bab", "km", v, adoptPlatform, []byte(v)) if err := Promote(ctx, s, priv, "bab", "km", "canary/x", v); err != nil { t.Fatal(err) } } st := BuildState{LastExecuted: "v1", Resolved: &ResolvedRequest{ID: 1, Target: "v2"}} settingsRoute(t, reg, fenceHost, "km", "canary/x", UpdateSettings{Updates: UpdatesManual, Channel: "canary/x", InstallRequest: 1}, BuildStateCopy{Known: true, Replica: true, State: &st}) if err := WriteConsumed(opt, Consumed{Host: fenceHost, Resolved: &ResolvedRequest{ID: 1, Target: "v2"}}); err != nil { t.Fatal(err) } book := func() (FailedBuild, error) { return func(_, _, build, _ string, _ bool) bool { return build == "v2" }, nil } return st, []PassOption{WithUpdateSettings(book, func(registry.Route) bool { return false }, true)} }, published: func(c *Consumed) bool { return c != nil && c.Completed == 1 && c.CompletedFailed }, }, } } func withFenceHook(t *testing.T, hook func(host, at string)) { t.Helper() old := beforeConsumedWrite beforeConsumedWrite = hook t.Cleanup(func() { beforeConsumedWrite = old }) } // leftBehind reports whether the source's consumed file holds request state // the sealed build state lacks: carrying it again would change the state. func leftBehind(t *testing.T, opt string, sealed BuildState) bool { t.Helper() c, err := ReadConsumed(opt, fenceHost) if err != nil { t.Fatal(err) } again := CarryRequest(&sealed, c) return !SameRequestState(&sealed, &again) } // A cut that holds the row and carries while root's pass is paused before // its fence: root then takes the fence, sees the hold, publishes nothing and // installs nothing. The record is never published, so nothing is left behind. func TestARootPassPausedBeforeTheFenceDoesNotPublishOverACut(t *testing.T) { for _, sc := range fenceScenarios() { t.Run(sc.name, func(t *testing.T) { s := NewDirStore(t.TempDir()) priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) state, opts := sc.setup(t, s, priv, opt, reg) before, _ := ReadConsumed(opt, fenceHost) var cut fenceCutResult withFenceHook(t, func(host, at string) { if at == "fence" { cut = fenceCut(reg, opt, state, make(chan struct{})) } }) passWithSettings(t, reg, s, pub, opt, opts...) if cut.err != nil { t.Fatalf("the cut: %v", cut.err) } after, _ := ReadConsumed(opt, fenceHost) if sc.published(after) && !sc.published(before) { t.Fatalf("root published %+v after the cut held the row and carried", after) } if leftBehind(t, opt, cut.state) { t.Fatalf("a record was left behind: consumed %+v, sealed %+v", after, cut.state) } if v, _ := LiveVersion(opt, fenceHost); v != "" { t.Fatalf("live %q: root installed for a route the cut holds", v) } }) } } // A root pass paused immediately before WriteConsumed, its fence held and // its re-check passed, while a cut holds the row and asks for the fence: // the cut waits, root's record lands, and the cut carries it. func TestARootPassPausedBeforeItsWriteIsCarriedByTheCut(t *testing.T) { for _, sc := range fenceScenarios() { t.Run(sc.name, func(t *testing.T) { s := NewDirStore(t.TempDir()) priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) state, opts := sc.setup(t, s, priv, opt, reg) done := make(chan fenceCutResult, 1) early := false withFenceHook(t, func(host, at string) { if at != "write" { return } marked := make(chan struct{}) go func() { done <- fenceCut(reg, opt, state, marked) }() <-marked select { case <-done: early = true case <-time.After(200 * time.Millisecond): } }) passWithSettings(t, reg, s, pub, opt, opts...) if early { t.Fatal("the cut carried while root held the fence") } var cut fenceCutResult select { case cut = <-done: case <-time.After(10 * time.Second): t.Fatal("the cut never took the fence") } if cut.err != nil { t.Fatalf("the cut: %v", cut.err) } after, _ := ReadConsumed(opt, fenceHost) if !sc.published(after) { t.Fatalf("consumed %+v: root's record did not land", after) } if leftBehind(t, opt, cut.state) { t.Fatalf("a record was left behind: consumed %+v, sealed %+v", after, cut.state) } }) } } // With no move in progress the fenced pass is the pass it was: the hook runs // at both points and the record lands as without it. func TestARootPassWithNoMoveIsUnchangedByTheFence(t *testing.T) { for _, sc := range fenceScenarios() { t.Run(sc.name, func(t *testing.T) { var want *Consumed var wantLive string for _, hooked := range []bool{false, true} { s := NewDirStore(t.TempDir()) priv, pub := testKeys(t) opt, reg := t.TempDir(), testRegistry(t) _, opts := sc.setup(t, s, priv, opt, reg) var seen []string if hooked { withFenceHook(t, func(_, at string) { seen = append(seen, at) }) } passWithSettings(t, reg, s, pub, opt, opts...) got, err := ReadConsumed(opt, fenceHost) if err != nil || !sc.published(got) { t.Fatalf("hooked %v: consumed %+v, %v; want the pass's record", hooked, got, err) } live, _ := LiveVersion(opt, fenceHost) if !hooked { want, wantLive = got, live continue } if len(seen) != 2 || seen[0] != "fence" || seen[1] != "write" { t.Fatalf("hook points %v, want fence then write", seen) } g, w := *got, *want g.At, w.At = time.Time{}, time.Time{} if !sameConsumedForTest(g, w) || live != wantLive { t.Fatalf("fenced pass: consumed %+v live %q; unfenced: %+v live %q", g, live, w, wantLive) } } }) } } func sameConsumedForTest(a, b Consumed) bool { ra, rb := a.Resolved, b.Resolved a.Resolved, b.Resolved = nil, nil return a == b && ((ra == nil && rb == nil) || (ra != nil && rb != nil && *ra == *rb)) } // publishGate: a held or missing row is refused; a ready one is not. func TestPublishGateRefusesAMoveHold(t *testing.T) { for _, st := range []string{"moving", "landing", "restoring", "releasing"} { if err := publishGate(®istry.Route{Host: fenceHost, Status: st}); !errors.Is(err, ErrMoveHeld) { t.Errorf("status %q: %v; want ErrMoveHeld", st, err) } } if err := publishGate(nil); !errors.Is(err, ErrMoveHeld) { t.Errorf("no row: %v; want ErrMoveHeld", err) } for _, st := range []string{"", "ready"} { if err := publishGate(®istry.Route{Host: fenceHost, Status: st}); err != nil { t.Errorf("status %q: %v; want nil", st, err) } } }