package release import ( "bytes" "context" "crypto/ed25519" "errors" "os" "path/filepath" "strings" "testing" "github.com/carlosframework/platform/internal/hostwrite" "github.com/carlosframework/platform/internal/registry" ) func groupFixtureRoute(t *testing.T, reg *registry.Registry, host, socket, addr, unit string) { t.Helper() if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", App: "console", Channel: "canary/x", Socket: socket, Addr: addr, Unit: unit}); err != nil { t.Fatalf("upsert %s: %v", host, err) } } func TestProcessGroupKeysOnTheEffectiveUpstream(t *testing.T) { reg := testRegistry(t) groupFixtureRoute(t, reg, "platform.example.com", "", "127.0.0.1:8090", "console.service") groupFixtureRoute(t, reg, "console.example.com", "", "127.0.0.1:8090", "") groupFixtureRoute(t, reg, "other.example.com", "", "127.0.0.1:9000", "") groupFixtureRoute(t, reg, "sock.example.com", "/run/x.sock", "127.0.0.1:8090", "") bound, _ := reg.Get("platform.example.com") got := processGroup(reg, *bound) if len(got) != 1 || got[0].Host != "console.example.com" { t.Fatalf("group = %+v, want only console.example.com (socket wins over addr; 9000 is another process)", got) } } func TestProcessGroupNeverMatchesEmptyUpstreams(t *testing.T) { reg := testRegistry(t) groupFixtureRoute(t, reg, "a.example.com", "", "", "a.service") groupFixtureRoute(t, reg, "b.example.com", "", "", "") bound, _ := reg.Get("a.example.com") if got := processGroup(reg, *bound); len(got) != 0 { t.Fatalf("group = %+v, want none: two empty upstreams are not one process", got) } } // The flagship shape: three hosts, one process, the unit runs the LAST // host's link. Hosts before it were repointed and stamped; the bound host's // restart reports the unit did not come up. func TestNotUpRollsTheWholeGroupBackAndRefusesEveryHost(t *testing.T) { ctx, reg, s, pub, opt := startUnitFixture(t, "console.example.com") if err := reg.SetUnit("console.example.com", ""); err != nil { t.Fatal(err) } groupFixtureRoute(t, reg, "platform.example.com", "", "127.0.0.1:8090", "console.service") if err := reg.SetUpstreams("console.example.com", "127.0.0.1:8090", "", ""); err != nil { t.Fatal(err) } if err := PointLive(opt, "platform.example.com", "bab", "old", "console"); err != nil { t.Fatal(err) } bake := t.TempDir() restart := func(h string) error { if h == "platform.example.com" { return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"} } return nil } var rolledBackHost string rb := func(h string) error { rolledBackHost = h; return nil } var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), WithRollbackRestart(rb)); err != nil { t.Fatalf("Pass: %v", err) } }) for _, h := range []string{"console.example.com", "platform.example.com"} { if v, _ := LiveVersion(opt, h); v != "old" { t.Fatalf("%s live = %q, want old", h, v) } rt, _ := reg.Get(h) if rt.Version == "new" { t.Fatalf("%s still stamped new: %+v", h, rt) } if r, _ := ReadRefusal(bake, h); r == nil || r.Version != "new" { t.Fatalf("%s refusal = %+v, want one naming new", h, r) } } if rolledBackHost != "platform.example.com" { t.Fatalf("rollback restart ran for %q, want the bound host", rolledBackHost) } for _, h := range []string{"console.example.com", "platform.example.com"} { if r, _ := ReadRefusal(bake, h); !r.Recovered || r.RestoredVersion != "old" || r.RecoveryError != "" { t.Fatalf("%s refusal = %+v, want recovered onto old", h, r) } } } // notUpGroupFixture is the flagship shape again: console.example.com is // served by platform.example.com's console.service, the bound route has a // history (old over older), and the canary pointer names the rejected new. func notUpGroupFixture(t *testing.T) (context.Context, *registry.Registry, Store, ed25519.PublicKey, string) { t.Helper() ctx, reg, s, pub, opt := startUnitFixture(t, "console.example.com") if err := reg.SetUnit("console.example.com", ""); err != nil { t.Fatal(err) } groupFixtureRoute(t, reg, "platform.example.com", "", "127.0.0.1:8090", "console.service") if err := reg.SetUpstreams("console.example.com", "127.0.0.1:8090", "", ""); err != nil { t.Fatal(err) } if err := PointLive(opt, "platform.example.com", "bab", "old", "console"); err != nil { t.Fatal(err) } if err := reg.SetVersions("platform.example.com", "old", "older"); err != nil { t.Fatal(err) } return ctx, reg, s, pub, opt } // Final review F6 (and the ledger's Task 3 minor): a sibling rolled back with // its process is stamped with the BOUND route's previous version. Its own // PrevVersion names restored by then (this pass stamped it new over old). func TestRollbackGroupStampsTheBoundRoutesHistory(t *testing.T) { ctx, reg, s, pub, opt := notUpGroupFixture(t) restart := func(h string) error { if h == "platform.example.com" { return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"} } return nil } var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(t.TempDir()), WithRollbackRestart(func(string) error { return nil })); err != nil { t.Fatalf("Pass: %v", err) } }) if rt := mustRoute(t, reg, "console.example.com"); rt.Version != "old" || rt.PrevVersion != "older" { t.Fatalf("sibling stamp = %q over %q, want old over older (bound's history)", rt.Version, rt.PrevVersion) } } // Final review F6: the rollback restart failed. Every record says so, with // the error, so the watcher does not publish a recovery that never happened. func TestRollbackRestartFailureIsRecordedAsNotRecovered(t *testing.T) { ctx, reg, s, pub, opt := notUpGroupFixture(t) restart := func(h string) error { if h == "platform.example.com" { return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"} } return nil } rb := func(string) error { return errors.New("rollback did not come up either: console.service did not come up: unit entered failed state") } bake := t.TempDir() var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), WithRollbackRestart(rb)); err != nil { t.Fatalf("Pass: %v", err) } }) for _, h := range []string{"console.example.com", "platform.example.com"} { r, _ := ReadRefusal(bake, h) if r == nil || r.Recovered || r.RestoredVersion != "old" || !strings.Contains(r.RecoveryError, "rollback did not come up either") { t.Fatalf("%s refusal = %+v, want not recovered, restored old, carrying the restart's error", h, r) } } } // Final review F6: the BOUND route's own symlink rollback failed, so the // process still runs (or will run) the rejected build. Siblings are not // repointed onto a version their process is not running, and the records // say there was no recovery. func TestBoundRollbackFailureLeavesSiblingsAlone(t *testing.T) { ctx, reg, s, pub, opt := notUpGroupFixture(t) restart := func(h string) error { if h == "platform.example.com" { // A non-empty directory where PointLive's temp link goes: the // repoint back to old fails for this host only. if err := os.MkdirAll(filepath.Join(opt, "live", "platform.example.com.tmp", "x"), 0o755); err != nil { t.Fatal(err) } return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"} } return nil } rbCalled := false bake := t.TempDir() var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), WithRollbackRestart(func(string) error { rbCalled = true; return nil })); err != nil { t.Fatalf("Pass: %v", err) } }) if v, _ := LiveVersion(opt, "platform.example.com"); v != "new" { t.Fatalf("bound live = %q; the fixture meant its rollback to fail", v) } if v, _ := LiveVersion(opt, "console.example.com"); v != "new" { t.Fatalf("sibling live = %q, want it left on new: its process never went back to old", v) } if rbCalled { t.Fatal("the rollback restart ran with the link still on the rejected build") } r, _ := ReadRefusal(bake, "platform.example.com") if r == nil || r.Recovered || r.RestoredVersion != "" || !strings.Contains(r.RecoveryError, "could not roll the symlink back to old") { t.Fatalf("bound refusal = %+v, want not recovered with the repoint's error", r) } } // Final review F4: a same-upstream route following ANOTHER channel is a // separate promotion. It is neither repointed nor refused with the bound // route's PromotedAt. func TestRollbackGroupLeavesAnotherChannelAlone(t *testing.T) { ctx, reg, s, pub, opt := startUnitFixture(t, "console.example.com") if err := reg.SetUnit("console.example.com", "console.service"); err != nil { t.Fatal(err) } if err := reg.SetUpstreams("console.example.com", "127.0.0.1:8090", "", ""); err != nil { t.Fatal(err) } if err := reg.Upsert(registry.Route{Host: "zz-beta.example.com", Kind: "instance", App: "console", Channel: "beta", Addr: "127.0.0.1:8090"}); err != nil { t.Fatal(err) } if err := StageArtifact(opt, "bab", "new", "console", []byte("bad build")); err != nil { t.Fatal(err) } if err := PointLive(opt, "zz-beta.example.com", "bab", "new", "console"); err != nil { t.Fatal(err) } if got := processGroup(reg, mustRoute(t, reg, "console.example.com")); len(got) != 0 { t.Fatalf("group = %+v, want none: zz-beta follows another channel", got) } restart := func(h string) error { if h == "console.example.com" { return &UnitNotUpError{Unit: "console.service", Reason: "not ready after 30s"} } return nil } bake := t.TempDir() var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), WithRollbackRestart(func(string) error { return nil })); err != nil { t.Fatalf("Pass: %v", err) } }) if v, _ := LiveVersion(opt, "zz-beta.example.com"); v != "new" { t.Fatalf("another channel's route was rolled back to %q", v) } if r, _ := ReadRefusal(bake, "zz-beta.example.com"); r != nil { t.Fatalf("another channel's route was refused with the bound route's promotion: %+v", r) } if r, _ := ReadRefusal(bake, "console.example.com"); r == nil { t.Fatal("the bound route has no refusal") } } func mustRoute(t *testing.T, reg *registry.Registry, host string) registry.Route { t.Helper() rt, err := reg.Get(host) if err != nil || rt == nil { t.Fatalf("get %s: %+v, %v", host, rt, err) } return *rt } func TestRollbackGroupLeavesOtherUpstreamAlone(t *testing.T) { ctx, reg, s, pub, opt := startUnitFixture(t, "console.example.com") if err := reg.SetUnit("console.example.com", "console.service"); err != nil { t.Fatal(err) } if err := reg.SetUpstreams("console.example.com", "127.0.0.1:8090", "", ""); err != nil { t.Fatal(err) } groupFixtureRoute(t, reg, "zz-other.example.com", "", "127.0.0.1:9000", "") if err := StageArtifact(opt, "bab", "new", "console", []byte("bad build")); err != nil { t.Fatal(err) } if err := PointLive(opt, "zz-other.example.com", "bab", "new", "console"); err != nil { t.Fatal(err) } restart := func(h string) error { if h == "console.example.com" { return &UnitNotUpError{Unit: "console.service", Reason: "not ready after 30s"} } return nil } bake := t.TempDir() if err := Pass(ctx, reg, s, pub, opt, restart, BakeDir(bake), WithRollbackRestart(func(string) error { return nil })); err != nil { t.Fatalf("Pass: %v", err) } if v, _ := LiveVersion(opt, "zz-other.example.com"); v != "new" { t.Fatalf("another process's route was rolled back to %q", v) } if r, _ := ReadRefusal(bake, "zz-other.example.com"); r != nil { t.Fatalf("another process's route was refused: %+v", r) } } // scriptedWork is one host's release selection under lifecycle host writes. // It counts With calls, refuses every effect once stale, and lets a test go // stale right after a chosen call's effect has landed. type scriptedWork struct { name string calls int stale bool after func(call int) } func (w *scriptedWork) With(_ context.Context, id hostwrite.Identity, fn func() error) error { w.calls++ if id.Release == "" { return errors.New("missing release identity") } if w.stale { return errors.New(w.name + ": release selection changed") } err := fn() if w.after != nil { w.after(w.calls) } return err } // scriptedSelector hands out each host's works in order, one per Select, and // keeps the last for any later Select. type scriptedSelector struct { works map[string][]*scriptedWork selects map[string]int } func (s *scriptedSelector) Select(_ context.Context, host, _ string) (hostwrite.Work, error) { list := s.works[host] if len(list) == 0 { return nil, errors.New("no work for " + host) } i := min(s.selects[host], len(list)-1) s.selects[host]++ return list[i], nil } func claimWrites(sel *scriptedSelector) PassOption { return WithHostWrites(hostwrite.Config{Enabled: true, Selector: sel}) } // boundNotUp is the selected restart for notUpGroupFixture: the bound route's // unit does not come up, the sibling's restart is the shared process's no-op. func boundNotUp(restarts *int) hostwrite.Callback { return func(_ context.Context, s hostwrite.Selection) error { *restarts++ if s.Host == "platform.example.com" { return &UnitNotUpError{Unit: "console.service", Reason: "exited: status=2"} } return nil } } // Merge review P1: authority lost after the group rolled back, before the // rollback start. The start is skipped, but every group host is still refused // (not recovered, carrying the authority error), so the next pass, with fresh // authority and an empty poisoned map, does not adopt the rejected build. func TestClaimAuthorityLostBeforeRollbackStartStillRefusesTheGroup(t *testing.T) { ctx, reg, s, pub, opt := notUpGroupFixture(t) bake := t.TempDir() bound := &scriptedWork{name: "platform"} // Bound's calls: 1 forward repoint, 2 the pre-restart check, 3 the // rollback repoint, 4 the pre-start check. Stale after 3 has landed. bound.after = func(call int) { if call == 3 { bound.stale = true } } sibling := &scriptedWork{name: "console"} sel := &scriptedSelector{selects: map[string]int{}, works: map[string][]*scriptedWork{ "platform.example.com": {bound}, "console.example.com": {sibling}}} restarts, starts := 0, 0 start := func(ctx context.Context, s hostwrite.Selection) error { starts++; return nil } var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, func(string) error { t.Fatal("legacy restart"); return nil }, BakeDir(bake), claimWrites(sel), WithSelectedRestarts(boundNotUp(&restarts), start), WithRollbackRestart(func(string) error { t.Fatal("legacy rollback restart"); return nil })); err != nil { t.Fatalf("Pass: %v", err) } }) if !bound.stale { t.Fatalf("the fixture never reached the rollback repoint:\n%s", out.String()) } if starts != 0 { t.Fatalf("rollback start ran %d times after authority changed", starts) } for _, h := range []string{"console.example.com", "platform.example.com"} { if v, _ := LiveVersion(opt, h); v != "old" { t.Fatalf("%s live = %q, want old (rolled back before authority changed)", h, v) } r, err := ReadRefusal(bake, h) if err != nil || r == nil || r.Version != "new" || r.Recovered || r.RestoredVersion != "old" || !strings.Contains(r.RecoveryError, "release authority changed before rollback start") { t.Fatalf("%s refusal = %+v, %v; want new refused, not recovered, with the authority error\n%s", h, r, err, out.String()) } } // The next pass: fresh, valid authority for both hosts, nothing poisoned. sel2 := &scriptedSelector{selects: map[string]int{}, works: map[string][]*scriptedWork{ "platform.example.com": {{name: "platform2"}}, "console.example.com": {{name: "console2"}}}} restarts = 0 out.Reset() withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, func(string) error { t.Fatal("legacy restart"); return nil }, BakeDir(bake), claimWrites(sel2), WithSelectedRestarts(boundNotUp(&restarts), start), WithRollbackRestart(func(string) error { t.Fatal("legacy rollback restart"); return nil })); err != nil { t.Fatalf("second Pass: %v", err) } }) if restarts != 0 || starts != 0 { t.Fatalf("second pass restarted %d / started %d: it re-adopted the rejected build\n%s", restarts, starts, out.String()) } for _, h := range []string{"console.example.com", "platform.example.com"} { if v, _ := LiveVersion(opt, h); v != "old" { t.Fatalf("%s live = %q after the second pass, want old", h, v) } if rt := mustRoute(t, reg, h); rt.Version == "new" { t.Fatalf("%s stamped new by the second pass", h) } } } // Merge review P1: the sibling's installation changes between its rollback // repoint and its stamp. The stamp is refused under the sibling's one // selection (the pass's original), never re-selected and handed to the // successor. func TestClaimSiblingStampRefusedWhenItsSelectionGoesStale(t *testing.T) { ctx, reg, s, pub, opt := notUpGroupFixture(t) bake := t.TempDir() bound := &scriptedWork{name: "platform"} sibling := &scriptedWork{name: "console"} // Sibling's calls: 1 forward repoint, 2 the pre-restart check, 3 the // forward stamp, 4 the rollback repoint. Stale after 4 has landed. sibling.after = func(call int) { if call == 4 { sibling.stale = true } } successor := &scriptedWork{name: "console-successor"} sel := &scriptedSelector{selects: map[string]int{}, works: map[string][]*scriptedWork{ "platform.example.com": {bound}, "console.example.com": {sibling, successor}}} restarts, starts := 0, 0 start := func(ctx context.Context, s hostwrite.Selection) error { return s.With(ctx, func() error { starts++; return nil }) } var out bytes.Buffer withCapturedStdout(t, &out, func() { if err := Pass(ctx, reg, s, pub, opt, func(string) error { t.Fatal("legacy restart"); return nil }, BakeDir(bake), claimWrites(sel), WithSelectedRestarts(boundNotUp(&restarts), start), WithRollbackRestart(func(string) error { t.Fatal("legacy rollback restart"); return nil })); err != nil { t.Fatalf("Pass: %v", err) } }) if n := sel.selects["console.example.com"]; n != 1 || successor.calls != 0 { t.Fatalf("sibling selected %d times, successor used %d times: want the pass's one selection only", n, successor.calls) } if !sibling.stale { t.Fatalf("the fixture never reached the sibling's rollback repoint:\n%s", out.String()) } if v, _ := LiveVersion(opt, "console.example.com"); v != "old" { t.Fatalf("sibling live = %q, want old (the repoint landed before the change)", v) } if rt := mustRoute(t, reg, "console.example.com"); rt.Version != "new" { t.Fatalf("sibling stamped %q over %q: the stamp should have been refused", rt.Version, rt.PrevVersion) } if !strings.Contains(out.String(), "console: release selection changed") { t.Fatalf("the refused stamp was not logged:\n%s", out.String()) } if starts != 1 { t.Fatalf("bound's rollback start ran %d times, want 1", starts) } for _, h := range []string{"console.example.com", "platform.example.com"} { if r, _ := ReadRefusal(bake, h); r == nil || r.Version != "new" { t.Fatalf("%s refusal = %+v, want one naming new", h, r) } } }