package host import ( "errors" "io/fs" "net" "net/http" "net/http/httptest" "os" "path/filepath" "strconv" "strings" "syscall" "testing" "time" "github.com/carlosframework/platform/internal/registry" "github.com/carlosframework/platform/internal/release" ) type fakeUnits struct { shows []unitProps // returned in order; the last repeats n int exe map[int]string exeErr map[int]error // optional: errors to return from Exe exeSeq []string // optional: Exe's answers in order, whatever the pid; the last repeats exeN int showErr map[int]error // optional: the Nth Show call (0-based) fails with this journal string restarts []string } func (f *fakeUnits) Show(string) (unitProps, error) { if err, ok := f.showErr[f.n]; ok { f.n++ return unitProps{}, err } p := f.shows[min(f.n, len(f.shows)-1)] f.n++ return p, nil } func (f *fakeUnits) TryRestart(u string) error { f.restarts = append(f.restarts, "try-restart "+u) return nil } func (f *fakeUnits) Restart(u string) error { f.restarts = append(f.restarts, "restart "+u) return nil } func (f *fakeUnits) ResetFailed(u string) error { f.restarts = append(f.restarts, "reset-failed "+u) return nil } func (f *fakeUnits) JournalTail(string, int) string { return f.journal } func (f *fakeUnits) StartTime(pid int) (string, error) { return strconv.Itoa(pid), nil } func (f *fakeUnits) Exe(pid int) (string, error) { if err, ok := f.exeErr[pid]; ok { return "", err } if len(f.exeSeq) > 0 { e := f.exeSeq[min(f.exeN, len(f.exeSeq)-1)] f.exeN++ return e, nil } if e, ok := f.exe[pid]; ok { return e, nil } return "", errors.New("no such pid") } // settleClock, not fakeClock: stage_relay_test.go already declares a // fakeClock type in this package. func settleClock() (func() time.Time, func(time.Duration)) { t := time.Unix(0, 0) return func() time.Time { return t }, func(d time.Duration) { t = t.Add(d) } } func opts(ready func() error) settleOpts { now, sleep := settleClock() return settleOpts{Timeout: 30 * time.Second, Poll: 500 * time.Millisecond, Hold: 5 * time.Second, WantDir: "/opt/carlos/versions/new", Ready: ready, Now: now, Sleep: sleep} } func TestParseShow(t *testing.T) { p := parseShow("ActiveState=active\nSubState=running\nInvocationID=abc\nMainPID=42\nNRestarts=0\n" + "ExecStart={ path=/opt/carlos/live/platform.example.com ; argv[]=/opt/carlos/live/platform.example.com serve ; ignore_errors=no }\n") if p.ActiveState != "active" || p.SubState != "running" || p.InvocationID != "abc" || p.MainPID != 42 || p.ExecPath != "/opt/carlos/live/platform.example.com" { t.Fatalf("parseShow = %+v", p) } } func TestSettleHealthyNewInvocation(t *testing.T) { f := &fakeUnits{shows: []unitProps{ {ActiveState: "activating", InvocationID: "new"}, {ActiveState: "active", InvocationID: "new", MainPID: 7}, }, exe: map[int]string{7: "/opt/carlos/versions/new/console"}} if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); err != nil { t.Fatalf("settle = %v, want healthy", err) } } // systemd resets NRestarts on a manual start: a healthy unit that had // auto-restarted before must still pass (spec ยง2.3, codex 4). func TestSettleIgnoresThePreviousInvocationsCounter(t *testing.T) { f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7, NRestarts: 0}}, exe: map[int]string{7: "/opt/carlos/versions/new/console"}} if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); err != nil { t.Fatalf("settle = %v", err) } } // Today's outage: the process exits in 55 ms and systemd cycles it. func TestSettleCatchesARestartLoop(t *testing.T) { f := &fakeUnits{shows: []unitProps{ {ActiveState: "active", InvocationID: "new1", MainPID: 7}, {ActiveState: "activating", InvocationID: "new2", NRestarts: 1}, }, exe: map[int]string{7: "/opt/carlos/versions/new/console"}, journal: "usage: carlos [flags]"} err := settleUnit(f, "console.service", "old", opts(func() error { return errors.New("connection refused") })) var nu *release.UnitNotUpError if !errors.As(err, &nu) || !strings.Contains(nu.Journal, "usage: carlos") { t.Fatalf("settle = %v, want UnitNotUpError carrying the journal", err) } } func TestSettleCatchesFailed(t *testing.T) { f := &fakeUnits{shows: []unitProps{{ActiveState: "failed", InvocationID: "new"}}} var nu *release.UnitNotUpError if err := settleUnit(f, "console.service", "old", opts(nil)); !errors.As(err, &nu) { t.Fatalf("settle = %v, want UnitNotUpError", err) } } func TestSettleDeadlineSaysNotReady(t *testing.T) { f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}}, exe: map[int]string{7: "/opt/carlos/versions/new/console"}} err := settleUnit(f, "console.service", "old", opts(func() error { return errors.New("dial: refused") })) var nu *release.UnitNotUpError if !errors.As(err, &nu) || !strings.Contains(nu.Reason, "not ready after 30s") { t.Fatalf("settle = %v, want a not-ready deadline", err) } } func TestSettleRejectsTheOldBinary(t *testing.T) { f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}}, exe: map[int]string{7: "/opt/carlos/versions/old/console"}} var nu *release.UnitNotUpError if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); !errors.As(err, &nu) || !strings.Contains(nu.Reason, "running /opt/carlos/versions/old/console, not the new build in /opt/carlos/versions/new") { t.Fatalf("settle = %v, want not-up at the deadline naming the old build's exe", err) } } // Final review F1: for Type=simple, systemd reports the new invocation active // once it has forked and before it execs ExecStart, so the first poll can see // systemd's executor as MainPID's exe. That is not-ready, not a failure. func TestSettleWaitsOutTheExecWindow(t *testing.T) { f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}}, exeSeq: []string{"/usr/lib/systemd/systemd-executor", "/opt/carlos/versions/new/console"}} readies := 0 err := settleUnit(f, "console.service", "old", opts(func() error { readies++; return nil })) if err != nil { t.Fatalf("settle = %v, want healthy once the new binary is exec'd", err) } if f.exeN != 2 || readies != 1 { t.Fatalf("exe reads %d, ready calls %d; want 2 and 1 (no readiness judged on the executor)", f.exeN, readies) } } func TestSettleExecutorAtTheDeadlineNamesTheExe(t *testing.T) { f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}}, exeSeq: []string{"/usr/lib/systemd/systemd-executor"}} var nu *release.UnitNotUpError err := settleUnit(f, "console.service", "old", opts(func() error { return nil })) if !errors.As(err, &nu) || !strings.Contains(nu.Reason, "running /usr/lib/systemd/systemd-executor, not the new build in /opt/carlos/versions/new") { t.Fatalf("settle = %v, want a deadline UnitNotUpError naming the exe seen", err) } } // Final review F5: one failed `systemctl show` is retried, not a verdict. func TestSettleRetriesATransientShowError(t *testing.T) { f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}}, exe: map[int]string{7: "/opt/carlos/versions/new/console"}, showErr: map[int]error{0: errors.New("Failed to get properties: Connection timed out")}} if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); err != nil { t.Fatalf("settle = %v, want healthy after one show error", err) } } func TestSettleShowErrorAtTheDeadline(t *testing.T) { sys := &showFails{fakeUnits: &fakeUnits{}} var nu *release.UnitNotUpError err := settleUnit(sys, "console.service", "old", opts(func() error { return nil })) if !errors.As(err, &nu) || !strings.Contains(nu.Reason, "cannot observe console.service: Access denied") { t.Fatalf("settle = %v, want a deadline UnitNotUpError saying the unit could not be observed", err) } } // Final review F2: inactive/dead after the restart is an operator's stop. func TestSettleStoppedUnitIsErrUnitStopped(t *testing.T) { f := &fakeUnits{shows: []unitProps{ {ActiveState: "active", InvocationID: "new", MainPID: 7}, {ActiveState: "inactive", SubState: "dead", InvocationID: "new"}, }, exe: map[int]string{7: "/opt/carlos/versions/new/console"}} err := settleUnit(f, "console.service", "old", opts(func() error { return errors.New("refused") })) if !errors.Is(err, errUnitStopped) { t.Fatalf("settle = %v, want errUnitStopped", err) } } func TestSettleWithoutReadyHoldsFiveSeconds(t *testing.T) { f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "new", MainPID: 7}}, exe: map[int]string{7: "/opt/carlos/versions/new/console"}} o := opts(nil) start := o.Now() if err := settleUnit(f, "console.service", "old", o); err != nil { t.Fatalf("settle = %v", err) } if got := o.Now().Sub(start); got < 5*time.Second { t.Fatalf("settled after %v, want at least the 5s hold", got) } } func TestSettleSkipsAUnitThatWasNotRunning(t *testing.T) { // beforeInvocation "" means the caller saw the unit inactive; settleUnit // is not called at all in that case (Task 6). This pins the caller's // contract from this side: an inactive unit that stays inactive after // try-restart is "nothing started", not a failure. f := &fakeUnits{shows: []unitProps{{ActiveState: "inactive"}}} if err := settleUnit(f, "console.service", "", opts(nil)); err != nil { t.Fatalf("settle on a stopped unit = %v, want nil", err) } } // Fix round 1, issue 1: NRestarts unchecked on the poll that fixes the new invocation. func TestSettleCatchesRestartLoopOnFreshInvocation(t *testing.T) { f := &fakeUnits{shows: []unitProps{ {ActiveState: "active", InvocationID: "new", MainPID: 7, NRestarts: 1}, }, exe: map[int]string{7: "/opt/carlos/versions/new/console"}} var nu *release.UnitNotUpError if err := settleUnit(f, "console.service", "old", opts(func() error { return nil })); !errors.As(err, &nu) { t.Fatalf("settle = %v, want UnitNotUpError for restart loop on fresh invocation", err) } } // Fix round 1, issue 2a: Exe error (not permission) should be remembered and deadline should mention it. func TestSettleExeErrorDeadline(t *testing.T) { f := &fakeUnits{shows: []unitProps{ {ActiveState: "active", InvocationID: "new", MainPID: 7}, }, exe: map[int]string{}} var nu *release.UnitNotUpError if err := settleUnit(f, "console.service", "old", opts(nil)); !errors.As(err, &nu) || !strings.Contains(nu.Reason, "exe unreadable") { t.Fatalf("settle = %v, want UnitNotUpError with 'exe unreadable' in reason", err) } } // Fix round 1, issue 2b: Permission error on Exe should be logged once and skipped, allowing settle to succeed. func TestSettleExePermissionErrorSkipped(t *testing.T) { f := &fakeUnits{ shows: []unitProps{ {ActiveState: "active", InvocationID: "new", MainPID: 7}, }, exeErr: map[int]error{7: &fs.PathError{Op: "readlink", Path: "/proc/7/exe", Err: syscall.EACCES}}, } if err := settleUnit(f, "console.service", "old", opts(nil)); err != nil { t.Fatalf("settle = %v, want nil (permission error on Exe should be skipped)", err) } } func TestUnitSettleTimeoutFloor(t *testing.T) { env := func(v string) func(string) string { return func(string) string { return v } } for in, want := range map[string]time.Duration{"": 30 * time.Second, "1s": 5 * time.Second, "45s": 45 * time.Second, "junk": 30 * time.Second} { if got := unitSettleTimeout(env(in)); got != want { t.Fatalf("CARLOS_UNIT_SETTLE=%q -> %v, want %v", in, got, want) } } } func TestRouteReadyProbesTheUnixSocket(t *testing.T) { dir, err := os.MkdirTemp("", "rr") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(dir) }) sock := filepath.Join(dir, "s.sock") ln, err := net.Listen("unix", sock) if err != nil { t.Fatal(err) } srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/healthz" && r.Host == "console.example.com" { w.WriteHeader(200) return } w.WriteHeader(500) })} go srv.Serve(ln) t.Cleanup(func() { srv.Close() }) rt := registry.Route{Host: "console.example.com", Socket: sock, Addr: "127.0.0.1:1", Health: `{"path":"/healthz","status":200}`} if err := routeReady(rt, time.Second)(); err != nil { t.Fatalf("ready over the socket = %v (Addr must be ignored when Socket is set)", err) } } func TestRouteReadyWrongStatus(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(503) })) t.Cleanup(srv.Close) rt := registry.Route{Host: "x.example.com", Addr: strings.TrimPrefix(srv.URL, "http://"), Health: `{"path":"/healthz","status":200}`} if err := routeReady(rt, time.Second)(); err == nil || !strings.Contains(err.Error(), "503") { t.Fatalf("ready = %v, want a 503 failure", err) } } func TestRouteReadyNoProbeIsAConnect(t *testing.T) { ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } t.Cleanup(func() { ln.Close() }) if err := routeReady(registry.Route{Host: "x", Addr: ln.Addr().String()}, time.Second)(); err != nil { t.Fatalf("connect readiness = %v", err) } } func TestRouteReadyNoUpstreamIsNil(t *testing.T) { if routeReady(registry.Route{Host: "x"}, time.Second) != nil { t.Fatal("a route with no upstream got a ready check; settle must use the hold instead") } } func TestRouteReadyDoesNotFollowRedirects(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/healthz" { w.Header().Set("Location", "/ok") w.WriteHeader(302) return } if r.URL.Path == "/ok" { w.WriteHeader(200) return } w.WriteHeader(500) })) t.Cleanup(srv.Close) // Case 1: expect 200 at /healthz, but get 302; should error rt1 := registry.Route{Host: "x.example.com", Addr: strings.TrimPrefix(srv.URL, "http://"), Health: `{"path":"/healthz","status":200}`} if err := routeReady(rt1, time.Second)(); err == nil || !strings.Contains(err.Error(), "302") { t.Fatalf("ready with redirect = %v, want a 302 error", err) } // Case 2: expect 302 at /healthz; should succeed rt2 := registry.Route{Host: "x.example.com", Addr: strings.TrimPrefix(srv.URL, "http://"), Health: `{"path":"/healthz","status":302}`} if err := routeReady(rt2, time.Second)(); err != nil { t.Fatalf("ready expecting 302 = %v, want nil", err) } } func boundRoute() registry.Route { return registry.Route{Host: "platform.example.com", App: "console", Unit: "console.service", Addr: "127.0.0.1:1"} } func liveTo(t *testing.T, opt, host, ver string) { t.Helper() dir := filepath.Join(opt, "versions", ver) if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "console"), []byte("x"), 0o755); err != nil { t.Fatal(err) } if err := os.MkdirAll(filepath.Join(opt, "live"), 0o755); err != nil { t.Fatal(err) } os.Remove(filepath.Join(opt, "live", host)) if err := os.Symlink(filepath.Join(dir, "console"), filepath.Join(opt, "live", host)); err != nil { t.Fatal(err) } } func TestNamedRestartBindingMismatchKeepsTheError(t *testing.T) { opt := t.TempDir() liveTo(t, opt, "platform.example.com", "new") f := &fakeUnits{shows: []unitProps{{ActiveState: "active", InvocationID: "old", ExecPath: filepath.Join(opt, "live", "console.example.com")}}} failing := &failingTry{fakeUnits: f} err := agentCheckedUnitRestart(failing, opt, func(string) string { return "" }, nil)(boundRoute()) if err == nil || !strings.Contains(err.Error(), "try-restart failed") { t.Fatalf("err = %v, want the try-restart error propagated (codex 12)", err) } } type failingTry struct{ *fakeUnits } func (failingTry) TryRestart(string) error { return errors.New("try-restart failed") } func TestNamedRestartShowFailsFallsBack(t *testing.T) { opt := t.TempDir() sys := &showFails{fakeUnits: &fakeUnits{}} if err := agentCheckedUnitRestart(sys, opt, func(string) string { return "" }, nil)(boundRoute()); err != nil { t.Fatalf("err = %v, want today's behaviour (plain try-restart, its nil)", err) } if len(sys.restarts) != 1 || sys.restarts[0] != "try-restart console.service" { t.Fatalf("restarts = %v", sys.restarts) } } type showFails struct{ *fakeUnits } func (showFails) Show(string) (unitProps, error) { return unitProps{}, errors.New("Access denied") } func TestNamedRestartStoppedUnitIsNotChecked(t *testing.T) { opt := t.TempDir() liveTo(t, opt, "platform.example.com", "new") f := &fakeUnits{shows: []unitProps{{ActiveState: "inactive", ExecPath: filepath.Join(opt, "live", "platform.example.com")}}} if err := agentCheckedUnitRestart(f, opt, func(string) string { return "" }, nil)(boundRoute()); err != nil { t.Fatalf("err = %v, want nil for an operator-stopped unit", err) } } func TestRollbackRestartResetsThenRestarts(t *testing.T) { t.Parallel() opt := t.TempDir() liveTo(t, opt, "platform.example.com", "old") reg := restartTestRegistry(t) r := boundRoute() r.Addr = "" // no upstream: readiness is nil and settle holds instead of dialling if err := reg.Upsert(r); err != nil { t.Fatal(err) } f := &fakeUnits{shows: []unitProps{ {ActiveState: "active", InvocationID: "bad"}, {ActiveState: "active", InvocationID: "restored", MainPID: 9}, }, exe: map[int]string{9: filepath.Join(opt, "versions", "old", "console")}} rb := agentRollbackUnitRestart(reg, f, opt, func(string) string { return "" }) if err := rb("platform.example.com"); err != nil { t.Fatalf("rollback = %v", err) } if strings.Join(f.restarts, ",") != "reset-failed console.service,restart console.service" { t.Fatalf("restarts = %v, want reset-failed then restart (never start)", f.restarts) } } // newBuildDir is where checkedUnitRestart expects the new process to run // from: the directory of the live link's resolved target, the same // EvalSymlinks the production code does (t.TempDir can sit under a symlink). func newBuildDir(t *testing.T, opt, host string) string { t.Helper() want, err := filepath.EvalSymlinks(filepath.Join(opt, "live", host)) if err != nil { t.Fatal(err) } return filepath.Dir(want) } // The production wiring, end to end: checkedUnitRestart reads the unit, // try-restarts it, and settles a healthy forward restart on the new build // (ledger Task 6 minor). Real clock; the first poll is healthy. func TestNamedRestartSettlesAHealthyForwardRestart(t *testing.T) { opt := t.TempDir() liveTo(t, opt, "platform.example.com", "new") ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } t.Cleanup(func() { ln.Close() }) rt := boundRoute() rt.Addr = ln.Addr().String() f := &fakeUnits{shows: []unitProps{ {ActiveState: "active", SubState: "running", InvocationID: "old", MainPID: 6, ExecPath: filepath.Join(opt, "live", "platform.example.com")}, {ActiveState: "active", SubState: "running", InvocationID: "new", MainPID: 7}, }, exe: map[int]string{7: filepath.Join(newBuildDir(t, opt, "platform.example.com"), "console")}} if err := agentCheckedUnitRestart(f, opt, func(string) string { return "" }, nil)(rt); err != nil { t.Fatalf("forward restart = %v, want settled", err) } if strings.Join(f.restarts, ",") != "try-restart console.service" { t.Fatalf("restarts = %v, want exactly the try-restart", f.restarts) } } // Final review F2: the unit read active, then an operator stopped it (before // or during the settle). checkedUnitRestart reports nil, so adoptHost does // not roll back, refuse or restart it, and the only systemctl verb issued is // the try-restart itself. func TestNamedRestartStopDuringSettleIsNotJudged(t *testing.T) { opt := t.TempDir() liveTo(t, opt, "platform.example.com", "new") f := &fakeUnits{shows: []unitProps{ {ActiveState: "active", SubState: "running", InvocationID: "old", MainPID: 6, ExecPath: filepath.Join(opt, "live", "platform.example.com")}, {ActiveState: "inactive", SubState: "dead", InvocationID: "old"}, }} if err := agentCheckedUnitRestart(f, opt, func(string) string { return "" }, nil)(boundRoute()); err != nil { t.Fatalf("err = %v, want nil: a stop is not a verdict on the build", err) } if strings.Join(f.restarts, ",") != "try-restart console.service" { t.Fatalf("restarts = %v, want only the try-restart (no restart, no reset-failed)", f.restarts) } } // The other side of F2: failed is a crash, never a stop. func TestNamedRestartFailedStillFails(t *testing.T) { opt := t.TempDir() liveTo(t, opt, "platform.example.com", "new") f := &fakeUnits{shows: []unitProps{ {ActiveState: "active", SubState: "running", InvocationID: "old", MainPID: 6, ExecPath: filepath.Join(opt, "live", "platform.example.com")}, {ActiveState: "failed", SubState: "failed", InvocationID: "new"}, }} var nu *release.UnitNotUpError if err := agentCheckedUnitRestart(f, opt, func(string) string { return "" }, nil)(boundRoute()); !errors.As(err, &nu) { t.Fatalf("err = %v, want UnitNotUpError for a failed unit", err) } } // A stop during the rollback's own settle is the operator's too: the link // names the restored build, and nothing restarts it a second time. func TestRollbackRestartStopDuringSettleIsNotJudged(t *testing.T) { opt := t.TempDir() liveTo(t, opt, "platform.example.com", "old") reg := restartTestRegistry(t) r := boundRoute() r.Addr = "" if err := reg.Upsert(r); err != nil { t.Fatal(err) } f := &fakeUnits{shows: []unitProps{ {ActiveState: "active", InvocationID: "bad"}, {ActiveState: "inactive", SubState: "dead", InvocationID: "bad"}, }} if err := agentRollbackUnitRestart(reg, f, opt, func(string) string { return "" })("platform.example.com"); err != nil { t.Fatalf("rollback = %v, want nil for a stop during its settle", err) } if strings.Join(f.restarts, ",") != "reset-failed console.service,restart console.service" { t.Fatalf("restarts = %v, want the one reset-failed and restart only", f.restarts) } }