package instancectl

import (
	"context"
	"encoding/json"
	"errors"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/carlosframework/platform/internal/version"
)

type fakeCtl struct {
	mu    sync.Mutex
	calls []string
	// answers maps a leading argument pattern ("show -p InvocationID") to output.
	answers map[string]string
	fail    map[string]error
	// onStart, if set, runs once, synchronously, the first time args[0] is
	// "start", before run returns to its caller. A test uses it to
	// simulate a client that disconnects the instant systemctl start
	// answers (fix round 1, item 4).
	onStart func()
	// before, keyed by a verb (args[0]), runs once, synchronously, before
	// that verb's call is recorded: a test holds a handler there.
	before map[string]func()
	// unseenCancel models a handler that has not yet seen its client go
	// (#292): its systemctl calls run although the request's context is
	// done, the race exec.CommandContext cannot close on its own.
	unseenCancel bool
}

func (f *fakeCtl) run(ctx context.Context, args ...string) (string, error) {
	f.mu.Lock()
	var pre func()
	if len(args) > 0 && f.before[args[0]] != nil {
		pre = f.before[args[0]]
		delete(f.before, args[0])
	}
	f.mu.Unlock()
	if pre != nil {
		pre()
	}
	f.mu.Lock()
	line := strings.Join(args, " ")
	f.calls = append(f.calls, line)
	var hook func()
	if len(args) > 0 && args[0] == "start" && f.onStart != nil {
		hook, f.onStart = f.onStart, nil
	}
	unseen := f.unseenCancel
	f.mu.Unlock()
	if hook != nil {
		hook()
	}
	// Production's Systemctl is exec.CommandContext, which fails outright
	// once its context is done rather than completing anyway; the fake
	// must behave the same way, or a test cannot tell a handler that used
	// the request's context apart from one that used a detached one (fix
	// round 1, item 4).
	if ctx.Err() != nil && !unseen {
		return "", ctx.Err()
	}
	f.mu.Lock()
	defer f.mu.Unlock()
	for k, err := range f.fail {
		if strings.HasPrefix(line, k) {
			return "", err
		}
	}
	for k, out := range f.answers {
		if strings.HasPrefix(line, k) {
			return out, nil
		}
	}
	return "", nil
}

func (f *fakeCtl) joined() string {
	f.mu.Lock()
	defer f.mu.Unlock()
	return strings.Join(f.calls, "\n")
}

// newServer starts a Server behind a real unix socket. Any configure hooks
// run before the accept loop starts, so a test that wants a non-default
// PeerOK (or anything else) sets it up front rather than racing Serve's
// first Accept (codex review fix round 2 item 1).
func newServer(t *testing.T, ctl *fakeCtl, configure ...func(*Server)) (*Server, *http.Client) {
	t.Helper()
	s, sock := serveOnSocket(t, ctl, configure...)
	client := &http.Client{Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
		return (&net.Dialer{}).DialContext(ctx, "unix", sock)
	}}}
	return s, client
}

// serveOnSocket is newServer without the http.Client: it returns the
// socket path so a test of the agent-side Client can dial it the way
// production does.
func serveOnSocket(t *testing.T, ctl *fakeCtl, configure ...func(*Server)) (*Server, string) {
	t.Helper()
	root := t.TempDir()
	s := &Server{
		Systemctl:  ctl.run,
		PeerOK:     func(net.Conn) error { return nil },
		CgroupRoot: filepath.Join(root, "cgroup"),
		ProbeOut:   filepath.Join(root, "probe.out"),
		FixtureOK:  func() error { return nil },
		ProbeEvery: time.Hour,
		Logf:       t.Logf,
		// The same binary as the Client this test process is, as in
		// production, where both are /opt/carlos/carlos.
		Build: version.Build,
	}
	for _, configure := range configure {
		configure(s)
	}
	dir, err := os.MkdirTemp("", "ic")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { os.RemoveAll(dir) })
	sock := filepath.Join(dir, "c.sock")
	ln, err := net.Listen("unix", sock)
	if err != nil {
		t.Skipf("cannot bind a unix socket here (%v); run with TMPDIR=/tmp/g outside the sandbox", err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	t.Cleanup(cancel)
	go s.Serve(ctx, ln) //nolint:errcheck
	t.Cleanup(func() { ln.Close() })
	return s, sock
}

func post(t *testing.T, c *http.Client, path string) *http.Response {
	t.Helper()
	resp, err := c.Post("http://unix"+path, "", nil)
	if err != nil {
		t.Fatalf("POST %s: %v", path, err)
	}
	return resp
}

func TestStartSetsTheTierThenStartsThenReadsTheInvocation(t *testing.T) {
	ctl := &fakeCtl{answers: map[string]string{"show -p InvocationID --value": "abc123\n"}}
	_, c := newServer(t, ctl)
	resp := post(t, c, "/start?host=bill-2.example.com&role=run&memory_max=536870912")
	if resp.StatusCode != 200 {
		t.Fatalf("status %d", resp.StatusCode)
	}
	var out StartResponse
	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil || out.InvocationID != "abc123" {
		t.Errorf("response = %+v, %v", out, err)
	}
	want := "set-property --runtime carlos-instances-bill\\x2d2.example.com.slice MemoryMax=536870912 MemoryHigh=483183819 MemorySwapMax=536870912 CPUWeight=100 CPUQuota=100% IOWeight=100\n" +
		"start carlos-instance@bill\\x2d2.example.com.service\n" +
		"show -p InvocationID --value carlos-instance@bill\\x2d2.example.com.service"
	if got := ctl.joined(); got != want {
		t.Errorf("systemctl calls:\n%s\nwant:\n%s", got, want)
	}
}

func TestStartRefusesBadInputsWithoutCallingSystemctl(t *testing.T) {
	ctl := &fakeCtl{}
	_, c := newServer(t, ctl)
	for _, q := range []string{
		"host=Bad_Host&role=run&memory_max=536870912",
		"host=h.example.com&role=exec&memory_max=536870912",
		"host=h.example.com&role=run&memory_max=1",
		"host=h.example.com&role=seed&memory_max=536870912", // seed is not a /start role
		"role=run&memory_max=536870912",
	} {
		resp := post(t, c, "/start?"+q)
		if resp.StatusCode != 400 {
			t.Errorf("%s: status %d, want 400", q, resp.StatusCode)
		}
	}
	if ctl.joined() != "" {
		t.Errorf("systemctl was called: %s", ctl.joined())
	}
}

func TestStartReportsSystemctlFailureAs502(t *testing.T) {
	ctl := &fakeCtl{fail: map[string]error{"start ": errors.New("Job for carlos-instance@h.example.com.service failed")}}
	_, c := newServer(t, ctl)
	resp := post(t, c, "/start?host=h.example.com&role=sidecar&memory_max=1073741824")
	if resp.StatusCode != 502 {
		t.Errorf("status %d, want 502", resp.StatusCode)
	}
	if !strings.Contains(ctl.joined(), "MemoryMax=1073741824 MemoryHigh=966367638 MemorySwapMax=1073741824 CPUWeight=100 CPUQuota=200% IOWeight=100") {
		t.Errorf("children tier not applied: %s", ctl.joined())
	}
}

func TestSeedBlocksReadsTheRetainedResultWaitsAndStops(t *testing.T) {
	ctl := &fakeCtl{answers: map[string]string{
		"show -p InvocationID,Result,ExecMainStatus,ExecMainCode": "InvocationID=feed\nResult=success\nExecMainStatus=0\nExecMainCode=exited\n",
	}}
	s, c := newServer(t, ctl)
	// An empty cgroup: cgroup.events says populated 0.
	ev := filepath.Join(s.CgroupRoot, "carlos.slice", "carlos-instances.slice", "carlos-instances-h.example.com.slice", "carlos-instance-seed@h.example.com.service", "cgroup.events")
	if err := os.MkdirAll(filepath.Dir(ev), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(ev, []byte("populated 0\nfrozen 0\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	resp := post(t, c, "/seed?host=h.example.com&memory_max=536870912")
	if resp.StatusCode != 200 {
		t.Fatalf("status %d", resp.StatusCode)
	}
	var out SeedResponse
	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil || out.InvocationID != "feed" || out.Result != "success" || out.Status != "0" || out.Code != "exited" {
		t.Errorf("response = %+v, %v", out, err)
	}
	calls := ctl.joined()
	for _, want := range []string{"set-property --runtime carlos-instances-h.example.com.slice", "start carlos-instance-seed@h.example.com.service", "show -p InvocationID,Result,ExecMainStatus,ExecMainCode carlos-instance-seed@h.example.com.service", "stop carlos-instance-seed@h.example.com.service"} {
		if !strings.Contains(calls, want) {
			t.Errorf("missing %q in\n%s", want, calls)
		}
	}
	if strings.Index(calls, "show -p") > strings.Index(calls, "stop ") {
		t.Errorf("stop before show:\n%s", calls)
	}
}

func TestSeedFailsWhenTheCgroupStaysPopulated(t *testing.T) {
	ctl := &fakeCtl{answers: map[string]string{"show -p InvocationID,Result": "InvocationID=feed\nResult=success\nExecMainStatus=0\nExecMainCode=exited\n"}}
	s, c := newServer(t, ctl)
	s.cgroupWaitBudget = 200 * time.Millisecond
	ev := filepath.Join(s.CgroupRoot, "carlos.slice", "carlos-instances.slice", "carlos-instances-h.example.com.slice", "carlos-instance-seed@h.example.com.service", "cgroup.events")
	if err := os.MkdirAll(filepath.Dir(ev), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(ev, []byte("populated 1\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	resp := post(t, c, "/seed?host=h.example.com&memory_max=536870912")
	if resp.StatusCode != 502 {
		t.Errorf("status %d, want 502 while processes remain", resp.StatusCode)
	}
	// stop is issued (on the detached context) before the wait gives up, not
	// only after: it is the action that actually kills the survivors (final
	// review, plan 2, item 4).
	if !strings.Contains(ctl.joined(), "stop carlos-instance-seed@h.example.com.service") {
		t.Errorf("stop was not issued while the cgroup stayed populated: %s", ctl.joined())
	}
}

// TestSeedStopRunsDespiteACancelledClientContext proves the WHOLE
// post-start sequence (the second InvocationID read, show, the cleanup
// stop, WaitCgroupEmpty) runs on a context detached from the request, not
// only the final stop (fix round 1, item 4). fakeCtl.run now honours
// ctx.Err(), and its onStart hook cancels the request context the instant
// the first "start" answers, simulating a client that hung up right then.
// Before this fix, "show" ran on the request context: a cancelled one
// failed it outright, the handler 502'd there, and the cleanup stop after
// it was never reached. Confirmed empirically: temporarily swapping in
// commit 7576db8e's handleSeed (the pre-fix-round-1 version, which used
// ctx throughout after start) and running just this test failed with
// "status 502, body show: context canceled" and never reached a stop
// call, exactly the reported bug. The swap was reverted immediately
// after, see the report for both runs.
func TestSeedStopRunsDespiteACancelledClientContext(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	ctl := &fakeCtl{
		answers: map[string]string{
			"show -p InvocationID,Result,ExecMainStatus,ExecMainCode": "InvocationID=feed\nResult=success\nExecMainStatus=0\nExecMainCode=exited\n",
		},
		onStart: cancel,
	}
	root := t.TempDir()
	s := &Server{
		Systemctl:  ctl.run,
		PeerOK:     func(net.Conn) error { return nil },
		CgroupRoot: filepath.Join(root, "cgroup"),
		Logf:       t.Logf,
	}
	s.cgroupWaitBudget = time.Second
	ev := filepath.Join(s.CgroupRoot, "carlos.slice", "carlos-instances.slice", "carlos-instances-h.example.com.slice", "carlos-instance-seed@h.example.com.service", "cgroup.events")
	if err := os.MkdirAll(filepath.Dir(ev), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(ev, []byte("populated 0\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	r := httptest.NewRequest("POST", "/seed?host=h.example.com&memory_max=536870912", nil).WithContext(ctx)
	w := httptest.NewRecorder()
	s.handleSeed(w, r)
	if w.Code != http.StatusOK {
		t.Fatalf("status %d, body %s, want 200 despite the cancelled client context", w.Code, w.Body.String())
	}
	var out SeedResponse
	if err := json.NewDecoder(w.Body).Decode(&out); err != nil || out.InvocationID != "feed" || out.Result != "success" {
		t.Errorf("response = %+v, %v", out, err)
	}
	if !strings.Contains(ctl.joined(), "stop carlos-instance-seed@h.example.com.service") {
		t.Errorf("stop did not run despite the cancelled request context:\n%s", ctl.joined())
	}
}

// TestSeedStopsAndRestartsAStaleInvocation proves the stale-result guard:
// when start leaves the SAME (non-empty) InvocationID in place, the
// handler stops the abandoned run and starts again so it reads its own
// result, never one left behind by a prior request (final review, plan 2,
// item 4).
func TestSeedStopsAndRestartsAStaleInvocation(t *testing.T) {
	var mu sync.Mutex
	var calls []string
	invocation := "stale00000000000000000000000001"
	starts := 0
	root := t.TempDir()
	systemctl := func(_ context.Context, args ...string) (string, error) {
		mu.Lock()
		calls = append(calls, strings.Join(args, " "))
		mu.Unlock()
		switch args[0] {
		case "start":
			starts++
			if starts == 2 {
				// The retry actually moves the invocation on, as a real
				// systemd start would after the stale unit was stopped.
				invocation = "fresh0000000000000000000000002"
			}
			return "", nil
		case "show":
			if len(args) >= 3 && args[2] == "InvocationID" {
				return invocation + "\n", nil
			}
			return "InvocationID=" + invocation + "\nResult=success\nExecMainStatus=0\nExecMainCode=exited\n", nil
		default:
			return "", nil
		}
	}
	s := &Server{
		Systemctl:  systemctl,
		PeerOK:     func(net.Conn) error { return nil },
		CgroupRoot: filepath.Join(root, "cgroup"),
		Logf:       t.Logf,
	}
	s.cgroupWaitBudget = time.Second
	ev := filepath.Join(s.CgroupRoot, "carlos.slice", "carlos-instances.slice", "carlos-instances-h.example.com.slice", "carlos-instance-seed@h.example.com.service", "cgroup.events")
	if err := os.MkdirAll(filepath.Dir(ev), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(ev, []byte("populated 0\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	r := httptest.NewRequest("POST", "/seed?host=h.example.com&memory_max=536870912", nil)
	w := httptest.NewRecorder()
	s.handleSeed(w, r)
	if w.Code != http.StatusOK {
		t.Fatalf("status %d, body %s", w.Code, w.Body.String())
	}
	if starts != 2 {
		t.Errorf("starts = %d, want 2 (a stale invocation must trigger a stop and retry)", starts)
	}
	mu.Lock()
	joined := strings.Join(calls, "\n")
	mu.Unlock()
	if !strings.Contains(joined, "stop carlos-instance-seed@h.example.com.service") {
		t.Errorf("no stop issued for the stale invocation:\n%s", joined)
	}
	var out SeedResponse
	if err := json.NewDecoder(w.Body).Decode(&out); err != nil || out.InvocationID != invocation {
		t.Errorf("response = %+v, %v, want the FRESH invocation %s", out, err, invocation)
	}
}

func TestStopResetAndTheirTolerances(t *testing.T) {
	ctl := &fakeCtl{fail: map[string]error{"reset-failed carlos-instance-sidecar@": errors.New("Unit carlos-instance-sidecar@h.example.com.service not loaded.")}}
	_, c := newServer(t, ctl)
	if resp := post(t, c, "/stop?host=h.example.com&role=seed"); resp.StatusCode != 204 {
		t.Errorf("stop status %d", resp.StatusCode)
	}
	if resp := post(t, c, "/reset?host=h.example.com&role=sidecar"); resp.StatusCode != 204 {
		t.Errorf("reset of an unloaded unit: status %d, want 204 (tolerated)", resp.StatusCode)
	}
	if resp := post(t, c, "/reset?host=h.example.com&role=nope"); resp.StatusCode != 400 {
		t.Errorf("bad role: status %d", resp.StatusCode)
	}
	if !strings.Contains(ctl.joined(), "stop carlos-instance-seed@h.example.com.service") {
		t.Errorf("stop not issued: %s", ctl.joined())
	}
}

func TestSweepStopsEveryInstanceUnitAndJob(t *testing.T) {
	ctl := &fakeCtl{answers: map[string]string{
		"list-units": "carlos-instance@a.example.com.service loaded active running carlos exec-backed instance a.example.com\n" +
			"carlos-instance-sidecar@a.example.com.service loaded inactive dead ...\n" +
			"carlos-instance-seed@_aaaa\\x2d0123456789abcdef.service loaded activating start ...\n",
		"list-jobs":                   "12 carlos-instance@b.example.com.service start waiting\n",
		"show -p ActiveState --value": "inactive\n",
	}}
	s, c := newServer(t, ctl)
	s.cgroupWaitBudget = time.Second
	resp := post(t, c, "/sweep")
	if resp.StatusCode != 200 {
		t.Fatalf("status %d", resp.StatusCode)
	}
	var out SweepResponse
	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
		t.Fatal(err)
	}
	got := strings.Join(out.Units, " ")
	for _, want := range []string{"carlos-instance@a.example.com.service", "carlos-instance-seed@_aaaa\\x2d0123456789abcdef.service", "carlos-instance@b.example.com.service"} {
		if !strings.Contains(got, want) {
			t.Errorf("sweep did not stop %s: %v", want, out.Units)
		}
	}
	if strings.Contains(got, "sidecar@a") {
		t.Errorf("sweep stopped an inactive unit with no job: %v", out.Units)
	}
	if len(out.Failed) != 0 {
		t.Errorf("nothing should have failed: %v", out.Failed)
	}
	calls := ctl.joined()
	if !strings.Contains(calls, "stop carlos-instance@a.example.com.service") || !strings.Contains(calls, "stop carlos-instance@b.example.com.service") {
		t.Errorf("stops not issued:\n%s", calls)
	}
}

// TestSweepStopsOthersAndReportsAStuckUnitAsFailed proves a sweep never
// aborts on the first unit whose cgroup won't empty: the other targets are
// still stopped and reported swept, and the stuck one is reported failed
// rather than losing the whole response (codex review fix round 2 item 3).
func TestSweepStopsOthersAndReportsAStuckUnitAsFailed(t *testing.T) {
	ctl := &fakeCtl{answers: map[string]string{
		"list-units": "carlos-instance@a.example.com.service loaded active running carlos exec-backed instance a.example.com\n" +
			"carlos-instance@stuck.example.com.service loaded active running carlos exec-backed instance stuck.example.com\n",
		"list-jobs":                   "",
		"show -p ActiveState --value": "inactive\n",
	}}
	s, c := newServer(t, ctl)
	s.cgroupWaitBudget = 200 * time.Millisecond
	ev := filepath.Join(s.CgroupRoot, "carlos.slice", "carlos-instances.slice", "carlos-instances-stuck.example.com.slice", "carlos-instance@stuck.example.com.service", "cgroup.events")
	if err := os.MkdirAll(filepath.Dir(ev), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(ev, []byte("populated 1\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	resp := post(t, c, "/sweep")
	if resp.StatusCode != 502 {
		t.Fatalf("status %d, want 502 while a unit is stuck", resp.StatusCode)
	}
	var out SweepResponse
	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
		t.Fatal(err)
	}
	got := strings.Join(out.Units, " ")
	if !strings.Contains(got, "carlos-instance@a.example.com.service") {
		t.Errorf("the other unit was not stopped and reported: %v", out.Units)
	}
	if strings.Contains(got, "stuck") {
		t.Errorf("the stuck unit must not be reported as swept: %v", out.Units)
	}
	if _, ok := out.Failed["carlos-instance@stuck.example.com.service"]; !ok {
		t.Errorf("the stuck unit is missing from failed: %v", out.Failed)
	}
	calls := ctl.joined()
	if !strings.Contains(calls, "stop carlos-instance@a.example.com.service") || !strings.Contains(calls, "stop carlos-instance@stuck.example.com.service") {
		t.Errorf("both units were not stopped:\n%s", calls)
	}
}

// TestReadyBeforeTheFirstProbeNamesTheReason pins that /ready answers with
// an explicit "not yet run" line before RunProbe has ever completed, not
// an empty Lines that could be misread as every check having passed
// (final review, plan 2, item 6).
func TestReadyBeforeTheFirstProbeNamesTheReason(t *testing.T) {
	ctl := &fakeCtl{}
	_, c := newServer(t, ctl)
	resp, err := c.Get("http://unix/ready")
	if err != nil || resp.StatusCode != 503 {
		t.Fatalf("status: %v, %v", resp, err)
	}
	var got ProbeReport
	if err := json.NewDecoder(resp.Body).Decode(&got); err != nil {
		t.Fatal(err)
	}
	if got.Ready || len(got.Lines) != 1 || got.Lines[0].Check != "probe" || got.Lines[0].Verdict != "fail" || got.Lines[0].Detail != "not yet run" {
		t.Errorf("body = %+v", got)
	}
}

func TestReadyReflectsTheProbeVerdict(t *testing.T) {
	ctl := &fakeCtl{}
	s, c := newServer(t, ctl)
	// The fake "start carlos-instance-probe.service" writes the report the
	// unit's StandardOutput=truncate: would.
	report := ProbeReport{Ready: false, Lines: []VerdictLine{{Check: "pid1", Verdict: "fail", Detail: "pid=4242"}}}
	ctl.answers = map[string]string{}
	ctl.fail = map[string]error{}
	writeReport := func(r ProbeReport) {
		b, _ := json.Marshal(r)
		if err := os.WriteFile(s.ProbeOut, b, 0o644); err != nil {
			t.Fatal(err)
		}
	}
	writeReport(report)
	s.RunProbe(context.Background())
	resp, err := c.Get("http://unix/ready")
	if err != nil || resp.StatusCode != 503 {
		t.Fatalf("not-ready: %v %v", resp, err)
	}
	var got ProbeReport
	if err := json.NewDecoder(resp.Body).Decode(&got); err != nil || got.Ready || len(got.Lines) != 1 {
		t.Errorf("body = %+v, %v", got, err)
	}
	writeReport(ProbeReport{Ready: true, Lines: []VerdictLine{{Check: "pid1", Verdict: "ok"}}})
	s.RunProbe(context.Background())
	if resp, err := c.Get("http://unix/ready"); err != nil || resp.StatusCode != 204 {
		t.Fatalf("ready: %v %v", resp, err)
	}
	if !strings.Contains(ctl.joined(), "start carlos-instance-probe.service") {
		t.Errorf("probe unit not started: %s", ctl.joined())
	}
	// A missing fixture is never a pass.
	s.FixtureOK = func() error { return errors.New("probe fixture absent") }
	s.RunProbe(context.Background())
	if resp, _ := c.Get("http://unix/ready"); resp.StatusCode != 503 {
		t.Errorf("fixture absent yet ready")
	}
}

func TestPeerCheckClosesTheConnectionWithoutAResponse(t *testing.T) {
	ctl := &fakeCtl{}
	_, c := newServer(t, ctl, func(s *Server) {
		s.PeerOK = func(net.Conn) error { return errors.New("not the agent") }
	})
	_, err := c.Get("http://unix/ready")
	if err == nil {
		t.Fatal("a refused peer got an HTTP response")
	}
	if ctl.joined() != "" {
		t.Errorf("systemctl ran for a refused peer: %s", ctl.joined())
	}
}

func TestWaitCgroupEmptyTreatsAnAbsentCgroupAsEmpty(t *testing.T) {
	root := t.TempDir()
	if err := WaitCgroupEmpty(context.Background(), root, "carlos-instances-h.slice", "carlos-instance@h.service", 10*time.Millisecond); err != nil {
		t.Errorf("absent cgroup: %v", err)
	}
}

// fakeRun answers inspectMountpoints' unshare invocations by the
// mountpoint named in the command: the listing under each raw directory,
// or an exec failure.
type fakeRun struct {
	listings map[string]string // "/data/instances" -> ls -A output
	err      error
	calls    []string
}

func (f *fakeRun) run(_ context.Context, args ...string) (string, error) {
	line := strings.Join(args, " ")
	f.calls = append(f.calls, line)
	if f.err != nil {
		return "", f.err
	}
	for where, out := range f.listings {
		if strings.Contains(line, "ls -A "+where) {
			return out, nil
		}
	}
	return "", nil
}

func probeOK(t *testing.T, s *Server) {
	t.Helper()
	b, _ := json.Marshal(ProbeReport{Ready: true, Lines: []VerdictLine{{Check: "pid1", Verdict: "ok"}}})
	if err := os.WriteFile(s.ProbeOut, b, 0o644); err != nil {
		t.Fatal(err)
	}
	s.RunProbe(context.Background())
}

// Both raw directories empty: two ok lines, and readiness follows the
// probe alone.
func TestMountpointInspectionCleanIsReady(t *testing.T) {
	run := &fakeRun{}
	s, c := newServer(t, &fakeCtl{}, func(s *Server) { s.Run = run.run })
	s.InspectMountpoints(context.Background())
	probeOK(t, s)
	if resp, err := c.Get("http://unix/ready"); err != nil || resp.StatusCode != 204 {
		t.Fatalf("ready = %v %v, want 204", resp, err)
	}
	rep, ready := s.Ready()
	if !ready || len(rep.Lines) != 3 || rep.Lines[0].Check != "mountpoint-data-instances" || rep.Lines[0].Verdict != "ok" || rep.Lines[1].Check != "mountpoint-run-carlos-instances" {
		t.Fatalf("report = %+v", rep)
	}
	if len(run.calls) != 2 || !strings.HasPrefix(run.calls[0], "unshare -m --propagation private sh -c umount /data/instances") {
		t.Errorf("calls = %q", run.calls)
	}
}

// Files under the raw /data/instances: the report carries a failing
// mountpoint line and /ready is 503 even after a passing probe, on every
// probe after it.
func TestMountpointHeldFilesBlockReadiness(t *testing.T) {
	run := &fakeRun{listings: map[string]string{"/data/instances": "a.example.com\nb.example.com\n"}}
	s, c := newServer(t, &fakeCtl{}, func(s *Server) { s.Run = run.run })
	s.InspectMountpoints(context.Background())
	for i := 0; i < 2; i++ {
		probeOK(t, s)
		resp, err := c.Get("http://unix/ready")
		if err != nil || resp.StatusCode != 503 {
			t.Fatalf("probe %d: ready = %v %v, want 503", i, resp, err)
		}
		var got ProbeReport
		if err := json.NewDecoder(resp.Body).Decode(&got); err != nil || got.Ready {
			t.Fatalf("body = %+v %v", got, err)
		}
		l := got.Lines[0]
		if l.Check != "mountpoint-data-instances" || l.Verdict != "fail" || !strings.Contains(l.Detail, "mountpoint held files while unmounted: 2 entries under /data/instances") {
			t.Errorf("line = %+v", l)
		}
		if got.Lines[1].Verdict != "ok" || got.Lines[2].Check != "pid1" {
			t.Errorf("lines = %+v", got.Lines)
		}
	}
}

// unshare cannot run: both lines are unverified, in the report, and
// readiness follows the probe alone.
func TestMountpointInspectionUnverifiedDoesNotBlock(t *testing.T) {
	run := &fakeRun{err: errors.New("exec: \"unshare\": executable file not found in $PATH")}
	s, c := newServer(t, &fakeCtl{}, func(s *Server) { s.Run = run.run })
	s.InspectMountpoints(context.Background())
	probeOK(t, s)
	if resp, err := c.Get("http://unix/ready"); err != nil || resp.StatusCode != 204 {
		t.Fatalf("ready = %v %v, want 204", resp, err)
	}
	rep, _ := s.Ready()
	if len(rep.Lines) != 3 || rep.Lines[0].Verdict != "unverified" || !strings.Contains(rep.Lines[0].Detail, "unshare") {
		t.Fatalf("report = %+v", rep)
	}
	// A failing probe is still a failing probe.
	b, _ := json.Marshal(ProbeReport{Ready: false, Lines: []VerdictLine{{Check: "pid1", Verdict: "fail"}}})
	os.WriteFile(s.ProbeOut, b, 0o644)
	s.RunProbe(context.Background())
	if resp, _ := c.Get("http://unix/ready"); resp.StatusCode != 503 {
		t.Error("a failing probe read as ready")
	}
}

// Discussion #256: carlos-instance-control.service ran the previous
// release's binary for three weeks and served the new release's units.
// /ready now asks, on every call, whether this process still runs the
// installed binary, and a mismatch is a failing line whatever the probe
// said; it names the restart that clears it.
func TestReadyRefusesWhenTheRunningBinaryIsNotTheInstalledOne(t *testing.T) {
	ctl := &fakeCtl{}
	stale := errors.New("instance control is running a binary that is no longer /opt/carlos/carlos; restart carlos-instance-control.service")
	var exeErr error
	s, c := newServer(t, ctl, func(s *Server) {
		s.ExeCurrent = func() error { return exeErr }
		s.Build = "abc1234"
	})
	b, _ := json.Marshal(ProbeReport{Ready: true, Lines: []VerdictLine{{Check: "pid1", Verdict: "ok"}}})
	if err := os.WriteFile(s.ProbeOut, b, 0o644); err != nil {
		t.Fatal(err)
	}
	s.RunProbe(context.Background())
	resp, err := c.Get("http://unix/ready")
	if err != nil || resp.StatusCode != 204 {
		t.Fatalf("current binary: %v %v", resp, err)
	}
	if got := resp.Header.Get(BuildHeader); got != "abc1234" {
		t.Fatalf("%s = %q, want the service's build", BuildHeader, got)
	}
	exeErr = stale
	resp, err = c.Get("http://unix/ready")
	if err != nil || resp.StatusCode != 503 {
		t.Fatalf("stale binary: %v %v", resp, err)
	}
	if got := resp.Header.Get(BuildHeader); got != "abc1234" {
		t.Fatalf("%s on a 503 = %q, want the service's build", BuildHeader, got)
	}
	var got ProbeReport
	if err := json.NewDecoder(resp.Body).Decode(&got); err != nil {
		t.Fatal(err)
	}
	if got.Ready || len(got.Lines) != 2 || got.Lines[1].Check != "binary" || got.Lines[1].Verdict != "fail" || !strings.Contains(got.Lines[1].Detail, "restart carlos-instance-control.service") {
		t.Fatalf("body = %+v", got)
	}
	// The stored report is not what grew: the next call carries one
	// binary line, not two.
	resp, _ = c.Get("http://unix/ready")
	got = ProbeReport{}
	_ = json.NewDecoder(resp.Body).Decode(&got)
	if len(got.Lines) != 2 {
		t.Fatalf("second call = %+v, want the probe line and one binary line", got)
	}
	exeErr = nil
	if resp, err := c.Get("http://unix/ready"); err != nil || resp.StatusCode != 204 {
		t.Fatalf("after the restart: %v %v", resp, err)
	}
}

func TestStopResetAndSweepCoverTheCmdRole(t *testing.T) {
	ctl := &fakeCtl{answers: map[string]string{
		"list-units":                  "carlos-instance-cmd@a.example.com.service loaded active exited carlos exec-backed instance a.example.com (cmd)\n",
		"show -p ActiveState --value": "inactive\n",
	}}
	s, c := newServer(t, ctl)
	s.cgroupWaitBudget = time.Second
	if resp := post(t, c, "/stop?host=a.example.com&role=cmd"); resp.StatusCode != http.StatusNoContent {
		t.Fatalf("stop status %d", resp.StatusCode)
	}
	if resp := post(t, c, "/reset?host=a.example.com&role=cmd"); resp.StatusCode/100 != 2 {
		t.Fatalf("reset status %d", resp.StatusCode)
	}
	if resp := post(t, c, "/sweep"); resp.StatusCode != http.StatusOK {
		t.Fatalf("sweep status %d", resp.StatusCode)
	}
	calls := ctl.joined()
	for _, want := range []string{
		"stop carlos-instance-cmd@a.example.com.service",
		"reset-failed carlos-instance-cmd@a.example.com.service",
		"carlos-instance-cmd@*.service",
	} {
		if !strings.Contains(calls, want) {
			t.Errorf("missing %q in\n%s", want, calls)
		}
	}
}