package instancectl

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/carlosframework/platform/internal/cgroup"
)

const genHost = "bill-2.example.com"

var genUnit = GenUnitName(genHost, 7)

// genShow is startGen's one look at systemd: the unit's drop-in lines and
// its host slice's tier, in one call.
var genShow = "show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax,CPUWeight,CPUQuotaPerSecUSec,IOWeight " + genUnit + " " + SliceName(genHost)

// genSliceAtTier is that call's second block for a slice already at the
// default tier.
const genSliceAtTier = "\nSlice=carlos.slice\nNeedDaemonReload=no\nMemoryMax=536870912\nMemoryHigh=483183819\nMemorySwapMax=536870912\nCPUWeight=100\nCPUQuotaPerSecUSec=1s\nIOWeight=100\n"

// genTierShow is systemd's answer for the default tier's memory lines.
const genTierShow = "MemoryMax=536870912\nMemoryHigh=483183819\nMemorySwapMax=536870912\n"

func genServer(t *testing.T, ctl *fakeCtl) (*Server, string, string) {
	t.Helper()
	dropins, exe := t.TempDir(), t.TempDir()
	s, sock := serveOnSocket(t, ctl, func(s *Server) { s.DropInDir = dropins; s.ExeRoot = exe })
	return s, sock, dropins
}

func genCtl() *fakeCtl {
	return &fakeCtl{answers: map[string]string{
		"show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax": "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=no\n" + genTierShow,
		"show -p InvocationID --value":                                      "abc123\n",
		"list-units":                                                        "",
		"list-jobs":                                                         "",
		"show -p ActiveState --value":                                       "inactive\n",
	}}
}

func TestGenDropInText(t *testing.T) {
	want := `# Written by carlos instance-control for one generation; removed when it stops (zero-downtime plan 2).
[Service]
Slice=carlos-instances-bill\x2d2.example.com.slice
BindPaths=/data/instances/bill-2.example.com /run/carlos/instances/bill-2.example.com
ReadWritePaths=/data/instances/bill-2.example.com /run/carlos/instances/bill-2.example.com
WorkingDirectory=/data/instances/bill-2.example.com
BindReadOnlyPaths=/run/carlos-control/exe/bill-2.example.com@7/app:/opt/carlos/app /run/carlos-control/exe/bill-2.example.com@7/live:/opt/carlos/live
StandardOutput=file:/run/carlos/logs/bill-2.example.com@7.run.sock
MemoryMax=536870912
MemoryHigh=483183819
MemorySwapMax=536870912
`
	if got := GenDropIn(genHost, 7, cgroup.TierDefault); got != want {
		t.Errorf("GenDropIn =\n%s\nwant\n%s", got, want)
	}
}

// The wake path's case: a generation's unit is new, so systemd reads its
// drop-in on the first reference and the box-wide reload is skipped.
func TestGenStartWritesTheDropInChecksItThenStartsWithoutAReload(t *testing.T) {
	ctl := genCtl()
	s, sock, dropins := genServer(t, ctl)
	inv, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
	if err != nil || inv != "abc123" {
		t.Fatalf("StartGen = %q, %v", inv, err)
	}
	raw, err := os.ReadFile(filepath.Join(dropins, genUnit+".d", DropInName))
	if err != nil || string(raw) != GenDropIn(genHost, 7, cgroup.TierDefault) {
		t.Fatalf("drop-in = %q, %v", raw, err)
	}
	if target, err := os.Readlink(filepath.Join(s.ExeRoot, GenKey(genHost, 7), "app")); err != nil || target != "/opt/carlos/live/"+genHost {
		t.Fatalf("exe app link = %q, %v", target, err)
	}
	want := strings.Join([]string{
		genShow,
		`set-property --runtime carlos-instances-bill\x2d2.example.com.slice MemoryMax=536870912 MemoryHigh=483183819 MemorySwapMax=536870912 CPUWeight=100 CPUQuota=100% IOWeight=100`,
		"start " + genUnit,
		"show -p InvocationID --value " + genUnit,
	}, "\n")
	if got := ctl.joined(); got != want {
		t.Errorf("calls =\n%s\nwant\n%s", got, want)
	}
}

// staleUntilReload makes systemd hold a stale copy of the unit (answer)
// until a daemon-reload, after which the drop-in reads as applied.
func staleUntilReload(ctl *fakeCtl, answer string) {
	ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = answer
	ctl.before = map[string]func(){"daemon-reload": func() {
		ctl.mu.Lock()
		defer ctl.mu.Unlock()
		ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=no\n" + genTierShow
	}}
}

// systemd still holding the unit loaded from older files says so with
// NeedDaemonReload, or shows a slice the drop-in does not name. Either way
// the start reloads, checks again, and only then starts.
func TestGenStartReloadsWhenSystemdHoldsAStaleCopy(t *testing.T) {
	for name, answer := range map[string]string{
		"files changed since load": "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=yes\n" + genTierShow,
		"loaded without drop-in":   `Slice=system-carlos\x2dinstance\x2dgen.slice` + "\nNeedDaemonReload=no\n" + genTierShow,
		"show failed to answer":    "",
	} {
		t.Run(name, func(t *testing.T) {
			ctl := genCtl()
			staleUntilReload(ctl, answer)
			_, sock, _ := genServer(t, ctl)
			if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
				t.Fatal(err)
			}
			want := strings.Join([]string{
				genShow,
				"daemon-reload",
				genShow,
			}, "\n")
			if got := ctl.joined(); !strings.HasPrefix(got, want+"\n") || !strings.Contains(got, "\nstart "+genUnit) {
				t.Errorf("calls =\n%s\nwant them to begin\n%s\nand then start", got, want)
			}
		})
	}
}

// systemd's change check is by mtime, so a reused generation still loaded
// with an older tier, whose rewritten drop-in is not newer (a clock that
// went back), reads NeedDaemonReload=no. The drop-in's other lines follow
// from the unit's name; its memory lines are checked against the tier.
func TestGenStartReloadsWhenSystemdHoldsAnotherTier(t *testing.T) {
	ctl := genCtl()
	staleUntilReload(ctl, "Slice="+SliceName(genHost)+"\nNeedDaemonReload=no\nMemoryMax=1073741824\nMemoryHigh=966367641\nMemorySwapMax=1073741824\n")
	_, sock, _ := genServer(t, ctl)
	if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
		t.Fatal(err)
	}
	if got := ctl.joined(); !strings.Contains(got, "daemon-reload\n") {
		t.Errorf("started on systemd's stale tier without a reload:\n%s", got)
	}
}

// A reload that leaves systemd still saying its files changed proves
// nothing about what it loaded: refuse rather than start on a guess.
func TestGenStartRefusesWhenTheReloadDidNotSettle(t *testing.T) {
	ctl := genCtl()
	ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=yes\n" + genTierShow
	s, sock, dropins := genServer(t, ctl)
	if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil {
		t.Fatal("started a generation systemd still held a stale copy of")
	}
	if strings.Contains(ctl.joined(), "start "+genUnit) {
		t.Errorf("the unit was started:\n%s", ctl.joined())
	}
	if dropIn, exe := genFilesPresent(t, s, dropins); dropIn || exe {
		t.Errorf("files left behind: drop-in %v exe %v", dropIn, exe)
	}
}

// A successor's start during an overlap: the slice holds the incumbent,
// and its raised limit is the swap's. The start leaves it alone.
func TestGenStartLeavesAPopulatedSliceAlone(t *testing.T) {
	ctl := genCtl()
	s, sock, _ := genServer(t, ctl)
	events := filepath.Join(s.CgroupRoot, parentSlices, SliceName(genHost), "cgroup.events")
	if err := os.MkdirAll(filepath.Dir(events), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(events, []byte("populated 1\nfrozen 0\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
		t.Fatal(err)
	}
	if strings.Contains(ctl.joined(), "set-property") {
		t.Errorf("a successor's start touched the live slice:\n%s", ctl.joined())
	}
}

// Review focus 1, nested: an empty slice is reset to the tier.
func TestGenStartResetsAnEmptySlice(t *testing.T) {
	ctl := genCtl()
	s, sock, _ := genServer(t, ctl)
	events := filepath.Join(s.CgroupRoot, parentSlices, SliceName(genHost), "cgroup.events")
	if err := os.MkdirAll(filepath.Dir(events), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(events, []byte("populated 0\nfrozen 0\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(ctl.joined(), "set-property --runtime "+SliceName(genHost)+" MemoryMax=536870912") {
		t.Errorf("a cold start left the slice as it was:\n%s", ctl.joined())
	}
}

// Review focus 5: a drop-in systemd did not apply means the default slice
// and an empty /data. Refuse, and leave nothing behind.
func TestGenStartRefusesWhenTheDropInDidNotApply(t *testing.T) {
	ctl := genCtl()
	ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = `Slice=system-carlos\x2dinstance\x2dgen.slice` + "\nNeedDaemonReload=no\n" + genTierShow
	s, sock, dropins := genServer(t, ctl)
	if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil {
		t.Fatal("started a generation whose drop-in did not apply")
	}
	if strings.Contains(ctl.joined(), "start "+genUnit) {
		t.Errorf("the unit was started:\n%s", ctl.joined())
	}
	if _, err := os.Stat(filepath.Join(dropins, genUnit+".d")); !os.IsNotExist(err) {
		t.Errorf("drop-in left behind: %v", err)
	}
	if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
		t.Errorf("exe dir left behind: %v", err)
	}
}

func TestGenStartRefusedWithoutADropInDir(t *testing.T) {
	ctl := genCtl()
	_, sock := serveOnSocket(t, ctl)
	if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil || !strings.Contains(err.Error(), "501") {
		t.Fatalf("StartGen without DropInDir: %v, want a 501", err)
	}
	if ctl.joined() != "" {
		t.Errorf("systemctl was called:\n%s", ctl.joined())
	}
}

func TestGenParametersRefusedWhereTheyDoNotBelong(t *testing.T) {
	ctl := genCtl()
	_, c := newServer(t, ctl, func(s *Server) { s.DropInDir = t.TempDir() })
	for _, path := range []string{
		"/start?host=" + genHost + "&role=sidecar&memory_max=536870912&gen=7",
		"/start?host=" + genHost + "&role=run&memory_max=536870912&gen=0",
		"/start?host=" + genHost + "&role=run&memory_max=536870912&gen=07",
		"/seed?host=" + genHost + "&memory_max=536870912&gen=7",
		"/stop?host=" + genHost + "&role=run&kill=1",
		"/start?host=" + genHost + "&role=run&memory_max=536870912&gen=7&kill=1",
	} {
		if resp := post(t, c, path); resp.StatusCode != 400 {
			t.Errorf("%s: status %d, want 400", path, resp.StatusCode)
		}
	}
	if ctl.joined() != "" {
		t.Errorf("systemctl was called:\n%s", ctl.joined())
	}
}

func TestGenStopWithKillKillsStopsWaitsAndRemovesFiles(t *testing.T) {
	ctl := genCtl()
	s, sock, dropins := genServer(t, ctl)
	if err := s.writeGenDropIn(genHost, 7, cgroup.TierDefault); err != nil {
		t.Fatal(err)
	}
	if err := s.ensureGenExe(genHost, 7); err != nil {
		t.Fatal(err)
	}
	if err := NewClient(sock).StopGen(context.Background(), genHost, 7, true); err != nil {
		t.Fatal(err)
	}
	if want := "kill --signal=SIGKILL " + genUnit + "\nstop " + genUnit + "\nreset-failed " + genUnit; ctl.joined() != want {
		t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
	}
	if _, err := os.Stat(filepath.Join(dropins, genUnit+".d")); !os.IsNotExist(err) {
		t.Errorf("drop-in survived the stop: %v", err)
	}
	if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
		t.Errorf("exe dir survived the stop: %v", err)
	}
}

func TestParentSetsOnlyTheSliceMemory(t *testing.T) {
	ctl := genCtl()
	_, sock := serveOnSocket(t, ctl)
	c := NewClient(sock)
	if err := c.Parent(context.Background(), genHost, cgroup.OverlapLimit(cgroup.TierDefault, cgroup.TierDefault).MemoryMax); err != nil {
		t.Fatal(err)
	}
	want := `set-property --runtime carlos-instances-bill\x2d2.example.com.slice MemoryMax=1073741824 MemoryHigh=966367638 MemorySwapMax=1073741824`
	if ctl.joined() != want {
		t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
	}
	if err := c.Parent(context.Background(), genHost, 64<<20); err == nil {
		t.Error("a parent limit that is no tier sum was accepted")
	}
}

func TestSweepStopsGenerationUnitsAndRemovesTheirFiles(t *testing.T) {
	ctl := genCtl()
	ctl.answers["list-units"] = genUnit + " loaded active running carlos exec-backed instance generation\n"
	s, sock, dropins := genServer(t, ctl)
	if err := s.writeGenDropIn(genHost, 7, cgroup.TierDefault); err != nil {
		t.Fatal(err)
	}
	if err := s.ensureGenExe(genHost, 7); err != nil {
		t.Fatal(err)
	}
	out, err := NewClient(sock).Sweep(context.Background())
	if err != nil || len(out.Units) != 1 || out.Units[0] != genUnit {
		raw, _ := json.Marshal(out)
		t.Fatalf("sweep = %s, %v", raw, err)
	}
	if !strings.Contains(ctl.joined(), UnitPrefixGen+"*.service") {
		t.Errorf("list-units did not ask for generation units:\n%s", ctl.joined())
	}
	if _, err := os.Stat(filepath.Join(dropins, genUnit+".d")); !os.IsNotExist(err) {
		t.Errorf("drop-in survived the sweep: %v", err)
	}
	if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
		t.Errorf("exe dir survived the sweep: %v", err)
	}
}

// A start that failed leaves nothing behind: the unit is stopped and the
// drop-in and exe layout are removed.
func TestGenStartFailureStopsTheUnitAndRemovesTheFiles(t *testing.T) {
	ctl := genCtl()
	ctl.fail = map[string]error{"start ": errors.New("boom")}
	s, sock, dropins := genServer(t, ctl)
	if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil {
		t.Fatal("a failed start was reported as started")
	}
	if !strings.Contains(ctl.joined(), "stop "+genUnit) {
		t.Errorf("the failed unit was not stopped:\n%s", ctl.joined())
	}
	if _, err := os.Stat(filepath.Join(dropins, genUnit+".d")); !os.IsNotExist(err) {
		t.Errorf("drop-in left behind: %v", err)
	}
	if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
		t.Errorf("exe dir left behind: %v", err)
	}
}

func TestGenResetResetsTheGenerationUnitOnly(t *testing.T) {
	ctl := genCtl()
	_, sock := serveOnSocket(t, ctl)
	if err := NewClient(sock).ResetGen(context.Background(), genHost, 7); err != nil {
		t.Fatal(err)
	}
	if want := "reset-failed " + genUnit; ctl.joined() != want {
		t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
	}
}

func TestGenAndParentInputsAreValidated(t *testing.T) {
	ctl := genCtl()
	_, c := newServer(t, ctl, func(s *Server) { s.DropInDir = t.TempDir() })
	for _, path := range []string{
		"/start?host=" + genHost + "&role=run&memory_max=12345&gen=7",
		"/start?host=" + genHost + "&role=run&memory_max=536870912&gen=",
		"/start?host=" + genHost + "&role=run&memory_max=536870912&gen=-1",
		"/start?host=" + genHost + "&role=run&memory_max=536870912&gen=18446744073709551616",
		"/stop?host=" + genHost + "&role=run&gen=7&kill=true",
		"/stop?host=" + genHost + "&role=seed&gen=7",
		"/reset?host=" + genHost + "&role=run&gen=0",
		"/reset?host=" + genHost + "&role=sidecar&gen=7",
		"/parent?host=not_a_host&memory_max=536870912",
		"/parent?host=" + genHost,
		"/parent?host=" + genHost + "&memory_max=-536870912",
		"/parent?host=" + genHost + "&memory_max=0",
	} {
		if resp := post(t, c, path); resp.StatusCode != 400 {
			t.Errorf("%s: status %d, want 400", path, resp.StatusCode)
		}
	}
	if ctl.joined() != "" {
		t.Errorf("systemctl was called:\n%s", ctl.joined())
	}
}

// genFilesPresent reports whether the generation's drop-in directory and
// its executable layout are on disk.
func genFilesPresent(t *testing.T, s *Server, dropins string) (dropIn, exe bool) {
	t.Helper()
	_, err1 := os.Stat(filepath.Join(dropins, genUnit+".d"))
	_, err2 := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7)))
	return err1 == nil, err2 == nil
}

// populateGenCgroup makes the generation unit's cgroup report processes,
// so no stop can prove it empty.
func populateGenCgroup(t *testing.T, s *Server) {
	t.Helper()
	events := filepath.Join(s.CgroupRoot, parentSlices, SliceName(genHost), genUnit, "cgroup.events")
	if err := os.MkdirAll(filepath.Dir(events), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(events, []byte("populated 1\nfrozen 0\n"), 0o644); err != nil {
		t.Fatal(err)
	}
}

// systemctl start succeeded, then the invocation lookup failed: the unit
// may be running. It is killed and proven empty before its files go, and
// the answer is an ordinary start failure.
func TestGenStartWhoseInvocationLookupFailsIsKilledBeforeItsFilesGo(t *testing.T) {
	ctl := genCtl()
	ctl.fail = map[string]error{"show -p InvocationID": errors.New("bus timeout")}
	s, sock, dropins := genServer(t, ctl)
	_, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
	if err == nil || errors.Is(err, ErrGenUnproven) {
		t.Fatalf("StartGen = %v, want an ordinary start failure", err)
	}
	if want := "start " + genUnit + "\nshow -p InvocationID --value " + genUnit + "\nkill --signal=SIGKILL " + genUnit + "\nstop " + genUnit + "\nreset-failed " + genUnit; !strings.HasSuffix(ctl.joined(), want) {
		t.Errorf("calls =\n%s\nwant them to end\n%s", ctl.joined(), want)
	}
	if dropIn, exe := genFilesPresent(t, s, dropins); dropIn || exe {
		t.Errorf("files left after a proven stop: drop-in %v exe %v", dropIn, exe)
	}
}

// The same, but the generation's cgroup never empties: termination is
// unproven, so the files stay and the answer says so.
func TestGenStartWhoseCleanupIsUnprovenKeepsTheFiles(t *testing.T) {
	ctl := genCtl()
	ctl.fail = map[string]error{"show -p InvocationID": errors.New("bus timeout")}
	s, sock, dropins := genServer(t, ctl)
	s.cgroupWaitBudget = 100 * time.Millisecond
	populateGenCgroup(t, s)
	_, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
	if !errors.Is(err, ErrGenUnproven) {
		t.Fatalf("StartGen = %v, want ErrGenUnproven", err)
	}
	if dropIn, exe := genFilesPresent(t, s, dropins); !dropIn || !exe {
		t.Errorf("files removed while termination is unproven: drop-in %v exe %v", dropIn, exe)
	}
}

// A failed systemctl start whose cleanup stop also fails: nothing proves
// the unit gone, so the files stay.
func TestGenStartFailureWhoseStopFailsKeepsTheFiles(t *testing.T) {
	ctl := genCtl()
	ctl.fail = map[string]error{"start ": errors.New("boom"), "stop ": errors.New("stop timed out")}
	s, sock, dropins := genServer(t, ctl)
	_, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
	if !errors.Is(err, ErrGenUnproven) {
		t.Fatalf("StartGen = %v, want ErrGenUnproven", err)
	}
	if dropIn, exe := genFilesPresent(t, s, dropins); !dropIn || !exe {
		t.Errorf("files removed after a failed stop: drop-in %v exe %v", dropIn, exe)
	}
}

// Nothing was started when the drop-in cannot be written: the exe layout
// already made goes, and systemctl is never asked to start.
func TestGenStartDropInWriteFailureRemovesTheFiles(t *testing.T) {
	ctl := genCtl()
	s, sock, dropins := genServer(t, ctl)
	// A file where the unit's drop-in directory goes: MkdirAll fails.
	if err := os.WriteFile(filepath.Join(dropins, genUnit+".d"), nil, 0o644); err != nil {
		t.Fatal(err)
	}
	_, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
	if err == nil || errors.Is(err, ErrGenUnproven) {
		t.Fatalf("StartGen = %v, want an ordinary failure", err)
	}
	if ctl.joined() != "" {
		t.Errorf("systemctl was called:\n%s", ctl.joined())
	}
	if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); !os.IsNotExist(err) {
		t.Errorf("exe dir left behind: %v", err)
	}
}

func TestGenStartDaemonReloadFailureRemovesTheFiles(t *testing.T) {
	ctl := genCtl()
	ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=yes\n" + genTierShow
	ctl.fail = map[string]error{"daemon-reload": errors.New("bus gone")}
	s, sock, dropins := genServer(t, ctl)
	_, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax)
	if err == nil || errors.Is(err, ErrGenUnproven) {
		t.Fatalf("StartGen = %v, want an ordinary failure", err)
	}
	if strings.Contains(ctl.joined(), "start "+genUnit) {
		t.Errorf("the unit was started:\n%s", ctl.joined())
	}
	if dropIn, exe := genFilesPresent(t, s, dropins); dropIn || exe {
		t.Errorf("files left behind: drop-in %v exe %v", dropIn, exe)
	}
}

// A generation that stopped while nobody removed its files (its stop's
// answer was lost, the agent restarted) is inactive, so the sweep has no
// unit to stop. Its leftovers still go once systemd reports it inactive
// and its cgroup is empty.
func TestSweepRemovesTheFilesOfAnInactiveGeneration(t *testing.T) {
	ctl := genCtl()
	s, sock, dropins := genServer(t, ctl)
	if err := s.writeGenDropIn(genHost, 7, cgroup.TierDefault); err != nil {
		t.Fatal(err)
	}
	if err := s.ensureGenExe(genHost, 7); err != nil {
		t.Fatal(err)
	}
	out, err := NewClient(sock).Sweep(context.Background())
	if err != nil || len(out.Units) != 0 {
		t.Fatalf("sweep = %+v, %v", out, err)
	}
	if !strings.Contains(ctl.joined(), "show -p ActiveState --value "+genUnit) {
		t.Errorf("the leftover unit's state was not checked:\n%s", ctl.joined())
	}
	if dropIn, exe := genFilesPresent(t, s, dropins); dropIn || exe {
		t.Errorf("leftovers of an inactive generation survived the sweep: drop-in %v exe %v", dropIn, exe)
	}
}

// Task 15 (Astra IMPL-2-R2 residual 4): a generation can leave only its
// executable directory under ExeRoot, when its drop-in went and the exe's
// removal failed (removeGenFiles ignores errors). The sweep listed only
// DropInDir, so that leftover stayed in /run for good. It now walks
// ExeRoot too, with the same checks: the unit inactive or failed, its
// cgroup empty, and no start of it under way. A host's own directory (no
// '@') and a name that is not a generation key are never touched.
func TestSweepRemovesAnExeOnlyGenerationLeftover(t *testing.T) {
	ctl := genCtl()
	s, sock, dropins := genServer(t, ctl)
	if err := s.ensureGenExe(genHost, 7); err != nil {
		t.Fatal(err)
	}
	if err := s.ensureHostExe(genHost); err != nil {
		t.Fatal(err)
	}
	odd := []string{genHost + "@0", genHost + "@x", "@7", genHost + "@07"}
	for _, name := range odd {
		if err := os.Mkdir(filepath.Join(s.ExeRoot, name), 0o755); err != nil {
			t.Fatal(err)
		}
	}
	if _, err := NewClient(sock).Sweep(context.Background()); err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(ctl.joined(), "show -p ActiveState --value "+genUnit) {
		t.Errorf("the exe-only leftover's unit state was not checked:\n%s", ctl.joined())
	}
	if dropIn, exe := genFilesPresent(t, s, dropins); dropIn || exe {
		t.Errorf("an exe-only leftover of an inactive generation survived the sweep: drop-in %v exe %v", dropIn, exe)
	}
	if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); err == nil {
		t.Error("the generation's exe directory is still there")
	}
	if _, err := os.Lstat(filepath.Join(s.ExeRoot, genHost, "app")); err != nil {
		t.Errorf("the host's own exe directory was touched: %v", err)
	}
	for _, name := range odd {
		if _, err := os.Stat(filepath.Join(s.ExeRoot, name)); err != nil {
			t.Errorf("%s, not a generation key, was removed: %v", name, err)
		}
	}
}

// Task 15 fix round 1 (d): a leftover the sweep cannot remove is logged as
// not removed, with the removal's error. removeGenFiles dropped every
// error, so the sweep logged "removed the files of" over an exe directory
// still on disk.
func TestSweepLogsAnExeLeftoverItCouldNotRemove(t *testing.T) {
	if os.Geteuid() == 0 {
		t.Skip("root removes from a read-only directory")
	}
	var (
		mu     sync.Mutex
		logged []string
	)
	ctl := genCtl()
	exe := t.TempDir()
	s, _ := serveOnSocket(t, ctl, func(s *Server) {
		s.DropInDir, s.ExeRoot = t.TempDir(), exe
		s.Logf = func(f string, a ...any) {
			mu.Lock()
			logged = append(logged, fmt.Sprintf(f, a...))
			mu.Unlock()
		}
	})
	if err := s.ensureGenExe(genHost, 7); err != nil {
		t.Fatal(err)
	}
	if err := os.Chmod(exe, 0o555); err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = os.Chmod(exe, 0o755) })
	s.sweepGenLeftovers(context.Background(), nil)
	if _, err := os.Stat(filepath.Join(exe, GenKey(genHost, 7))); err != nil {
		t.Fatalf("the fixture's exe directory went after all: %v", err)
	}
	mu.Lock()
	all := strings.Join(logged, "\n")
	mu.Unlock()
	if strings.Contains(all, "removed the files of "+genUnit) {
		t.Errorf("the sweep logged a removal that failed:\n%s", all)
	}
	if !strings.Contains(all, "could not remove the files of "+genUnit) || !strings.Contains(all, "permission denied") {
		t.Errorf("the sweep did not log the failed removal and its error:\n%s", all)
	}
}

// An exe-only leftover is kept while its unit is not proven gone (active,
// or its cgroup populated) and while a start of that unit is preparing it:
// the start lays the exe out before it writes the drop-in, so for a moment
// a live start looks exactly like an exe-only leftover.
func TestSweepKeepsAnExeOnlyGenerationNotProvenGone(t *testing.T) {
	for name, setup := range map[string]func(*fakeCtl, *Server){
		"active":    func(c *fakeCtl, _ *Server) { c.answers["show -p ActiveState --value"] = "active\n" },
		"populated": func(_ *fakeCtl, s *Server) { populateGenCgroup(t, s) },
		"starting":  func(_ *fakeCtl, s *Server) { s.markGenStarting(genUnit, true) },
	} {
		t.Run(name, func(t *testing.T) {
			ctl := genCtl()
			s, sock, _ := genServer(t, ctl)
			setup(ctl, s)
			if err := s.ensureGenExe(genHost, 7); err != nil {
				t.Fatal(err)
			}
			if _, err := NewClient(sock).Sweep(context.Background()); err != nil {
				t.Fatal(err)
			}
			if _, err := os.Stat(filepath.Join(s.ExeRoot, GenKey(genHost, 7))); err != nil {
				t.Errorf("the exe of a generation not proven gone was removed: %v", err)
			}
		})
	}
}

// Leftovers whose unit is not proven gone stay: one systemd reports
// active, and one whose cgroup still holds processes.
func TestSweepKeepsTheFilesOfAGenerationNotProvenGone(t *testing.T) {
	for name, setup := range map[string]func(*fakeCtl, *Server){
		"active":    func(c *fakeCtl, _ *Server) { c.answers["show -p ActiveState --value"] = "active\n" },
		"populated": func(_ *fakeCtl, s *Server) { populateGenCgroup(t, s) },
	} {
		t.Run(name, func(t *testing.T) {
			ctl := genCtl()
			s, sock, dropins := genServer(t, ctl)
			setup(ctl, s)
			if err := s.writeGenDropIn(genHost, 7, cgroup.TierDefault); err != nil {
				t.Fatal(err)
			}
			if err := s.ensureGenExe(genHost, 7); err != nil {
				t.Fatal(err)
			}
			if _, err := NewClient(sock).Sweep(context.Background()); err != nil {
				t.Fatal(err)
			}
			if dropIn, exe := genFilesPresent(t, s, dropins); !dropIn || !exe {
				t.Errorf("files of a generation not proven gone were removed: drop-in %v exe %v", dropIn, exe)
			}
		})
	}
}

// genLogServer is genServer with the server's log captured.
func genLogServer(t *testing.T, ctl *fakeCtl) (*Server, string, *syncLog) {
	t.Helper()
	logs := &syncLog{}
	dropins, exe := t.TempDir(), t.TempDir()
	s, sock := serveOnSocket(t, ctl, func(s *Server) { s.DropInDir = dropins; s.ExeRoot = exe; s.Logf = logs.logf })
	return s, sock, logs
}

type syncLog struct {
	mu    sync.Mutex
	lines []string
}

func (l *syncLog) logf(format string, args ...any) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.lines = append(l.lines, fmt.Sprintf(format, args...))
}

func (l *syncLog) text() string {
	l.mu.Lock()
	defer l.mu.Unlock()
	return strings.Join(l.lines, "\n")
}

func TestStopGenResetsTheFailedStateAfterAProvenStop(t *testing.T) {
	for _, kill := range []bool{false, true} {
		t.Run(fmt.Sprintf("kill=%v", kill), func(t *testing.T) {
			ctl := genCtl()
			_, sock, _ := genLogServer(t, ctl)
			if err := NewClient(sock).StopGen(context.Background(), genHost, 7, kill); err != nil {
				t.Fatal(err)
			}
			want := "stop " + genUnit + "\nreset-failed " + genUnit
			if kill {
				want = "kill --signal=SIGKILL " + genUnit + "\n" + want
			}
			if ctl.joined() != want {
				t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want)
			}
		})
	}
}

// reset-failed is cosmetic: the stop succeeded, so the answer stays 204 and
// the failure is logged; a unit that is no longer loaded is not logged.
func TestStopGenAnswersNoContentWhenResetFailedFails(t *testing.T) {
	for _, kill := range []bool{false, true} {
		t.Run(fmt.Sprintf("kill=%v", kill), func(t *testing.T) {
			ctl := genCtl()
			ctl.fail = map[string]error{"reset-failed ": errors.New("bus timeout")}
			s, sock, logs := genLogServer(t, ctl)
			if err := s.writeGenDropIn(genHost, 7, cgroup.TierDefault); err != nil {
				t.Fatal(err)
			}
			if err := NewClient(sock).StopGen(context.Background(), genHost, 7, kill); err != nil {
				t.Fatalf("StopGen = %v, want success", err)
			}
			if want := "instance-control: reset-failed " + genUnit + ": bus timeout"; !strings.Contains(logs.text(), want) {
				t.Errorf("log =\n%s\nwant it to contain %q", logs.text(), want)
			}
			if _, err := os.Stat(filepath.Join(s.DropInDir, genUnit+".d")); !os.IsNotExist(err) {
				t.Errorf("drop-in survived the stop: %v", err)
			}
		})
	}
}

func TestStopGenIgnoresResetFailedOnAnUnloadedUnit(t *testing.T) {
	ctl := genCtl()
	ctl.fail = map[string]error{"reset-failed ": errors.New("Unit " + genUnit + " not loaded.")}
	_, sock, logs := genLogServer(t, ctl)
	if err := NewClient(sock).StopGen(context.Background(), genHost, 7, false); err != nil {
		t.Fatal(err)
	}
	if strings.Contains(logs.text(), "reset-failed") {
		t.Errorf("an unloaded unit was logged: %s", logs.text())
	}
}

func TestStopGenWhoseStopFailsDoesNotResetFailed(t *testing.T) {
	for _, kill := range []bool{false, true} {
		t.Run(fmt.Sprintf("kill=%v", kill), func(t *testing.T) {
			ctl := genCtl()
			ctl.fail = map[string]error{"stop ": errors.New("stop timed out")}
			_, sock, _ := genLogServer(t, ctl)
			if err := NewClient(sock).StopGen(context.Background(), genHost, 7, kill); err == nil {
				t.Fatal("StopGen succeeded though the stop failed")
			}
			if strings.Contains(ctl.joined(), "reset-failed") {
				t.Errorf("reset-failed after a failed stop:\n%s", ctl.joined())
			}
		})
	}
}

func TestStopGenWhoseCgroupNeverEmptiesDoesNotResetFailed(t *testing.T) {
	ctl := genCtl()
	s, sock, _ := genLogServer(t, ctl)
	s.cgroupWaitBudget = 100 * time.Millisecond
	populateGenCgroup(t, s)
	if err := NewClient(sock).StopGen(context.Background(), genHost, 7, true); err == nil {
		t.Fatal("StopGen succeeded though the cgroup never emptied")
	}
	if strings.Contains(ctl.joined(), "reset-failed") {
		t.Errorf("reset-failed after an unproven kill:\n%s", ctl.joined())
	}
}

func TestAbandonedGenStartResetsFailedOnlyAfterAProvenKill(t *testing.T) {
	ctl := genCtl()
	ctl.fail = map[string]error{"start ": errors.New("boom")}
	_, sock, _ := genLogServer(t, ctl)
	if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err == nil || errors.Is(err, ErrGenUnproven) {
		t.Fatalf("StartGen = %v, want an ordinary start failure", err)
	}
	if want := "kill --signal=SIGKILL " + genUnit + "\nstop " + genUnit + "\nreset-failed " + genUnit; !strings.HasSuffix(ctl.joined(), want) {
		t.Errorf("calls =\n%s\nwant them to end\n%s", ctl.joined(), want)
	}

	ctl = genCtl()
	ctl.fail = map[string]error{"start ": errors.New("boom"), "stop ": errors.New("stop timed out")}
	_, sock, _ = genLogServer(t, ctl)
	if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); !errors.Is(err, ErrGenUnproven) {
		t.Fatalf("StartGen = %v, want ErrGenUnproven", err)
	}
	if strings.Contains(ctl.joined(), "reset-failed") {
		t.Errorf("reset-failed after a failed kill:\n%s", ctl.joined())
	}
}

// A cold start whose slice already carries the tier (it kept it from the
// last generation: set-property --runtime lasts until a reboot) skips the
// set-property: one systemctl call fewer on a wake. The same show read it.
func TestGenStartSkipsTheTierWhenTheSliceAlreadyHasIt(t *testing.T) {
	ctl := genCtl()
	ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=no\n" + genTierShow + genSliceAtTier
	_, sock, _ := genServer(t, ctl)
	if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
		t.Fatal(err)
	}
	if got := ctl.joined(); strings.Contains(got, "set-property") {
		t.Errorf("the slice already had the tier but was set again:\n%s", got)
	}
}

// Any one value off, or the slice missing from the answer, and the tier is
// set as before.
func TestGenStartSetsTheTierWhenTheSliceDiffers(t *testing.T) {
	for name, block := range map[string]string{
		"memory":         strings.Replace(genSliceAtTier, "MemoryMax=536870912", "MemoryMax=1073741824", 1),
		"cpu quota":      strings.Replace(genSliceAtTier, "CPUQuotaPerSecUSec=1s", "CPUQuotaPerSecUSec=2s", 1),
		"io weight":      strings.Replace(genSliceAtTier, "IOWeight=100", "IOWeight=[not set]", 1),
		"no slice block": "",
	} {
		t.Run(name, func(t *testing.T) {
			ctl := genCtl()
			ctl.answers["show -p Slice,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax"] = "Slice=" + SliceName(genHost) + "\nNeedDaemonReload=no\n" + genTierShow + block
			_, sock, _ := genServer(t, ctl)
			if _, err := NewClient(sock).StartGen(context.Background(), genHost, 7, cgroup.TierDefault.MemoryMax); err != nil {
				t.Fatal(err)
			}
			if got := ctl.joined(); !strings.Contains(got, "set-property --runtime "+SliceName(genHost)) {
				t.Errorf("the slice differs (%s) but the tier was not set:\n%s", name, got)
			}
		})
	}
}

func TestCPUQuotaShowsAsSystemdShowsIt(t *testing.T) {
	for pct, want := range map[int]string{50: "500ms", 100: "1s", 150: "1.500000s", 200: "2s", 250: "2.500000s"} {
		if got := cpuQuotaShown(pct); got != want {
			t.Errorf("cpuQuotaShown(%d) = %q, want %q (systemd 257)", pct, got, want)
		}
	}
}