//go:build linux

package host

import (
	"errors"
	"os"
	"path/filepath"
	"sync/atomic"
	"syscall"
	"testing"
	"time"

	"github.com/carlosframework/platform/internal/hostbox"
	"github.com/carlosframework/platform/internal/hostclaim"
	"github.com/carlosframework/platform/internal/registry"
	"github.com/carlosframework/platform/internal/release"
)

// The held-stage lock's file is removed when the stage finishes (region HA
// M-K25, answered by moves on #236): main's empty-installation census
// refuses a file that names the host being installed, so a lock left
// behind would refuse every later from-empty install of that host on the
// box that held a stage for it.

// heldLockCensus runs main's empty-installation census for a from-empty
// install of host on a box whose data root is data, with the production
// classification (claimGlobalSharedPaths) and any extra data entries.
func heldLockCensus(t *testing.T, data, host string, extra ...string) error {
	t.Helper()
	base := t.TempDir()
	inventory, err := openLegacyInventory(t, filepath.Join(base, "inventory.db"))
	if err != nil {
		t.Fatal(err)
	}
	defer inventory.Close()
	reg, err := registry.Open(filepath.Join(base, "registry.db"))
	if err != nil {
		t.Fatal(err)
	}
	defer reg.Close()
	launches, err := hostbox.OpenLaunchJournal(filepath.Join(base, "launches.db"))
	if err != nil {
		t.Fatal(err)
	}
	defer launches.Close()
	r := ClaimRuntimeResources{DataDir: data, OptDir: filepath.Join(base, "opt"), UnitsDir: filepath.Join(base, "units")}
	for _, dir := range []string{r.OptDir, r.UnitsDir} {
		if err := os.Mkdir(dir, 0o755); err != nil {
			t.Fatal(err)
		}
	}
	shared := claimGlobalSharedPaths(r)
	shared["data"] = append(shared["data"], extra...)
	c := hostbox.EmptyInstallationCensus{Inventory: inventory, Registry: reg, Launches: launches, Box: "box", Enrollment: "enrollment", DirectoryRevision: "1",
		Roots: map[string]string{"data": r.DataDir, "opt": r.OptDir, "units": r.UnitsDir}, SharedPaths: shared}
	obs := hostclaim.Observed{Host: host, Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "inc", Stamp: 1}}}
	return c.CheckLocal(t.Context(), obs, hostbox.RecordSelection{Host: host, Incarnation: "inc"})
}

func heldLockInode(t *testing.T, path string) uint64 {
	t.Helper()
	info, err := os.Lstat(path)
	if err != nil {
		t.Fatal(err)
	}
	return info.Sys().(*syscall.Stat_t).Ino
}

func TestHeldStageLockIsRemovedWhenTheStageFinishes(t *testing.T) {
	t.Parallel()
	data := t.TempDir()
	path := filepath.Join(data, ".moves", heldStageLockFile(hsHost))
	if err := heldLockCensus(t, data, hsHost); err != nil {
		t.Fatalf("census before any stage: %v", err)
	}
	for _, result := range []error{nil, errors.New("the stage failed")} {
		err := withHeldStageLock(data, hsHost, func() error {
			if _, err := os.Lstat(path); err != nil {
				t.Fatalf("no lock file while the lock is held: %v", err)
			}
			return result
		})
		if err != result {
			t.Fatalf("stage: %v, want %v", err, result)
		}
		if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
			t.Fatalf("after a stage that returned %v, %s remains: %v", result, path, err)
		}
	}
	// The publication path: heldStageAuthority's then, inside the freeze.
	s := release.NewDirStore(t.TempDir())
	hsClaim(t, s, movingOp())
	hsReport(t, s, hsOp, "v7")
	auth := heldStageAuthority(testRegistry(t), s, hsBox, data)
	published := false
	if err := auth(hsRequest(), func() error { published = true; return nil }); err != nil || !published {
		t.Fatalf("publication: %v (ran %v)", err, published)
	}
	if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
		t.Fatalf("after a publication, %s remains: %v", path, err)
	}
	// A from-empty install of that host on this box passes main's census.
	if err := heldLockCensus(t, data, hsHost); err != nil {
		t.Fatalf("census after held stages of the host: %v", err)
	}
}

// A waiter that opened the file a holder then removes must not lock that
// file and run beside whoever holds the new one at the same name: it
// re-checks the name after its flock and retries.
func TestHeldStageLockWaiterOnARemovedFileRetries(t *testing.T) {
	prevWait, prevHook := heldLockWait, lockNamedOpened
	t.Cleanup(func() { heldLockWait, lockNamedOpened = prevWait, prevHook })
	heldLockWait = 10 * time.Second
	data := t.TempDir()
	path := filepath.Join(data, ".moves", heldStageLockFile(hsHost))
	var holders, overlaps atomic.Int32
	hold := func(entered chan<- struct{}, leave <-chan struct{}) func() error {
		return func() error {
			if holders.Add(1) != 1 {
				overlaps.Add(1)
			}
			defer holders.Add(-1)
			close(entered)
			if leave != nil {
				<-leave
			}
			return nil
		}
	}

	// The first holder holds the lock on the first file.
	in1, out1, done1 := make(chan struct{}), make(chan struct{}), make(chan error, 1)
	go func() { done1 <- withHeldStageLock(data, hsHost, hold(in1, out1)) }()
	<-in1

	// The waiter opens that file and is parked before its flock.
	opened, resume := make(chan struct{}), make(chan struct{})
	var opens atomic.Int32
	lockNamedOpened = func(name string) {
		if name == heldStageLockFile(hsHost) && opens.Add(1) == 1 {
			close(opened)
			<-resume
		}
	}
	inW, doneW := make(chan struct{}), make(chan error, 1)
	go func() { doneW <- withHeldStageLock(data, hsHost, hold(inW, nil)) }()
	<-opened

	// The first holder finishes: it removes its file and releases.
	close(out1)
	if err := <-done1; err != nil {
		t.Fatal(err)
	}
	if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
		close(resume)
		t.Fatalf("the first holder left %s: %v", path, err)
	}

	// A second holder arrives and locks a new file at the same name.
	in2, out2, done2 := make(chan struct{}), make(chan struct{}), make(chan error, 1)
	go func() { done2 <- withHeldStageLock(data, hsHost, hold(in2, out2)) }()
	<-in2

	// The waiter resumes. Its fd names the removed file, which nobody
	// holds: it must not run until the second holder is done.
	close(resume)
	select {
	case <-inW:
		close(out2)
		t.Fatalf("the waiter ran while the second holder held the lock (overlaps %d)", overlaps.Load())
	case <-time.After(500 * time.Millisecond):
	}
	close(out2)
	if err := <-done2; err != nil {
		t.Fatal(err)
	}
	select {
	case <-inW:
	case <-time.After(5 * time.Second):
		t.Fatal("the waiter never took the lock after the second holder released it")
	}
	if err := <-doneW; err != nil {
		t.Fatal(err)
	}
	if n := overlaps.Load(); n != 0 {
		t.Fatalf("%d holders ran beside another", n)
	}
	if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
		t.Fatalf("after every holder finished, %s remains: %v", path, err)
	}
}

// A holder killed mid-stage never reaches the removal, so its file stays
// (the existing behaviour). Main's census then refuses an install of that
// host; region HA's fallback (CensusTree(".moves")) refuses that host only.
// The next stage of the host locks the file the crash left and removes it.
func TestHeldStageLockLeftByACrashIsReusedThenRemoved(t *testing.T) {
	t.Parallel()
	data := t.TempDir()
	path := filepath.Join(data, ".moves", heldStageLockFile(hsHost))
	if err := os.Mkdir(filepath.Dir(path), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(path, nil, 0o644); err != nil {
		t.Fatal(err)
	}
	left := heldLockInode(t, path)
	if err := heldLockCensus(t, data, hsHost); !errors.Is(err, hostbox.ErrTypedInventory) {
		t.Fatalf("main's census with a crashed stage's lock: %v, want ErrTypedInventory", err)
	}
	if err := heldLockCensus(t, data, hsHost, hostbox.CensusTree(".moves")); !errors.Is(err, hostbox.ErrTypedInventory) {
		t.Fatalf("the fallback census for the lock's host: %v, want ErrTypedInventory", err)
	}
	if err := heldLockCensus(t, data, "other.example.com", hostbox.CensusTree(".moves")); err != nil {
		t.Fatalf("the fallback census for another host: %v", err)
	}
	ran := false
	if err := withHeldStageLock(data, hsHost, func() error {
		ran = true
		if got := heldLockInode(t, path); got != left {
			t.Fatalf("the next stage locked a new file (inode %d), not the one the crash left (%d)", got, left)
		}
		return nil
	}); err != nil || !ran {
		t.Fatalf("the next stage: %v (ran %v)", err, ran)
	}
	if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
		t.Fatalf("the next stage left %s: %v", path, err)
	}
	if err := heldLockCensus(t, data, hsHost); err != nil {
		t.Fatalf("census after the next stage: %v", err)
	}
}

// A holder's removal can land inside a waiter's open: the name vanishes
// between the create and the reopen, or the file reopened has no link
// left. lockNamed opens again then, but keeps the refusal of anything
// else at the name.
func TestHeldStageLockOpenRetriesOnlyTheRemovalRace(t *testing.T) {
	data := t.TempDir()
	d, err := openMovesDir(data, true)
	if err != nil {
		t.Fatal(err)
	}
	defer d.Close()
	at := func(name string) string { return filepath.Join(data, ".moves", name) }
	if err := os.WriteFile(at("fresh.lock"), nil, 0o644); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(at("linked.lock"), nil, 0o644); err != nil {
		t.Fatal(err)
	}
	if err := os.Link(at("linked.lock"), at("second.lock")); err != nil {
		t.Fatal(err)
	}
	if err := os.Symlink(t.TempDir(), at("planted.lock")); err != nil {
		t.Fatal(err)
	}
	if err := syscall.Mkfifo(at("fifo.lock"), 0o644); err != nil {
		t.Fatal(err)
	}
	unlinked := &os.PathError{Op: "open", Path: "x", Err: errors.New("not a regular file with one link")}
	for _, c := range []struct {
		name string
		err  error
		want bool
	}{
		{"gone.lock", &os.PathError{Op: "open", Path: "x", Err: syscall.ENOENT}, true},
		{"gone.lock", unlinked, true},
		{"fresh.lock", unlinked, true},
		{"fresh.lock", &os.PathError{Op: "open", Path: "x", Err: syscall.EACCES}, false},
		{"linked.lock", unlinked, false},
		{"planted.lock", unlinked, false},
		{"fifo.lock", unlinked, false},
	} {
		if got := d.removedDuringOpen(c.name, c.err); got != c.want {
			t.Errorf("%s after %v: retry %v, want %v", c.name, c.err, got, c.want)
		}
	}
	// A .moves removed under an open movesDir: nothing can be created in
	// it, so the open's error stands at once.
	gone := t.TempDir()
	g, err := openMovesDir(gone, true)
	if err != nil {
		t.Fatal(err)
	}
	defer g.Close()
	if err := os.Remove(filepath.Join(gone, ".moves")); err != nil {
		t.Fatal(err)
	}
	if g.removedDuringOpen("gone.lock", &os.PathError{Op: "open", Path: "x", Err: syscall.ENOENT}) {
		t.Error("a removed .moves was retried")
	}
}

// Fix round 1: the re-check's retries are bounded by the one wait. With
// every file the waiter opens replaced at the name before its flock, each
// re-check finds another file there; the call still ends with the busy
// error once the wait has passed, and never runs fn. Publication waits
// inside the registry's freeze, so the wait must hold.
func TestHeldStageLockRetriesEndWithTheWait(t *testing.T) {
	prevWait, prevHook := heldLockWait, lockNamedOpened
	const wait = 200 * time.Millisecond
	heldLockWait = wait
	data := t.TempDir()
	path := filepath.Join(data, ".moves", heldStageLockFile(hsHost))
	var stop atomic.Bool
	var replaced atomic.Int32
	lockNamedOpened = func(name string) {
		if name != heldStageLockFile(hsHost) || stop.Load() {
			return
		}
		if err := os.Remove(path); err != nil {
			return
		}
		if err := os.WriteFile(path, nil, 0o644); err == nil {
			replaced.Add(1)
		}
	}
	var ran atomic.Bool
	done := make(chan error, 1)
	start := time.Now()
	go func() { done <- withHeldStageLock(data, hsHost, func() error { ran.Store(true); return nil }) }()
	t.Cleanup(func() {
		// A waiter still retrying is let through before the seam goes.
		stop.Store(true)
		select {
		case <-done:
		case <-time.After(10 * time.Second):
		}
		heldLockWait, lockNamedOpened = prevWait, prevHook
	})
	var err error
	select {
	case err = <-done:
		done <- err
	case <-time.After(5 * time.Second):
		t.Fatalf("still retrying %v after a %v wait (%d files replaced, ran %v)", time.Since(start), wait, replaced.Load(), ran.Load())
	}
	elapsed := time.Since(start)
	if !errors.Is(err, errHeldLockBusy) {
		t.Fatalf("with every opened file replaced: %v, want errHeldLockBusy", err)
	}
	if ran.Load() {
		t.Fatal("fn ran without the lock")
	}
	if elapsed > wait+2*time.Second {
		t.Fatalf("gave up after %v, want about %v", elapsed, wait)
	}
	if replaced.Load() < 2 {
		t.Fatalf("only %d files were replaced; the re-check was not exercised", replaced.Load())
	}
	// Fix round 2: replacement retries pause (lockNamedRetryPause, capped
	// at what is left of the wait), so they cannot spin. A pause is a floor
	// on each retry's time, so this bound holds however loaded the box.
	if most := int32(wait/lockNamedRetryPause) + 2; replaced.Load() > most {
		t.Fatalf("%d replacement retries in a %v wait, want at most %d", replaced.Load(), wait, most)
	}
}

// Fix round 2: an open that meets a holder's removal after the wait has
// passed ends in the caller's busy error, like every other way the wait
// runs out, never the open's own error. Every open here meets that race:
// it fails as an unlinked file does and the name is gone.
func TestHeldStageLockOpenRaceAfterTheWaitIsBusy(t *testing.T) {
	prevWait, prevOpen := heldLockWait, lockNamedOpenAt
	t.Cleanup(func() { heldLockWait, lockNamedOpenAt = prevWait, prevOpen })
	const wait = 150 * time.Millisecond
	heldLockWait = wait
	var opens atomic.Int32
	lockNamedOpenAt = func(dir int, name string, flags int, mode os.FileMode) (*os.File, error) {
		opens.Add(1)
		return nil, &os.PathError{Op: "open", Path: name, Err: errors.New("not a regular file with one link")}
	}
	ran := false
	start := time.Now()
	err := withHeldStageLock(t.TempDir(), hsHost, func() error { ran = true; return nil })
	elapsed := time.Since(start)
	if !errors.Is(err, errHeldLockBusy) {
		t.Fatalf("every open raced a removal until past the wait: %v, want errHeldLockBusy", err)
	}
	if ran {
		t.Fatal("fn ran without the lock")
	}
	if elapsed < wait || elapsed > wait+time.Second {
		t.Fatalf("gave up after %v, want about %v", elapsed, wait)
	}
	if opens.Load() < 2 {
		t.Fatalf("only %d opens; the race was not retried", opens.Load())
	}
}