#!/bin/bash
# Mutation checks for the swap invariants (zero-downtime plan 3, task 13).
# Each mutant rewrites one marked line (// swapsim:Mn) in a scratch copy of
# the tree, never in the checkout. Against that copy it then runs, as
# separate go test calls, the simulation (TestSwapInvariants) and each of
# the mutant's designated witnesses, so a witness that panics cannot keep
# the sweep from running, and another witness's failure is not counted.
#
#   internal/activator/testdata/swapsim/mutate.sh 300
#
# Names after the seed count run only those mutants (a fix loop's re-check):
#   internal/activator/testdata/swapsim/mutate.sh 300 M5 M7
#
# The script runs at nice 19 (it re-executes itself under nice if it has
# to, and warns if it cannot) and one mutant at a time with GOMAXPROCS=2
# (the CPU budget on shared boxes). No witness binds a socket, so TMPDIR
# stays whatever it is; Go's own scratch goes under the script's.
#
# Each verdict line reports the sweep and every designated witness:
#   sweep: "violated at K seed(s)" (an I1-I3 violation; the sweep stops
#   after three failing seeds, so K is at most 3), "liveness only at K
#   seed(s)", "passed", "TIMEOUT", "PANIC", or "NOT RUN" (no PASS or FAIL
#   line for TestSwapInvariants).
#   witness: FAIL (an assertion failed), PASS, TIMEOUT, PANIC or NOT RUN.
# Execution integrity is recorded apart from detection. Every run the
# mutant needs (the sweep and each designated witness) must build, start,
# and end in its own PASS or FAIL line. A run that panicked, timed out, never
# ran, or did not build fails the mutant's gate even when another run caught
# the mutation; that kill is still printed, as a diagnostic.
# Verdicts:
#   CAUGHT    every run executed intact, and a designated witness failed by
#             assertion or the sweep reported an I1-I3 violation
#   BROKEN    a run panicked, timed out or never ran (the detail names it);
#             "caught" in the detail is the diagnostic kill, not a pass
#   LIVENESS  not caught; the sweep's only failure is a Swap that never
#             returned
#   MISSED    everything passed under the mutation
#   ERROR     the marker is not on exactly one line, the substitution did
#             not apply, or the mutant did not build or set up
# Exit 1 on anything but CAUGHT. Scratch: a fresh directory per run, made
# with mktemp -d under $TMPDIR (named zdd3-mut.XXXXXX) and printed first.
# Its tree copy is removed on exit; its logs stay. Remove that directory
# after. The script never touches any other path under $TMPDIR.
set -u
if [ "$(nice)" -lt 19 ] && [ -z "${SWAPSIM_RENICED:-}" ]; then
	SWAPSIM_RENICED=1 exec nice -n 19 "$0" "$@"
fi
if [ "$(nice)" -lt 19 ]; then
	echo "warning: running at niceness $(nice), not 19" >&2
fi
SEEDS=${1:-300}
shift $(($# > 0 ? 1 : 0))
ONLY=" $* "
export GOMAXPROCS=2
ROOT=$(git -C "$(dirname "$0")" rev-parse --show-toplevel) || exit 2
# This run's own scratch: never a fixed name, so two runs sharing TMPDIR
# cannot delete each other's trees or logs.
SCRATCH=$(mktemp -d "${TMPDIR:?}/zdd3-mut.XXXXXX") || exit 2
echo "scratch: $SCRATCH"
A=internal/activator
# The default build cache is read-only inside a sandbox.
if [ -z "${GOCACHE:-}" ] && [ ! -w "$(go env GOCACHE)" ]; then
	export GOCACHE=$TMPDIR/gocache
fi
mkdir -p "$SCRATCH/tree" "$SCRATCH/logs" "$SCRATCH/gowork" || exit 2
trap 'rm -rf "$SCRATCH/tree" "$SCRATCH/gowork"' EXIT
export GOTMPDIR=$SCRATCH/gowork
cp -a "$ROOT/go.mod" "$ROOT/go.sum" "$ROOT/internal" "$ROOT/launchctx" "$SCRATCH/tree/" || exit 2

# name|file|marker|designated witnesses (comma-separated)|perl substitution
# applied to the marked line only. The marker match is anchored (\b), so
# swapsim:M1 never matches swapsim:M10. The substitution is the last field,
# so it may contain "|".
MUTANTS='M1|activator.go|swapsim:M1|TestSwapWitnessBarrierJoinsBeforeTheLease,TestSwapWitnessAgentRestartMidSwap|s/a\.joinOps\(e\)/true/
M2|swap.go|swapsim:M2|TestSwapWitnessAgentRestartMidSwap|s/a\.addMemberLocked\(e, retiring\)/_ = retiring/
M3|activator.go|swapsim:M3|TestSwapWitnessHoldExecsTheStagedBuild|s/e\.pin != nil/false/
M4|opset.go|swapsim:M4|TestSwapWitnessBarrierJoinsBeforeTheLease,TestSwapWitnessAgentRestartMidSwap|s/proven := a\.stopMember\(e, m\)/proven := true; go a.stopMember(e, m)/
M5|swapadmit.go|swapsim:M5|TestSwapWitnessNoAdmissionOntoAStoppingEntry|s/!waking \&\& e\.state != actAwake/false/
M6|activator.go|swapsim:M6|TestSwapWitnessShutdownJoinsAWakingEntry,TestShutdownJoinsAWakingEntryWithSwapOff|s/waking = append\(waking, e\)/_ = waking/
M7|opset.go|swapsim:M7|TestSwapWitnessMemberStopProvesTheGenerationChildEmpty|s/return a\.genTerminated\(rt, m\.gen, m\.proc\)/return true || a.genTerminated(rt, m.gen, m.proc)/
M8|activator.go|swapsim:M8|TestSwapWitnessStopProvesTheCurrentGenerationEmpty|s/if a\.gens \&\& !a\.releasable\(e\)/if false \&\& !a.releasable(e)/
M9|activator.go|swapsim:M9|TestSwapWitnessUnprovenWakeKeepsTheLease|s/w\.genUnproven = genStartUnproven\(err\)/w.genUnproven = false/'

# runtest LOG TIMEOUT RUN [ENV...]: one go test call against the scratch
# tree, verbose so that a test that ran prints its --- PASS or --- FAIL.
runtest() {
	local log=$1 timeout=$2 run=$3
	shift 3
	(cd "$SCRATCH/tree" && env "$@" go test -v -p 1 -count=1 -timeout "$timeout" -run "$run" ./$A/) >"$log" 2>&1
}

# crashed LOG: TIMEOUT or PANIC if the test binary died, else empty.
crashed() {
	if grep -q '^panic: test timed out' "$1"; then
		echo TIMEOUT
	elif grep -qE '^(panic: |fatal error: )' "$1"; then
		echo PANIC
	fi
}

fail=0
while IFS='|' read -r name file marker witnesses sub; do
	if [ "$ONLY" != "  " ] && [[ "$ONLY" != *" $name "* ]]; then
		continue
	fi
	src="$SCRATCH/tree/$A/$file"
	# Each marker sits on exactly one line of the package's Go sources.
	lines=$(cat "$ROOT/$A"/*.go | grep -cE "$marker\\b")
	if [ "$lines" -ne 1 ]; then
		echo "ERROR     $name: $marker is on $lines lines of $A/*.go, not one"
		fail=1
		continue
	fi
	cp "$ROOT/$A/$file" "$src"
	perl -pi -e "if (/\\Q$marker\\E\\b/) { $sub }" "$src"
	if cmp -s "$ROOT/$A/$file" "$src"; then
		echo "ERROR     $name: the substitution did not apply to $file"
		fail=1
		continue
	fi

	# The sweep, on its own. Its integrity is judged apart from what it
	# found: a crash or a missing result line breaks the run even when the
	# log also reports a violation.
	slog="$SCRATCH/logs/$name.sweep.log"
	runtest "$slog" 20m '^TestSwapInvariants$' SWAP_SIM_SEEDS="$SEEDS"
	built=1
	grep -q '\[build failed\]\|\[setup failed\]' "$slog" && built=0
	inv=$(grep -c 'invariant violated at seed' "$slog")
	live=$(grep -c 'liveness failed at seed' "$slog")
	broken=""
	scrash=$(crashed "$slog")
	if [ -n "$scrash" ]; then
		broken="$broken sweep=$scrash"
	elif ! grep -qE -- '--- (PASS|FAIL): TestSwapInvariants' "$slog"; then
		broken="$broken sweep=NOT-RUN"
	fi
	if [ "$inv" -gt 0 ]; then
		sweep="violated at $inv seed(s)"
	elif [ -n "$scrash" ]; then
		sweep=$scrash
	elif ! grep -qE -- '--- (PASS|FAIL): TestSwapInvariants' "$slog"; then
		sweep="NOT RUN"
	elif [ "$live" -gt 0 ]; then
		sweep="liveness only at $live seed(s)"
	else
		sweep=passed
	fi

	# Each designated witness, on its own. A witness that crashed or never
	# reported is broken even if its log also holds a FAIL line.
	wres=""
	wcaught=0
	for w in ${witnesses//,/ }; do
		wlog="$SCRATCH/logs/$name.$w.log"
		runtest "$wlog" 5m "^$w\$"
		grep -q '\[build failed\]\|\[setup failed\]' "$wlog" && built=0
		wcrash=$(crashed "$wlog")
		if [ -n "$wcrash" ]; then
			r=$wcrash
			broken="$broken $w=$wcrash"
		elif grep -q -- "--- FAIL: $w " "$wlog"; then
			r=FAIL
			wcaught=1
		elif grep -q -- "--- PASS: $w " "$wlog"; then
			r=PASS
		else
			r="NOT RUN"
			broken="$broken $w=NOT-RUN"
		fi
		wres="$wres $w=$r"
	done
	cp "$ROOT/$A/$file" "$src"

	caught=0
	if [ $wcaught -eq 1 ] || [ "$inv" -gt 0 ]; then
		caught=1
	fi
	detail="sweep $sweep; witnesses${wres}"
	if [ $built -eq 0 ]; then
		echo "ERROR     $name: the mutant did not build or set up; see $SCRATCH/logs/$name.*"
		fail=1
	elif [ -n "$broken" ]; then
		kill="not caught"
		if [ $caught -eq 1 ]; then
			kill="caught (diagnostic only)"
		fi
		printf 'BROKEN    %s: execution integrity failed:%s; %s; %s; see %s\n' "$name" "$broken" "$kill" "$detail" "$SCRATCH/logs/$name.*"
		fail=1
	elif [ $caught -eq 1 ]; then
		echo "CAUGHT    $name: $detail"
	elif [ "$live" -gt 0 ]; then
		echo "LIVENESS  $name: $detail"
		fail=1
	else
		echo "MISSED    $name: $detail"
		fail=1
	fi
done <<<"$MUTANTS"
exit $fail