#!/bin/bash # Mutation checks for the hostlife model (move-any-app spec ยง14.2). # # Build the test binary once (this is the only step that writes the Go # build cache), then run the sweep. The sweep never copies or edits the # source: each rule runs the same binary with HOSTLIFE_MUTATE set. # # go test -c -o "$TMPDIR/hostlife-sim/hostlife.test" ./internal/sim/hostlife/ # internal/sim/hostlife/mutate.sh 3000 # every rule # internal/sim/hostlife/mutate.sh 30000 rule-a rule-b # named rules # internal/sim/hostlife/mutate.sh 30000 rule-a,rule-b # a combination, dropped together # HOSTLIFE_BOXES=2 internal/sim/hostlife/mutate.sh 3000 # the two-box shape (region HA L18); unset is one box per region # HOSTLIFE_BOXES=2 internal/sim/hostlife/mutate.sh 3000 @regionha # the region HA rules alone (rules_test.go's region HA block) # HOSTLIFE_LIST=1 internal/sim/hostlife/mutate.sh 0 @regionha # print the rules a run would check, and run nothing # # Each rule gets two runs under its mutation (rev 7.3): the random sweep # (TestHostLifecycle over the seeds) and the rule's deterministic witness # scenarios (TestWitnesses, witness_test.go). The verdict line reports # both, separately: # # CAUGHT rule: sweep N of S (first seed n); witness violated|held|none # # Classification (joint round 2, R2-10; rev 7.3): # # CAUGHT the sweep completed and reported at least one seed that # violated an invariant, with the seed number parsed from its # output (so it can be replayed with HOSTLIFE_SEED), OR a # witness of the rule violated an invariant under the mutation; # MISSED the sweep completed with every seed passing, and every # witness held (or the rule has none); # ERROR anything else: an unknown rule, a stale binary, a panic, a # timeout, output the harness cannot parse, or a witness run # that failed (a witness that must violate under this mutation # and held: the witness is broken). An error is never a # detection. # # Expected outcomes live in expected-outcomes.txt beside this script: a # rule listed there as `missed` (liveness-only, unreachable, or caught # only in a stated combination: its witness holds alone) is reported as # EXPECTED-MISS. A rule listed as `rare ` (rev 7.11) has no witness # and is caught by the random sweep only at N seeds or more: below N a # miss is EXPECTED-MISS and a catch is CAUGHT; at N or more a miss is # MISSED. The sweep exits nonzero on any ERROR, on any unexpected MISSED, # and on any rule listed as `missed` that is CAUGHT (STALE). # At HOSTLIFE_BOXES=2 expected-outcomes.two-box.txt overrides it, rule by # rule: a line there replaces the rule's line from the base file, and # `rule - reason` drops the base line (the rule must then be caught). # Unset or 1, the base file is read as it is. # # Scratch (the binary, every run's log, the verdicts) lives under # ${HOSTLIFE_SCRATCH:-$TMPDIR/hostlife-sim}; remove it once, after the # whole sweep. set -u SEEDS=${1:?seeds} shift ROOT=$(git -C "$(dirname "$0")" rev-parse --show-toplevel) PKG="$ROOT/internal/sim/hostlife" SCRATCH=${HOSTLIFE_SCRATCH:-${TMPDIR:?}/hostlife-sim} BIN=${HOSTLIFE_BIN:-$SCRATCH/hostlife.test} SHAPE=${HOSTLIFE_BOXES:-1} RULES="$*" if [ -z "$RULES" ]; then RULES=$(sed -n '/^var allRules/,/^}/p' "$PKG/rules_test.go" | grep -o '^\s*"[a-z0-9-]*"' | tr -d ' "\t') elif [ "$RULES" = "@regionha" ]; then # every rule of the region HA block: one mutation check per guard RULES=$(sed -n '/^\t\/\/ region HA (two boxes per region, L18)/,/^}/p' "$PKG/rules_test.go" | grep -o '^\s*"[a-z0-9-]*"' | tr -d ' "\t') fi if [ -n "${HOSTLIFE_LIST:-}" ]; then printf '%s\n' $RULES exit 0 fi mkdir -p "$SCRATCH/logs" "$SCRATCH/verdicts" || { echo "cannot create $SCRATCH" >&2; exit 2; } EXPECT="$PKG/expected-outcomes.txt" EXPECT_NAME=expected-outcomes.txt # the file the verdicts cite if [ "$SHAPE" = 2 ] && [ -f "$PKG/expected-outcomes.two-box.txt" ]; then EXPECT="$SCRATCH/expected-outcomes.merged" EXPECT_NAME="expected-outcomes.txt merged with expected-outcomes.two-box.txt" awk 'FNR==NR { if ($0 !~ /^#/ && NF) ov[$1]=$0; next } !($1 in ov)' "$PKG/expected-outcomes.two-box.txt" "$PKG/expected-outcomes.txt" >"$EXPECT" awk '$0 !~ /^#/ && NF && $2 != "-"' "$PKG/expected-outcomes.two-box.txt" >>"$EXPECT" fi if [ ! -x "$BIN" ]; then echo "no test binary at $BIN; build it with: go test -c -o $BIN ./internal/sim/hostlife/" >&2 exit 2 fi if [ -n "$(find "$PKG" -name '*.go' -newer "$BIN")" ]; then echo "the test binary $BIN is older than the source; rebuild it" >&2 exit 2 fi # the rules that have witnesses (witness_test.go): a run of one of them # with no witness output is an ERROR, never "none" (R3-6) WRULES=$(grep -o '^\s*{"[a-z0-9-]*",' "$PKG/witness_test.go" | tr -d ' \t{",' | sort -u) rm -f "$SCRATCH"/verdicts/* || exit 2 one() { r=$1 tag=$(echo "$r" | tr ',' '+') log="$SCRATCH/logs/$tag.log" verdict="$SCRATCH/verdicts/$tag" for x in $(echo "$r" | tr ',' ' '); do if ! sed -n '/^var allRules/,/^}/p' "$PKG/rules_test.go" | grep -q "^\s*\"$x\","; then echo "ERROR $r [boxes=$SHAPE]: no rule named $x" | tee "$verdict" return fi done # the random sweep HOSTLIFE_MUTATE=$r HOSTLIFE_SEEDS=$SEEDS "$BIN" -test.run '^TestHostLifecycle$' -test.count=1 -test.v -test.timeout 180m >"$log" 2>&1 rc=$? expect=$(awk -v r="$r" '$1==r {print $2}' "$EXPECT" 2>/dev/null) depth=$(awk -v r="$r" '$1==r {print $3}' "$EXPECT" 2>/dev/null) nviol=$(grep -o "[0-9]* of $SEEDS seeds violated an invariant" "$log" | head -1 | awk '{print $1}') seed=$(grep -o "first seed [0-9]*" "$log" | head -1 | awk '{print $3}') clean=$(grep -c "^ sim_test.go:[0-9]*: $SEEDS seeds, 0 failed.*mutated=\"$r\"" "$log") sweep="" if [ $rc -ne 0 ] && [ -n "$nviol" ] && [ -n "$seed" ] && ! grep -q "^panic:\|test timed out" "$log"; then sweep="sweep $nviol of $SEEDS (first seed $seed)" elif [ $rc -eq 0 ] && [ "$clean" -eq 1 ]; then sweep="sweep 0 of $SEEDS" fi # the witnesses (rev 7.3): the rule's own hand-scheduled scenarios, run # under the same mutation; "violated" if any of them violates an # invariant, "held" if they all hold, "none" if the rule has no witness. # A witness that must violate under this mutation and holds fails the # test, which is an ERROR here: the witness is broken, not the rule wlog="$SCRATCH/logs/$tag.witness.log" HOSTLIFE_MUTATE=$r "$BIN" -test.run '^TestWitnesses$' -test.count=1 -test.v -test.timeout 10m >"$wlog" 2>&1 wrc=$? wit=none if grep -q "^ witness_test.go:[0-9]*: witness .*: violated" "$wlog"; then wit=violated elif grep -q "^ witness_test.go:[0-9]*: witness .*: held" "$wlog"; then wit=held fi if [ $wrc -ne 0 ] || grep -q "^panic:\|test timed out" "$wlog"; then echo "ERROR $r [boxes=$SHAPE]: witness run rc=$wrc; see $wlog" | tee "$verdict" return fi if [ "$wit" = none ]; then for x in $(echo "$r" | tr ',' ' '); do if echo "$WRULES" | grep -qx "$x"; then echo "ERROR $r [boxes=$SHAPE]: $x has witnesses but the run produced no witness output; see $wlog" | tee "$verdict" return fi done fi if [ -z "$sweep" ]; then echo "ERROR $r [boxes=$SHAPE]: rc=$rc, no parsable sweep verdict; see $log" | tee "$verdict" return fi caught=0 if [ "$sweep" != "sweep 0 of $SEEDS" ] || [ "$wit" = violated ]; then caught=1 fi if [ $caught -eq 1 ]; then if [ "$expect" = "missed" ]; then echo "STALE $r [boxes=$SHAPE]: $sweep; witness $wit; but $EXPECT_NAME says missed" | tee "$verdict" else echo "CAUGHT $r [boxes=$SHAPE]: $sweep; witness $wit" | tee "$verdict" fi elif [ "$expect" = "missed" ]; then echo "EXPECTED-MISS $r [boxes=$SHAPE]: $sweep; witness $wit" | tee "$verdict" elif [ "$expect" = "rare" ] && [ -n "$depth" ] && [ "$SEEDS" -lt "$depth" ]; then echo "EXPECTED-MISS $r [boxes=$SHAPE]: $sweep; witness $wit (rare: caught at $depth seeds)" | tee "$verdict" else echo "MISSED $r [boxes=$SHAPE]: $sweep; witness $wit" | tee "$verdict" fi } export -f one export SEEDS PKG SCRATCH BIN WRULES SHAPE EXPECT EXPECT_NAME printf '%s\n' $RULES | xargs -P "${HOSTLIFE_PAR:-8}" -I{} bash -c 'one "$@"' _ {} xrc=$? if [ $xrc -ne 0 ]; then echo "sweep: orchestration failed (xargs rc=$xrc)" >&2 exit 2 fi # exactly one well-formed verdict per requested mutation (R3-6) want=$(printf '%s\n' $RULES | wc -l) have=$(ls "$SCRATCH"/verdicts 2>/dev/null | wc -l) if [ "$have" -ne "$want" ]; then echo "sweep: $want mutation(s) requested, $have verdict(s) written" >&2 exit 2 fi for r in $RULES; do v="$SCRATCH/verdicts/$(echo "$r" | tr ',' '+')" if ! grep -qE '^(CAUGHT|MISSED|EXPECTED-MISS|STALE|ERROR) ' "$v" 2>/dev/null; then echo "sweep: no well-formed verdict for $r" >&2 exit 2 fi done bad=$(cat "$SCRATCH"/verdicts/* | grep -c "^ERROR\|^MISSED\|^STALE") if [ "$bad" -gt 0 ]; then echo "sweep: $bad rule(s) ERROR, MISSED or STALE" >&2 exit 1 fi