From 4738aee4c0436343fef6b7cddb6e914c12622316 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 18:56:20 +0000 Subject: [PATCH] ci: acceptance asserts POST's test count against a floor in the tree (#173) Captain Bob, 2026-11-12: "Yes". Acceptance checked POST: PASSED, so a POST that lost cases still passed. Now scripts/nightly.sh post-count, which the stick-high stage runs after dict-hash: each of the four boots must report PARITY:V4_POST with every test passed and at least the floor's tests -- v4/system/POST_FLOOR, 538 today, raised when POST grows and never lowered without Captain Bob. A boot with no POST line, not run, or no floor file is refused, not skipped. scripts/test_nightly_post_count.sh, written first and seen to fail (the stage not existing), then 7 cases: 538 and 600 pass; fewer than the floor, a failure, no POST line, a boot not run, no floor -- refused. Run on the boots of the last protocol (d7b2f6bf): all four 538 of 538. logs/20261112-185620/. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017RzZgrf7V7Zk2iczBXDRbm --- .../nightly-tests/post-count.log | 3 ++ scripts/nightly.sh | 25 +++++++++- scripts/test_nightly_post_count.sh | 46 +++++++++++++++++++ v4/system/POST_FLOOR | 3 ++ 4 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 logs/20261112-185620/nightly-tests/post-count.log create mode 100755 scripts/test_nightly_post_count.sh create mode 100644 v4/system/POST_FLOOR diff --git a/logs/20261112-185620/nightly-tests/post-count.log b/logs/20261112-185620/nightly-tests/post-count.log new file mode 100644 index 00000000..00e6d03b --- /dev/null +++ b/logs/20261112-185620/nightly-tests/post-count.log @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab567de2893e43c5ef4eb2854e2bf1b5e1ed1f5e321e456b6d4d25d96ccc9fda +size 651 diff --git a/scripts/nightly.sh b/scripts/nightly.sh index 17b9073f..18b7e62f 100755 --- a/scripts/nightly.sh +++ b/scripts/nightly.sh @@ -78,7 +78,7 @@ VERSION=${NIGHTLY_VERSION:-$LITHOS-SNAPSHOT} usage() { echo "Usage: scripts/nightly.sh accept | test-32 | proofs | images" >&2 echo " | publish | docs | publish-docs | gitea-release " >&2 - echo " | protocol [...] | protocol-report | dict-hash" >&2 + echo " | protocol [...] | protocol-report | dict-hash | post-count" >&2 echo " protocol stages: $PROTOCOL_STAGES" >&2 exit 2 } @@ -134,7 +134,28 @@ accept) python3 scripts/qemu_session.py $mode $arch $extra --label "nightly-$what" --make "QEMU_DISPLAY=none" \ | tee "$OUT/accept-$what.txt" [ "$what" != amd64 ] || cp build/amd64/kernel/starkernel.iso "$OUT/StarshipOS.iso" - [ "$what" != stick-high ] || "$(pwd)/scripts/nightly.sh" dict-hash + if [ "$what" = stick-high ]; then "$(pwd)/scripts/nightly.sh" dict-hash; "$(pwd)/scripts/nightly.sh" post-count; fi + ;; +post-count) + # Every boot of the acceptance reports POST, all passed, with at least the floor's tests -- + # the count in v4/system/POST_FLOOR, raised with POST (#173; Captain Bob, 2026-11-12: "Yes"). + # A POST that lost cases fails here, not only one that failed a case. + floor_file=${POST_FLOOR_FILE:-v4/system/POST_FLOOR} + floor=$(grep -oE '^[0-9]+' "$floor_file" 2>/dev/null | head -1 || true) + [ -n "$floor" ] || { echo "nightly: no POST floor in $floor_file -- POST's count cannot be checked" >&2; exit 1; } + bad=() + for b in amd64 aarch64 riscv64 stick-high; do + # each lookup may find nothing: under set -e that must be an answer, not the end of the script + log=$(sed -n 's/^LOG: //p' "$OUT/accept-$b.txt" 2>/dev/null | head -1 || true) + line=$( { [ -n "$log" ] && [ -f "$log" ] && grep -aoE "PARITY:V4_POST tests=[0-9]+ pass=[0-9]+ fail=[0-9]+" "$log" | head -1; } || true) + if [ -z "$line" ]; then bad+=("$b: no PARITY:V4_POST line"); continue; fi + t=$(sed -E 's/.*tests=([0-9]+).*/\1/' <<<"$line"); p=$(sed -E 's/.*pass=([0-9]+).*/\1/' <<<"$line"); f=$(sed -E 's/.*fail=([0-9]+).*/\1/' <<<"$line") + if [ "$f" != 0 ] || [ "$p" != "$t" ]; then bad+=("$b: $p of $t passed") + elif [ "$t" -lt "$floor" ]; then bad+=("$b: $t tests, fewer than the floor $floor") + fi + done + echo "=== nightly: POST on the four boots, floor $floor: ${bad[*]:-all at or above it, all passed}" + [ ${#bad[@]} = 0 ] || { echo "nightly: POST below the floor or failing -- $(IFS=';'; echo "${bad[*]}")" >&2; exit 1; } ;; dict-hash) # One dict_hash from each of the three architectures' boots, and the same (#172): an diff --git a/scripts/test_nightly_post_count.sh b/scripts/test_nightly_post_count.sh new file mode 100755 index 00000000..39a13950 --- /dev/null +++ b/scripts/test_nightly_post_count.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: LicenseRef-Starship-1.0 +# Copyright (c) 2023-2026 Robert A. James +# test_nightly_post_count.sh -- the tests of `scripts/nightly.sh post-count` (#173): every boot of the +# acceptance reports POST with at least the floor's tests (v4/system/POST_FLOOR), all passed. No +# QEMU: each case is a scratch NIGHTLY_OUT with accept-.txt files and serial logs, and a +# scratch POST_FLOOR. +# +# Prints PASS:/FAIL: per check and RESULT:; exits 0 only if every check passes. +set -uo pipefail +NIGHTLY="$(cd "$(dirname "$0")" && pwd)/nightly.sh" +T=$(mktemp -d) +trap 'rm -rf "$T"' EXIT +ok=1 + +boot() { # boot | boot none + local b=$1 log="$T/logs/$1.log" + mkdir -p "$T/out" "$T/logs" + printf 'LOG: %s\nRESULT: PASS\n' "$log" > "$T/out/accept-$b.txt" + if [ "$2" = none ]; then printf 'POST: PASSED\n' > "$log" + else printf 'PARITY:V4_POST tests=%s pass=%s fail=%s\nPOST: PASSED\n' "$2" "$3" "$4" > "$log"; fi +} +all() { for b in amd64 aarch64 riscv64 stick-high; do boot "$b" "$@"; done; } + +expect() { # expect "what" + local want=$1 what=$2 out rc + out=$(NIGHTLY_OUT="$T/out" POST_FLOOR_FILE="$T/floor" bash "$NIGHTLY" post-count 2>&1); rc=$? + if { [ "$want" = pass ] && [ $rc -eq 0 ]; } || + { [ "$want" = refuse ] && [ $rc -eq 1 ] && grep -q "POST" <<<"$out"; }; then + echo "PASS: $what" + else + echo "FAIL: $what (exit $rc)"; printf ' %s\n' "$out" | head -4; ok=0 + fi +} + +echo 538 > "$T/floor" +all 538 538 0; expect pass "four boots, 538 of 538, floor 538: pass" +all 600 600 0; expect pass "POST grew past the floor: pass" +boot riscv64 537 537 0; expect refuse "one boot ran fewer tests than the floor (537 < 538): refused" +all 538 538 0; boot aarch64 538 537 1; expect refuse "one boot with a failure: refused" +all 538 538 0; boot stick-high none; expect refuse "one boot with no PARITY:V4_POST line: refused" +all 538 538 0; rm -f "$T/out/accept-amd64.txt"; expect refuse "one boot not run: refused" +all 538 538 0; rm -f "$T/floor"; expect refuse "no floor in the tree: refused, not skipped" + +echo "RESULT: $([ $ok = 1 ] && echo PASS || echo FAIL)" +[ $ok = 1 ] diff --git a/v4/system/POST_FLOOR b/v4/system/POST_FLOOR new file mode 100644 index 00000000..86bb7161 --- /dev/null +++ b/v4/system/POST_FLOOR @@ -0,0 +1,3 @@ +538 +# v4 POST: the fewest tests an accepted boot may report (#173). Raise it when POST grows; +# never lower it without Captain Bob. scripts/nightly.sh post-count reads the first line.