build: scripts/nightly.sh -- the nightly snapshot of the board images, for Nexus
Three stages, each a step of the nightly workflow:
boot <amd64|aarch64|riscv64> qemu_session.py headless, one QEMU at a time,
in the acceptance order; amd64 keeps the ISO it
booted as build/nightly/StarshipOS.iso
images make boot_image for RASPI (Pi 5) and MILKV
(Mars) -> raspi5.img.xz, MilkV.img.xz (for
Raspberry Pi Imager), and SHA256SUMS
publish <url> mvn deploy:deploy-file (maven-deploy-plugin
3.1.2): org.starship:kernelImage:
<LITHOS_VERSION>-SNAPSHOT, classifier pc (iso),
raspi5 and milkv (img.xz), in one deploy
Classifier by board, not ISA: two aarch64 boards would collide. No Pi 4:
there is no BCM2711 port. QEMU boots the aarch64 and riscv64 kernels, not
the Pi 5 and Mars images, which are for hardware QEMU does not model.
make init takes INIT_DOCS=0: without the TeX and Doxygen packages and
Isabelle, which an image build does not need.
Checked here: the three boot checks pass (logs in the next commit), the
images build, and publish to a local file:// repository gives the
timestamped snapshot files, the POM and maven-metadata.xml with one
snapshotVersion per classifier. Not checked: Nexus itself.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GE3SDvQn5EoxdGufk84zz2
This commit is contained in:
rajames
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
5b58b0b16e
commit
ca95567d14
@@ -43,7 +43,12 @@
|
||||
# three, for the full acceptance. syft and
|
||||
# Isabelle/HOL, not packaged, are downloaded,
|
||||
# checked by SHA-256 and linked into
|
||||
# /usr/local/bin (build/cache/tools)
|
||||
# /usr/local/bin (build/cache/tools).
|
||||
# INIT_DOCS=0: without the documents' and
|
||||
# proofs' tools (TeX, Doxygen, Isabelle)
|
||||
# scripts/nightly.sh boot <arch> | images | publish <url>
|
||||
# the nightly snapshot: boot-check, the
|
||||
# board images, and Nexus
|
||||
# make <other goal> goes to v3/Makefile (make help-hosted)
|
||||
# ==============================================================================
|
||||
|
||||
@@ -252,8 +257,12 @@ write_stick:
|
||||
# tools in build/cache/tools, which make clean keeps; each is linked into /usr/local/bin. syft's
|
||||
# sums are its release's checksums.txt; Isabelle publishes none, so its sums are of the archives
|
||||
# as downloaded on 2026-11-06 -- a download that differs is refused.
|
||||
INIT_COMMON := build-essential make git git-lfs python3 curl ca-certificates xorriso mtools gdisk dosfstools fdisk \
|
||||
pandoc latexmk texlive-xetex texlive-latex-base texlive-latex-recommended \
|
||||
# INIT_DOCS=0 leaves out what only the documents and the proofs need (the TeX and Doxygen
|
||||
# packages, and Isabelle): what a build of the images needs, as the nightly (scripts/nightly.sh).
|
||||
INIT_DOCS ?= 1
|
||||
INIT_COMMON := build-essential make git git-lfs python3 curl ca-certificates xz-utils \
|
||||
xorriso mtools gdisk dosfstools fdisk
|
||||
INIT_DOCPKGS := pandoc latexmk texlive-xetex texlive-latex-base texlive-latex-recommended \
|
||||
texlive-latex-extra texlive-pictures texlive-fonts-recommended lmodern \
|
||||
fonts-dejavu fonts-dejavu-extra doxygen graphviz
|
||||
INIT_amd64 := qemu-system-x86 ovmf
|
||||
@@ -301,20 +310,26 @@ init:
|
||||
echo "init: needs apt-get (Debian/Ubuntu). The packages are listed in the Makefile, at init."; exit 1; }
|
||||
$(SUDO) env DEBIAN_FRONTEND=noninteractive apt-get update -qq
|
||||
$(SUDO) env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
|
||||
$(INIT_COMMON) $(foreach a,$(INIT_ARCHES),$(INIT_$(a)))
|
||||
$(INIT_COMMON) $(if $(filter 0,$(INIT_DOCS)),,$(INIT_DOCPKGS)) $(foreach a,$(INIT_ARCHES),$(INIT_$(a)))
|
||||
git lfs install --local
|
||||
$(call fetch,$(SYFT_TGZ),$(SYFT_URL),$(SYFT_SHA_$(SYFT_ARCH)))
|
||||
@mkdir -p $(INIT_TOOLS)/syft-$(SYFT_VERSION)
|
||||
tar -xzf $(INIT_DL)/$(SYFT_TGZ) -C $(INIT_TOOLS)/syft-$(SYFT_VERSION) syft
|
||||
$(SUDO) ln -sf $(abspath $(INIT_TOOLS))/syft-$(SYFT_VERSION)/syft /usr/local/bin/syft
|
||||
@syft version | sed -n 's/^Version: */ syft /p'
|
||||
ifneq ($(INIT_DOCS),0)
|
||||
$(call fetch,$(ISABELLE_TGZ),$(ISABELLE_URL),$(ISABELLE_SHA_$(patsubst $(ISABELLE)_%.tar.gz,%,$(ISABELLE_TGZ))))
|
||||
@[ -x $(INIT_TOOLS)/$(ISABELLE)/bin/isabelle ] || { mkdir -p $(INIT_TOOLS) && tar -xzf $(INIT_DL)/$(ISABELLE_TGZ) -C $(INIT_TOOLS); }
|
||||
$(SUDO) ln -sf $(abspath $(INIT_TOOLS))/$(ISABELLE)/bin/isabelle /usr/local/bin/isabelle
|
||||
@syft version | sed -n 's/^Version: */ syft /p'
|
||||
@echo " isabelle $$(isabelle version)"
|
||||
@echo ""
|
||||
@echo "Ready for $(INIT_ARCHES): make ARCH=<arch> clean qemu, make -C v4 test, make docs,"
|
||||
@echo " make sbom, isabelle build -D proof/"
|
||||
else
|
||||
@echo ""
|
||||
@echo "Ready for $(INIT_ARCHES), without the documents' and proofs' tools: make ARCH=<arch> clean qemu,"
|
||||
@echo " make boot_image TARGET=<board>, make sbom"
|
||||
endif
|
||||
|
||||
help:
|
||||
@sed -n '/^# TARGET is/,/^# =====/p' Makefile | sed '$$d; s/^# \{0,1\}//'
|
||||
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/bin/bash
|
||||
# nightly.sh -- the nightly snapshot: boot-check each ISA, build the board images, publish them.
|
||||
# Three stages, run in this order (.gitea/workflows/nightly.yml runs each as a step):
|
||||
#
|
||||
# scripts/nightly.sh boot <amd64|aarch64|riscv64>
|
||||
# Boots that ISA headless in QEMU (scripts/qemu_session.py headless: POST: PASSED, and no
|
||||
# prompt with no Zuse drive), in the foreground; one QEMU at a time, the acceptance order
|
||||
# amd64, aarch64, riscv64. A failure ends the nightly. amd64 also keeps the ISO it booted,
|
||||
# byte for byte, as build/nightly/StarshipOS.iso.
|
||||
# scripts/nightly.sh images
|
||||
# The board images, into build/nightly/:
|
||||
# raspi5.img.xz Raspberry Pi 5 (make boot_image TARGET=RASPI), for Raspberry Pi Imager
|
||||
# MilkV.img.xz Milk-V Mars (make boot_image TARGET=MILKV)
|
||||
# SHA256SUMS of StarshipOS.iso and both images, before and after compression
|
||||
# (Imager's extract_sha256 is the uncompressed one)
|
||||
# QEMU boots the aarch64 and riscv64 kernels, not these images: they are built from the
|
||||
# same sources, for hardware QEMU does not model.
|
||||
# scripts/nightly.sh publish <URL>
|
||||
# Deploys the three to a Maven repository (Nexus's starshipos-snapshots) as
|
||||
# org.starship:kernelImage:<LITHOS_VERSION>-SNAPSHOT, one classifier per board --
|
||||
# pc (iso), raspi5 (img.xz), milkv (img.xz) -- in one deploy, so they share a timestamp.
|
||||
# NEXUS_USER (default ci-publisher) and NEXUS_PASSWORD. Needs mvn and a JRE.
|
||||
#
|
||||
# NIGHTLY_VERSION overrides the version. The serial logs are logs/<TS>/<arch>/ and the DoE CSVs
|
||||
# experiments/bare_metal/runs/, as for any boot (.claude/CLAUDE.md). Needs what
|
||||
# make ARCH=amd64,aarch64,riscv64 init INIT_DOCS=0 installs.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
OUT=build/nightly
|
||||
LITHOS=$(sed -n 's/^LITHOS_VERSION *?= *//p' kernel/Makefile)
|
||||
VERSION=${NIGHTLY_VERSION:-$LITHOS-SNAPSHOT}
|
||||
usage() { echo "Usage: scripts/nightly.sh boot <amd64|aarch64|riscv64> | images | publish <URL>" >&2; exit 2; }
|
||||
|
||||
case ${1:-} in
|
||||
boot)
|
||||
arch=${2:-}
|
||||
case $arch in amd64|aarch64|riscv64) ;; *) usage ;; esac
|
||||
[ "$arch" != amd64 ] || { rm -rf "$OUT"; mkdir -p "$OUT"; } # amd64 is the first stage
|
||||
echo "=== nightly: org.starship:kernelImage:$VERSION from $(git rev-parse --short HEAD); boot check $arch"
|
||||
git checkout -q -- disk/zuse-scratch # the committed Zuse pair, as before each acceptance boot
|
||||
python3 scripts/qemu_session.py headless "$arch" --label "nightly-$arch" --make "QEMU_DISPLAY=none"
|
||||
[ "$arch" != amd64 ] || cp build/amd64/kernel/starkernel.iso "$OUT/StarshipOS.iso"
|
||||
;;
|
||||
images)
|
||||
[ -f "$OUT/StarshipOS.iso" ] || { echo "nightly: no $OUT/StarshipOS.iso -- run boot amd64 first" >&2; exit 1; }
|
||||
for b in "RASPI raspi/lithos-raspi.img raspi5.img" "MILKV milkv/lithos-milkv.img MilkV.img"; do
|
||||
set -- $b
|
||||
echo "=== nightly: image $1"
|
||||
make --no-print-directory boot_image TARGET="$1"
|
||||
cp "build/boards/$2" "$OUT/$3"
|
||||
done
|
||||
# boot_image regenerates these (the source SBOM, and what qemu_session.py also puts back);
|
||||
# the nightly does not change the tree
|
||||
git checkout -q -- sbom.spdx sbom.spdx.json capsules/BLOCK_MAP.md v3/include/version.h
|
||||
(cd "$OUT" && rm -f raspi5.img.xz MilkV.img.xz && sha256sum StarshipOS.iso raspi5.img MilkV.img > SHA256SUMS)
|
||||
xz -T0 -6 "$OUT/raspi5.img" "$OUT/MilkV.img"
|
||||
(cd "$OUT" && sha256sum raspi5.img.xz MilkV.img.xz >> SHA256SUMS && cat SHA256SUMS)
|
||||
;;
|
||||
publish)
|
||||
url=${2:-}
|
||||
[ -n "$url" ] || usage
|
||||
: "${NEXUS_PASSWORD:?publish needs NEXUS_PASSWORD}"
|
||||
command -v mvn >/dev/null || { echo "nightly: publish needs mvn (apt-get install maven)" >&2; exit 1; }
|
||||
for f in StarshipOS.iso raspi5.img.xz MilkV.img.xz; do
|
||||
[ -f "$OUT/$f" ] || { echo "nightly: no $OUT/$f -- run boot and images first" >&2; exit 1; }
|
||||
done
|
||||
settings=$(mktemp)
|
||||
trap 'rm -f "$settings"' EXIT
|
||||
cat > "$settings" <<'EOF'
|
||||
<settings><servers><server>
|
||||
<id>starship-nexus</id>
|
||||
<username>${env.NEXUS_USER}</username>
|
||||
<password>${env.NEXUS_PASSWORD}</password>
|
||||
</server></servers></settings>
|
||||
EOF
|
||||
export NEXUS_USER=${NEXUS_USER:-ci-publisher}
|
||||
echo "=== nightly: publish org.starship:kernelImage:$VERSION -> $url"
|
||||
mvn -B -q -s "$settings" org.apache.maven.plugins:maven-deploy-plugin:3.1.2:deploy-file \
|
||||
-Durl="$url" -DrepositoryId=starship-nexus \
|
||||
-DgroupId=org.starship -DartifactId=kernelImage -Dversion="$VERSION" \
|
||||
-Dfile="$OUT/StarshipOS.iso" -Dpackaging=iso -Dclassifier=pc \
|
||||
-Dfiles="$OUT/raspi5.img.xz,$OUT/MilkV.img.xz" -Dclassifiers=raspi5,milkv -Dtypes=img.xz,img.xz \
|
||||
-DgeneratePom=true
|
||||
echo "=== nightly: published"
|
||||
;;
|
||||
*) usage ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user