From defd47e5ff354c7bfe641d6d96efa9329cb5796e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 11:25:28 +0000 Subject: [PATCH] scripts: ci_litter_check.sh -- #129's week-later check, run on the server Each cleanup #129 set up (Captain Bob, 2026-11-07), and what proves it ran: the Docker prune's and the package cleanup's logs written in the last 8 days; no image older than 8 days that no container uses; at most KEEP (5) lithosananke-runner tags; no artifact older than ARTIFACT_DAYS (30) + 1; no Nexus snapshot asset older than 8 days (with NEXUS_USER/NEXUS_PASSWORD, skipped and said so without); a dated disk line appended to HISTORY each run, so "disk use flat" is a comparison of weeks. PASS:/FAIL:/SKIP:/INFO:, RESULT:, exit 0 only if nothing fails. Tested here against fixtures (a stale log, an old artifact, no docker -- each FAIL; clean -- PASS) and a stand-in docker (one stale unused image of three -- FAIL, the in-use and the recent ones kept). The registry check ran against the real registry: 2 tags. Not run here: the real Docker, Nexus. Captain Bob, 2026-11-12: "Correct the issue text and write the check script". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017RzZgrf7V7Zk2iczBXDRbm --- scripts/ci_litter_check.sh | 136 +++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100755 scripts/ci_litter_check.sh diff --git a/scripts/ci_litter_check.sh b/scripts/ci_litter_check.sh new file mode 100755 index 00000000..706400a8 --- /dev/null +++ b/scripts/ci_litter_check.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: LicenseRef-Starship-1.0 +# Copyright (c) 2023-2026 Robert A. James +# ci_litter_check.sh -- #129's check that CI cleans up its litter, green, orange and red runs alike. +# Run ON THE SERVER, after a Sunday's cleanup, as a user who can run docker and read /var/log: +# +# sudo scripts/ci_litter_check.sh +# +# Each cleanup #129 set up on 2026-11-07 (Captain Bob's comment there), and what proves it ran: +# Docker prune, Sundays 03:00 /var/log/docker-prune.log written in the last 8 days; no image +# older than 8 days that no container uses +# Gitea package cleanup, Sun 03:15 /var/log/gitea-cleanup.log written in the last 8 days; at most +# KEEP lithosananke-runner tags in the registry +# Gitea artifacts, 30 days no file under GITEA_ARTIFACTS older than ARTIFACT_DAYS + 1 +# Nexus snapshots, 7 days no starshipos-snapshots asset older than 8 days (with +# NEXUS_USER and NEXUS_PASSWORD; skipped, and said so, without) +# Disk use flat a dated line appended to HISTORY each run; the last weeks shown +# +# Settings (environment): GITEA_ARTIFACTS (default: the first of /var/lib/gitea/data/actions_artifacts, +# /data/gitea/actions_artifacts that exists), ARTIFACT_DAYS (30), KEEP (5), GITEA_URL +# (https://gitea.starshipos-f18.org), GITEA_TOKEN (optional), NEXUS_URL +# (https://nexus.starshipos-f18.org), NEXUS_USER, NEXUS_PASSWORD, HISTORY (/var/log/ci-litter-check.log), +# PRUNE_LOG (/var/log/docker-prune.log), CLEANUP_LOG (/var/log/gitea-cleanup.log). +# +# Prints PASS:/FAIL:/SKIP:/INFO: per check and RESULT:; exits 0 only if no check fails. +set -uo pipefail +ARTIFACT_DAYS=${ARTIFACT_DAYS:-30} +KEEP=${KEEP:-5} +GITEA_URL=${GITEA_URL:-https://gitea.starshipos-f18.org} +NEXUS_URL=${NEXUS_URL:-https://nexus.starshipos-f18.org} +HISTORY=${HISTORY:-/var/log/ci-litter-check.log} +PRUNE_LOG=${PRUNE_LOG:-/var/log/docker-prune.log} +CLEANUP_LOG=${CLEANUP_LOG:-/var/log/gitea-cleanup.log} +if [ -z "${GITEA_ARTIFACTS:-}" ]; then + for d in /var/lib/gitea/data/actions_artifacts /data/gitea/actions_artifacts; do + [ -d "$d" ] && { GITEA_ARTIFACTS=$d; break; } + done +fi +ok=1 +pass() { echo "PASS: $*"; } +fail() { echo "FAIL: $*"; ok=0; } +skip() { echo "SKIP: $*"; } +info() { echo "INFO: $*"; } +now=$(date +%s) + +recent_log() { # recent_log : written in the last 8 days + local f=$1 what=$2 + if [ ! -f "$f" ]; then fail "$what: no $f"; return; fi + local age=$(( (now - $(stat -c %Y "$f")) / 86400 )) + if [ "$age" -le 8 ]; then pass "$what: $f written $age day(s) ago -- $(tail -1 "$f" | cut -c1-100)" + else fail "$what: $f last written $age days ago"; fi +} + +# 1. Docker +recent_log "$PRUNE_LOG" "the weekly Docker prune" +if command -v docker >/dev/null && docker info >/dev/null 2>&1; then + used=$(docker ps -a --format '{{.Image}}' | xargs -r docker image inspect -f '{{.Id}}' 2>/dev/null | sort -u) + stale=0 + while read -r id created; do + [ -n "$id" ] || continue + c=$(date -d "$created" +%s 2>/dev/null || echo "$now") + if [ $(( (now - c) / 86400 )) -gt 8 ] && ! grep -qx "$id" <<<"$used"; then stale=$((stale + 1)); fi + done < <(docker image ls -q --no-trunc | sort -u | xargs -r docker image inspect -f '{{.Id}} {{.Created}}') + if [ "$stale" = 0 ]; then pass "Docker: no image older than 8 days that no container uses" + else fail "Docker: $stale image(s) older than 8 days that no container uses -- the prune did not take them"; fi + info "docker system df: $(docker system df --format '{{.Type}} {{.Size}} ({{.Reclaimable}} reclaimable)' | paste -sd ';' -)" + dockerdf=$(docker system df --format '{{.Type}}={{.Size}}' | paste -sd ' ' -) +else + fail "Docker: docker is not usable here (run this on the server, as a user who may run docker)" + dockerdf=none +fi + +# 2. Gitea's package registry +recent_log "$CLEANUP_LOG" "the weekly Gitea package cleanup" +pkgs=$(curl -fsS ${GITEA_TOKEN:+-H "Authorization: token $GITEA_TOKEN"} \ + "$GITEA_URL/api/v1/packages/rajames?type=container&q=lithosananke-runner&limit=50" 2>/dev/null) +if [ -n "$pkgs" ]; then + tags=$(python3 -c 'import json,sys; print(sum(1 for p in json.load(sys.stdin) if not p["version"].startswith("sha256:")))' <<<"$pkgs") + if [ "$tags" -le "$KEEP" ]; then pass "registry: $tags lithosananke-runner tag(s), at most $KEEP" + else fail "registry: $tags lithosananke-runner tags, more than $KEEP"; fi +else + fail "registry: $GITEA_URL's package API did not answer (GITEA_TOKEN?)" +fi + +# 3. Gitea's run artifacts +if [ -n "${GITEA_ARTIFACTS:-}" ] && [ -d "$GITEA_ARTIFACTS" ]; then + old=$(find "$GITEA_ARTIFACTS" -type f -mtime +"$((ARTIFACT_DAYS + 1))" | wc -l) + if [ "$old" = 0 ]; then pass "artifacts: none older than $((ARTIFACT_DAYS + 1)) days in $GITEA_ARTIFACTS" + else fail "artifacts: $old file(s) older than $((ARTIFACT_DAYS + 1)) days in $GITEA_ARTIFACTS"; fi + info "artifacts: $(du -sh "$GITEA_ARTIFACTS" 2>/dev/null | cut -f1) in $(find "$GITEA_ARTIFACTS" -type f | wc -l) files" +else + fail "artifacts: no Gitea artifact directory found (set GITEA_ARTIFACTS)" +fi + +# 4. Nexus's snapshots +if [ -n "${NEXUS_USER:-}" ] && [ -n "${NEXUS_PASSWORD:-}" ]; then + oldest=$(NEXUS_URL=$NEXUS_URL python3 - <<'EOF' +import base64, json, os, urllib.request, datetime +url = os.environ["NEXUS_URL"] + "/service/rest/v1/assets?repository=starshipos-snapshots" +auth = base64.b64encode((os.environ["NEXUS_USER"] + ":" + os.environ["NEXUS_PASSWORD"]).encode()).decode() +token, dates = None, [] +while True: + req = urllib.request.Request(url + ("&continuationToken=" + token if token else ""), headers={"Authorization": "Basic " + auth}) + d = json.load(urllib.request.urlopen(req, timeout=60)) + dates += [a["lastModified"] for a in d.get("items", []) if a.get("lastModified")] + token = d.get("continuationToken") + if not token: + break +if not dates: + print("none") +else: + o = min(datetime.datetime.fromisoformat(x.replace("Z", "+00:00")) for x in dates) + print(len(dates), int((datetime.datetime.now(datetime.timezone.utc) - o).days)) +EOF +) + case $oldest in + none) pass "Nexus: starshipos-snapshots is empty" ;; + *\ *) n=${oldest% *}; days=${oldest#* } + if [ "$days" -le 8 ]; then pass "Nexus: $n snapshot asset(s), the oldest $days day(s) old" + else fail "Nexus: the oldest of $n snapshot assets is $days days old -- the 7-day cleanup did not take it"; fi ;; + *) fail "Nexus: the asset listing did not answer" ;; + esac +else + skip "Nexus: set NEXUS_USER and NEXUS_PASSWORD to check the 7-day snapshot cleanup" +fi + +# 5. Disk use, week on week +diskline="$(date -u +%Y-%m-%dT%H:%MZ) root=$(df -P / | awk 'NR==2 {print $3"K used of "$2"K"}') docker: $dockerdf" +if { echo "$diskline" >> "$HISTORY"; } 2>/dev/null; then + info "disk, the last weeks ($HISTORY):"; tail -4 "$HISTORY" | sed 's/^/ /' +else + fail "disk: could not append to $HISTORY"; info "disk: $diskline" +fi + +echo "RESULT: $([ $ok = 1 ] && echo PASS || echo FAIL)" +[ $ok = 1 ]