LithosAnanke/scripts/ci_litter_check.sh

#!/usr/bin/env bash
# SPDX-License-Identifier: LicenseRef-Starship-1.0
# Copyright (c) 2023-2026 Robert A. James
# ci_litter_check.sh -- #129's check that CI cleans up its litter, green, orange and red runs alike.
# Run ON THE SERVER, after a Sunday's cleanup, as a user who can run docker and read /var/log:
#
#   sudo scripts/ci_litter_check.sh
#
# Each cleanup #129 set up on 2026-11-07 (Captain Bob's comment there), and what proves it ran:
#   Docker prune, Sundays 03:00      /var/log/docker-prune.log written in the last 8 days; no image
#                                    older than 8 days that no container uses
#   Gitea package cleanup, Sun 03:15 /var/log/gitea-cleanup.log written in the last 8 days; at most
#                                    KEEP lithosananke-runner tags in the registry
#   Gitea artifacts, 30 days         no file under GITEA_ARTIFACTS older than ARTIFACT_DAYS + 1
#   Nexus snapshots, 7 days          no starshipos-snapshots asset older than 8 days (with
#                                    NEXUS_USER and NEXUS_PASSWORD; skipped, and said so, without)
#   Disk use flat                    a dated line appended to HISTORY each run; the last weeks shown
#
# Settings (environment): GITEA_ARTIFACTS (default: the first of /var/lib/gitea/data/actions_artifacts,
# /data/gitea/actions_artifacts that exists), ARTIFACT_DAYS (30), KEEP (5), GITEA_URL
# (https://gitea.starshipos-f18.org), GITEA_TOKEN (optional), NEXUS_URL
# (https://nexus.starshipos-f18.org), NEXUS_USER, NEXUS_PASSWORD, HISTORY (/var/log/ci-litter-check.log),
# PRUNE_LOG (/var/log/docker-prune.log), CLEANUP_LOG (/var/log/gitea-cleanup.log).
#
# Prints PASS:/FAIL:/SKIP:/INFO: per check and RESULT:; exits 0 only if no check fails.
set -uo pipefail
ARTIFACT_DAYS=${ARTIFACT_DAYS:-30}
KEEP=${KEEP:-5}
GITEA_URL=${GITEA_URL:-https://gitea.starshipos-f18.org}
NEXUS_URL=${NEXUS_URL:-https://nexus.starshipos-f18.org}
HISTORY=${HISTORY:-/var/log/ci-litter-check.log}
PRUNE_LOG=${PRUNE_LOG:-/var/log/docker-prune.log}
CLEANUP_LOG=${CLEANUP_LOG:-/var/log/gitea-cleanup.log}
if [ -z "${GITEA_ARTIFACTS:-}" ]; then
    for d in /var/lib/gitea/data/actions_artifacts /data/gitea/actions_artifacts; do
        [ -d "$d" ] && { GITEA_ARTIFACTS=$d; break; }
    done
fi
ok=1
pass() { echo "PASS: $*"; }
fail() { echo "FAIL: $*"; ok=0; }
skip() { echo "SKIP: $*"; }
info() { echo "INFO: $*"; }
now=$(date +%s)

recent_log() {  # recent_log <file> <what>: written in the last 8 days
    local f=$1 what=$2
    if [ ! -f "$f" ]; then fail "$what: no $f"; return; fi
    local age=$(( (now - $(stat -c %Y "$f")) / 86400 ))
    if [ "$age" -le 8 ]; then pass "$what: $f written $age day(s) ago -- $(tail -1 "$f" | cut -c1-100)"
    else fail "$what: $f last written $age days ago"; fi
}

# 1. Docker
recent_log "$PRUNE_LOG" "the weekly Docker prune"
if command -v docker >/dev/null && docker info >/dev/null 2>&1; then
    used=$(docker ps -a --format '{{.Image}}' | xargs -r docker image inspect -f '{{.Id}}' 2>/dev/null | sort -u)
    stale=0
    while read -r id created; do
        [ -n "$id" ] || continue
        c=$(date -d "$created" +%s 2>/dev/null || echo "$now")
        if [ $(( (now - c) / 86400 )) -gt 8 ] && ! grep -qx "$id" <<<"$used"; then stale=$((stale + 1)); fi
    done < <(docker image ls -q --no-trunc | sort -u | xargs -r docker image inspect -f '{{.Id}} {{.Created}}')
    if [ "$stale" = 0 ]; then pass "Docker: no image older than 8 days that no container uses"
    else fail "Docker: $stale image(s) older than 8 days that no container uses -- the prune did not take them"; fi
    info "docker system df: $(docker system df --format '{{.Type}} {{.Size}} ({{.Reclaimable}} reclaimable)' | paste -sd ';' -)"
    dockerdf=$(docker system df --format '{{.Type}}={{.Size}}' | paste -sd ' ' -)
else
    fail "Docker: docker is not usable here (run this on the server, as a user who may run docker)"
    dockerdf=none
fi

# 2. Gitea's package registry
recent_log "$CLEANUP_LOG" "the weekly Gitea package cleanup"
pkgs=$(curl -fsS ${GITEA_TOKEN:+-H "Authorization: token $GITEA_TOKEN"} \
       "$GITEA_URL/api/v1/packages/rajames?type=container&q=lithosananke-runner&limit=50" 2>/dev/null)
if [ -n "$pkgs" ]; then
    tags=$(python3 -c 'import json,sys; print(sum(1 for p in json.load(sys.stdin) if not p["version"].startswith("sha256:")))' <<<"$pkgs")
    if [ "$tags" -le "$KEEP" ]; then pass "registry: $tags lithosananke-runner tag(s), at most $KEEP"
    else fail "registry: $tags lithosananke-runner tags, more than $KEEP"; fi
else
    fail "registry: $GITEA_URL's package API did not answer (GITEA_TOKEN?)"
fi

# 3. Gitea's run artifacts
if [ -n "${GITEA_ARTIFACTS:-}" ] && [ -d "$GITEA_ARTIFACTS" ]; then
    old=$(find "$GITEA_ARTIFACTS" -type f -mtime +"$((ARTIFACT_DAYS + 1))" | wc -l)
    if [ "$old" = 0 ]; then pass "artifacts: none older than $((ARTIFACT_DAYS + 1)) days in $GITEA_ARTIFACTS"
    else fail "artifacts: $old file(s) older than $((ARTIFACT_DAYS + 1)) days in $GITEA_ARTIFACTS"; fi
    info "artifacts: $(du -sh "$GITEA_ARTIFACTS" 2>/dev/null | cut -f1) in $(find "$GITEA_ARTIFACTS" -type f | wc -l) files"
else
    fail "artifacts: no Gitea artifact directory found (set GITEA_ARTIFACTS)"
fi

# 4. Nexus's snapshots
if [ -n "${NEXUS_USER:-}" ] && [ -n "${NEXUS_PASSWORD:-}" ]; then
    oldest=$(NEXUS_URL=$NEXUS_URL python3 - <<'EOF'
import base64, json, os, urllib.request, datetime
url = os.environ["NEXUS_URL"] + "/service/rest/v1/assets?repository=starshipos-snapshots"
auth = base64.b64encode((os.environ["NEXUS_USER"] + ":" + os.environ["NEXUS_PASSWORD"]).encode()).decode()
token, dates = None, []
while True:
    req = urllib.request.Request(url + ("&continuationToken=" + token if token else ""), headers={"Authorization": "Basic " + auth})
    d = json.load(urllib.request.urlopen(req, timeout=60))
    dates += [a["lastModified"] for a in d.get("items", []) if a.get("lastModified")]
    token = d.get("continuationToken")
    if not token:
        break
if not dates:
    print("none")
else:
    o = min(datetime.datetime.fromisoformat(x.replace("Z", "+00:00")) for x in dates)
    print(len(dates), int((datetime.datetime.now(datetime.timezone.utc) - o).days))
EOF
)
    case $oldest in
        none) pass "Nexus: starshipos-snapshots is empty" ;;
        *\ *) n=${oldest% *}; days=${oldest#* }
              if [ "$days" -le 8 ]; then pass "Nexus: $n snapshot asset(s), the oldest $days day(s) old"
              else fail "Nexus: the oldest of $n snapshot assets is $days days old -- the 7-day cleanup did not take it"; fi ;;
        *)    fail "Nexus: the asset listing did not answer" ;;
    esac
else
    skip "Nexus: set NEXUS_USER and NEXUS_PASSWORD to check the 7-day snapshot cleanup"
fi

# 5. Disk use, week on week
diskline="$(date -u +%Y-%m-%dT%H:%MZ) root=$(df -P / | awk 'NR==2 {print $3"K used of "$2"K"}') docker: $dockerdf"
if { echo "$diskline" >> "$HISTORY"; } 2>/dev/null; then
    info "disk, the last weeks ($HISTORY):"; tail -4 "$HISTORY" | sed 's/^/        /'
else
    fail "disk: could not append to $HISTORY"; info "disk: $diskline"
fi

echo "RESULT: $([ $ok = 1 ] && echo PASS || echo FAIL)"
[ $ok = 1 ]