Reorganize source tree: kernel/, v3/, v4/ split and board infrastructure

Source tree reorganization: - Move StarForth v3 engine to v3/ (src/, include/, Makefile) - Move kernel to kernel/ (src/, include/, linker/, Makefile) - Create v4/ skeleton for F18-ISA golden model (DECOMPOSITION.md, JUSTIFICATION.md) - Move FABRIC-0..4.md to docs/fabric/ - Move ONTOLOGY.md and ROADMAP.md to docs/ Board infrastructure: - Add boards/ser5/, boards/raspi/, boards/milkv/, boards/zynq7020/ - Each board has board.mk (ISA, CPU flags, boot recipe) and README.md - Root Makefile becomes thin dispatcher: boot_image, all, clean, docs take TARGET - make boot_image TARGET=SER5|RASPI|MILKV builds one GPT/MBR image per board - ZYNQ7020 target exists but stops with clear error (ARMv7 port not built yet) - scripts/mkdiskimage.sh builds disk images for all boards Docs pipeline: - docs/book/ with LaTeX master (main.tex) and Makefile - pandoc converts Markdown to LaTeX at build time - Two Lua filters: table-widths.lua (wide tables wrap), code-breaks.lua (inline code breaks) - make docs builds single PDF (754 pages, 0 missing characters) - make docs TARGET=<board> adds board appendix - build/docs/<book|board>/meta.tex stamps git commit into PDF Bug fixes: - 42 include paths that only worked by accident now use correct relative paths - clang-18 hardcode replaced with configurable CC variable (fixed aarch64 build) - Pi 5: kernel_2712.img linked at 0x80000, .bss zeroed, memory reserved - Doxyfile, .clang-tidy, README.md, Kconfig paths updated Verified: - Hosted v3 build passes 1012 tests, 0 failures - SER5 image boots in QEMU (OVMF), POST passes, K exact (65536 = Q48_ONE) - Milk-V image boots in QEMU (OpenSBI + U-Boot + bootefi), POST passes - make clean TARGET=<board> removes only that board and its ISA objects - make all builds all boards, hosted v3, and docs in one run Co-authored-by: Junie <junie@jetbrains.com>
2026-10-01 15:40:09 -04:00
parent 3c709c115b
commit a8b70e88d3
630 changed files with 2646 additions and 2036 deletions
@@ -0,0 +1,112 @@
+# docs/book/Makefile -- the single LithosAnanke book, built from LaTeX.
+#
+# Run from the repo root (the root Makefile does this):
+#   make docs                   -> build/docs/LithosAnanke.pdf
+#   make docs TARGET=<board>    -> build/docs/LithosAnanke-<board>.pdf
+#                                  (same book + that board's appendix)
+#
+# docs/book/main.tex is the master. Markdown sources are converted by pandoc
+# into LaTeX fragments under build/docs/gen/ and \input from main.tex; the
+# Markdown stays the source of truth until a chapter is rewritten in LaTeX.
+# Figures and tables that present data read the CSV directly at LaTeX time
+# (pgfplots / pgfplotstable) and cite it with \datasource{path}; see
+# docs/book/README.md.
+
+BOOK_DIR := docs/book
+OUT      := build/docs
+GEN      := $(OUT)/gen
+
+BOARD      ?=
+BOARD_NAME ?=
+VARIANT    := $(if $(BOARD),$(BOARD),book)
+VAR_DIR    := $(OUT)/$(VARIANT)
+PDF        := $(OUT)/LithosAnanke$(if $(BOARD),-$(BOARD)).pdf
+
+PANDOC  ?= pandoc
+LATEXMK ?= latexmk
+PANDOC_FILTERS := $(BOOK_DIR)/pandoc/table-widths.lua $(BOOK_DIR)/pandoc/code-breaks.lua
+PANDOC_FLAGS := -f gfm -t latex --top-level-division=chapter --wrap=preserve \
+    $(foreach f,$(PANDOC_FILTERS),--lua-filter=$(f))
+
+# fragment name : Markdown source. The order of \input lines is main.tex's.
+CHAPTERS := \
+    ontology:docs/ONTOLOGY.md \
+    roadmap:docs/ROADMAP.md \
+    boards:boards/README.md \
+    justification:docs/v4.0.0/JUSTIFICATION.md \
+    decomposition:docs/v4.0.0/DECOMPOSITION.md \
+    fabric-0:docs/fabric/FABRIC-0.md \
+    fabric-1:docs/fabric/FABRIC-1.md \
+    fabric-2:docs/fabric/FABRIC-2.md \
+    fabric-3:docs/fabric/FABRIC-3.md \
+    fabric-3-5:docs/fabric/FABRIC-3.5.md \
+    fabric-3-6:docs/fabric/FABRIC-3.6.md \
+    fabric-3-7:docs/fabric/FABRIC-3.7.md \
+    fabric-4:docs/fabric/FABRIC-4.md
+
+ifneq ($(BOARD),)
+ifeq ($(wildcard boards/$(BOARD)/README.md),)
+    $(error boards/$(BOARD)/README.md is missing: it is the board's appendix)
+endif
+CHAPTERS += board-$(BOARD):boards/$(BOARD)/README.md
+endif
+
+chapter_name = $(word 1,$(subst :, ,$(1)))
+chapter_src  = $(word 2,$(subst :, ,$(1)))
+FRAGMENTS := $(foreach c,$(CHAPTERS),$(GEN)/$(call chapter_name,$(c)).tex)
+
+.PHONY: all tools FORCE
+all: $(PDF)
+
+tools:
+	@missing=""; \
+	for t in $(PANDOC) $(LATEXMK) xelatex; do command -v $$t >/dev/null 2>&1 || missing="$$missing $$t"; done; \
+	if [ -n "$$missing" ]; then \
+	    echo "Error: make docs needs:$$missing"; \
+	    echo "  sudo apt-get install -y pandoc latexmk texlive-xetex texlive-latex-extra texlive-pictures texlive-fonts-recommended fonts-dejavu fonts-dejavu-extra"; \
+	    echo "  (or a user-local TinyTeX + pandoc in ~/.local; see docs/book/README.md)"; \
+	    exit 1; \
+	fi
+
+# --id-prefix keeps heading labels unique across documents that reuse
+# section names ("Summary", "Open questions", ...).
+define chapter_rule
+$(GEN)/$(call chapter_name,$(1)).tex: $(call chapter_src,$(1)) $(PANDOC_FILTERS) | tools
+	@mkdir -p $(GEN)
+	@echo "  PANDOC  $$< -> $$@"
+	@$(PANDOC) $(PANDOC_FLAGS) --id-prefix=$(call chapter_name,$(1))- $$< -o $$@
+endef
+$(foreach c,$(CHAPTERS),$(eval $(call chapter_rule,$(c))))
+
+# Pandoc's syntax-highlighting macros, taken from the installed pandoc so the
+# fragments and their macros always come from the same version.
+$(GEN)/pandoc-highlighting.tex: $(BOOK_DIR)/pandoc/highlighting.latex | tools
+	@mkdir -p $(GEN)
+	@printf '```c\nx\n```\n' | $(PANDOC) -f gfm -t latex -s --template=$< -o $@
+
+# Per-build facts the book prints: the commit every \datasource refers to,
+# and which board appendix (if any) is included.
+$(VAR_DIR)/meta.tex: FORCE
+	@mkdir -p $(VAR_DIR)
+	@{ \
+	    c=$$(git rev-parse --short=12 HEAD 2>/dev/null || echo unknown); \
+	    git diff --quiet HEAD -- 2>/dev/null || c="$$c (+ uncommitted changes)"; \
+	    printf '\\newcommand{\\bookcommit}{%s}\n' "$$c"; \
+	    printf '\\newcommand{\\bookdate}{%s}\n' "$$(date -u +%Y-%m-%d)"; \
+	    printf '\\def\\bookboard{%s}\n' "$(BOARD_NAME)"; \
+	    $(if $(BOARD),printf '\\newcommand{\\bookboardappendix}{board-%s}\n' "$(BOARD)";) \
+	} > $@.tmp
+	@cmp -s $@.tmp $@ && rm -f $@.tmp || mv $@.tmp $@
+
+FORCE:
+
+TEXINPUTS_BOOK := $(abspath $(VAR_DIR)):$(abspath $(GEN)):$(abspath $(BOOK_DIR)):
+
+$(PDF): $(BOOK_DIR)/main.tex $(FRAGMENTS) $(GEN)/pandoc-highlighting.tex $(VAR_DIR)/meta.tex | tools
+	@echo "  LATEXMK $(BOOK_DIR)/main.tex -> $@"
+	@TEXINPUTS=$(TEXINPUTS_BOOK) $(LATEXMK) -xelatex -interaction=nonstopmode -halt-on-error \
+	    -file-line-error -outdir=$(VAR_DIR) $(BOOK_DIR)/main.tex > $(VAR_DIR)/latexmk.log 2>&1 || { \
+	    grep -A4 -E '^(.*:[0-9]+:|!)' $(VAR_DIR)/main.log | head -40; \
+	    echo "Error: LaTeX failed; full log: $(VAR_DIR)/main.log"; exit 1; }
+	@cp $(VAR_DIR)/main.pdf $@
+	@echo "  PDF     $@"
@@ -0,0 +1,66 @@
+# docs/book/
+
+`make docs` builds one PDF, `build/docs/LithosAnanke.pdf`, from `main.tex` with xelatex.
+`make docs TARGET=<board>` builds `build/docs/LithosAnanke-<board>.pdf`: the same book with
+`boards/<board>/README.md` added as an appendix.
+
+Tools: `sudo apt-get install -y pandoc latexmk texlive-xetex texlive-latex-extra texlive-pictures texlive-fonts-recommended fonts-dejavu fonts-dejavu-extra`
+
+Without root, a user-local TeX Live works too (this is how the book was first built):
+TinyTeX (`curl -sL https://yihui.org/tinytex/install-bin-unix.sh | sh`) plus
+`tlmgr install latexmk xetex fontspec pgf pgfplots booktabs multirow fancyvrb fvextra lineno upquote ulem enumitem newunicodechar bookmark hyperref geometry xcolor graphics tools etoolbox fancyhdr truncate amsfonts`,
+and the pandoc release tarball unpacked into `~/.local`.
+
+The installed DejaVu decides italics: `fonts-dejavu-core` alone has no serif italic, so
+`main.tex` falls back to a slanted upright face; `fonts-dejavu-extra` gives real italics.
+
+## How it fits together
+
+- `main.tex` is the master file. It sets the parts and the chapter order.
+- Chapters that are still Markdown are listed in `Makefile` (`CHAPTERS`, as `name:source.md`).
+  At build time pandoc converts each one into `build/docs/gen/<name>.tex`, and `main.tex`
+  includes it with `\input{<name>}`. The Markdown stays the source until the chapter is
+  rewritten in LaTeX. At that point the `.tex` moves into this directory and the `CHAPTERS`
+  entry is removed.
+- `pandoc/highlighting.latex` extracts the code-highlighting macros from the installed pandoc,
+  so the macros and the fragments always come from the same pandoc version.
+- Two Lua filters run on every chapter. `pandoc/table-widths.lua` gives wide tables
+  proportional wrapping columns, because pandoc's gfm reader leaves column widths unset.
+  `pandoc/code-breaks.lua` lets long inline identifiers and paths break after `_ / . - :`.
+  Code blocks wrap through fvextra (`breaklines`).
+- `build/docs/<book|board>/meta.tex` is written on every build. It records the commit (and
+  whether there were uncommitted changes), the date, and the board.
+
+## Rule for data
+
+Any figure or table that shows measured numbers is generated from the CSV when the book is
+built. Numbers are never typed in by hand:
+
+```latex
+\begin{tikzpicture}
+  \begin{axis}[xlabel=run, ylabel=K]
+    \addplot table[col sep=comma, x=run, y=K]{experiments/<campaign>/results.csv};
+  \end{axis}
+\end{tikzpicture}
+\datasource{experiments/<campaign>/results.csv}
+```
+
+Use `\pgfplotstabletypeset[col sep=comma]{...}` for tables. `\datasource` prints the CSV path
+and the commit, so a reader can find the exact file the figure was drawn from. CSV paths are
+repo-relative, because xelatex runs from the repo root.
+
+## Older pipelines to fold in or retire
+
+These still exist and still have their own targets. Each one needs a decision during the
+curation pass: move it into this book, or retire it.
+
+| Source | Current target | Produces |
+|---|---|---|
+| `docs/formal/vol1-vm-physics`, `vol2-kernel`, `vol3-research` | `make -C docs/formal vols` | three volume PDFs |
+| `docs/formal/dev-guide`, `user-guide`, `cookbook` | `make -C docs/formal books` | three practitioner PDFs |
+| `docs/formal/experiments`, `proofs`, `ssrn`, `patent` | `make -C docs/formal standalone` | four standalone PDFs |
+| Doxygen (`Doxyfile`) | `make -C docs/formal doxygen` | API reference PDF |
+| `scripts/generate-doxygen-appendix.sh` | `make -f v3/Makefile api-docs` | AsciiDoc API appendix |
+| `docs/src/internal/formal/*.thy` | `make -f v3/Makefile docs-isabelle` | Isabelle report |
+| `scripts/asciidoc-to-latex.sh` | `make -f v3/Makefile docs-latex` | `docs/latex/` |
+| `docs/SSRN_companion/Math_Companion_SSRN.tex` | `make -f v3/Makefile math-companion` | SSRN math companion |
@@ -0,0 +1,161 @@
+% docs/book/main.tex -- master file of the single LithosAnanke book.
+%
+% Built by `make docs [TARGET=<board>]` (docs/book/Makefile) with xelatex.
+% Fragments named below (ontology, fabric-0, ...) are generated by pandoc
+% from the Markdown listed in docs/book/Makefile's CHAPTERS into
+% build/docs/gen/; meta.tex is generated per build. Neither is committed.
+\documentclass[11pt,openany]{book}
+
+\usepackage{amsmath,amssymb}
+\usepackage{fontspec}
+% fonts-dejavu-core has no serif/sans italics (fonts-dejavu-extra does); when a
+% shape is missing, slant the upright one rather than silently drop emphasis.
+\setmainfont{DejaVu Serif}[AutoFakeSlant=0.2]
+\setsansfont{DejaVu Sans}[AutoFakeSlant=0.2]
+\setmonofont{DejaVu Sans Mono}[Scale=0.85]
+\newfontfamily\symbolfont{DejaVu Sans}
+
+\usepackage[letterpaper,margin=1in]{geometry}
+\usepackage{xcolor}
+\usepackage{graphicx}
+\usepackage{longtable,booktabs,array,calc,multirow}
+\usepackage{fancyvrb}
+\usepackage[normalem]{ulem}
+\usepackage{enumitem}
+\usepackage{newunicodechar}
+\usepackage{pgfplots}
+\usepackage{pgfplotstable}
+\pgfplotsset{compat=1.18}
+\usepackage[hidelinks]{hyperref}
+\usepackage{bookmark}
+
+% --- what pandoc's LaTeX fragments expect (normally from its own template)
+\providecommand{\tightlist}{\setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
+\providecommand{\pandocbounded}[1]{#1}
+\providecommand{\st}[1]{\sout{#1}}
+\newcounter{none} % pandoc: {\def\LTcaptype{none} ...} marks unnumbered tables
+\usepackage{etoolbox}
+\makeatletter
+\def\fnum@table{\tablename~\thetable}
+\patchcmd\longtable{\par}{\if@noskipsec\mbox{}\fi\par}{}{}
+\makeatother
+\usepackage{fvextra}
+\input{pandoc-highlighting}
+% Code and diagrams are wider than the page in places: wrap, never overflow.
+\RecustomVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\},breaklines,breakanywhere,fontsize=\small}
+\RecustomVerbatimEnvironment{verbatim}{Verbatim}{breaklines,breakanywhere,fontsize=\small}
+\setlistdepth{9}
+\renewlist{itemize}{itemize}{9}
+\setlist[itemize]{label=\textbullet}
+\renewlist{enumerate}{enumerate}{9}
+\setlist[enumerate]{label=\arabic*.}
+% Running heads: chapter on the left page, section on the right, both cut to
+% the page width (several design-record titles are a full sentence long).
+\usepackage{fancyhdr}
+\usepackage[fit]{truncate}
+\pagestyle{fancy}
+\fancyhf{}
+\renewcommand{\chaptermark}[1]{\markboth{#1}{}}
+\renewcommand{\sectionmark}[1]{\markright{#1}}
+\fancyhead[LE]{\small\truncate{\dimexpr\headwidth-3em}{\leftmark}}
+\fancyhead[RO]{\small\truncate{\dimexpr\headwidth-3em}{\rightmark}}
+\fancyhead[RE,LO]{\small\thepage}
+\renewcommand{\headrulewidth}{0.4pt}
+\setlength{\headheight}{14pt}
+\fancypagestyle{plain}{\fancyhf{}\fancyfoot[C]{\small\thepage}\renewcommand{\headrulewidth}{0pt}}
+\setlength{\emergencystretch}{3em}
+\setlength{\parindent}{0pt}
+\setlength{\parskip}{0.5em}
+
+% --- glyphs DejaVu Serif lacks: the status marks used across the docs
+\newunicodechar{✅}{{\symbolfont ✔}}
+\newunicodechar{✓}{{\symbolfont ✓}}
+\newunicodechar{❌}{{\symbolfont ✘}}
+\newunicodechar{✗}{{\symbolfont ✗}}
+\newunicodechar{⬜}{{\symbolfont ☐}}
+\newunicodechar{⭐}{{\symbolfont ★}}
+\newunicodechar{🔶}{{\symbolfont ◆}}
+\newunicodechar{⚠}{{\symbolfont ⚠}}
+\newunicodechar{📋}{}
+\newunicodechar{🐛}{[bug]}
+\newunicodechar{🎯}{[goal]}
+\newunicodechar{🟡}{{\symbolfont ●}}
+\newunicodechar{📍}{{\symbolfont ▸}}
+\newunicodechar{🔓}{[unlocked]}
+\newunicodechar{❓}{?}
+\newunicodechar{⟺}{\ensuremath{\Longleftrightarrow}}
+\newunicodechar{⋯}{\ensuremath{\cdots}}
+\newunicodechar{^^^^fe0f}{}
+
+% --- the documents number their own sections (§III.1, D-3, ...)
+\setcounter{secnumdepth}{0}
+\setcounter{tocdepth}{1}
+
+\input{meta}
+
+% \datasource{path/to/data.csv}: cite the data behind a figure or table.
+% Every figure or table of measured numbers reads its CSV at build time
+% (\addplot table / \pgfplotstabletypeset) and is followed by this line, so
+% the reader can find the exact file at the exact commit.
+\newcommand{\datasource}[1]{%
+  \par{\small Data: \texttt{\detokenize{#1}} at commit \texttt{\bookcommit}.}\par}
+
+\title{LithosAnanke and StarForth}
+\author{}
+\date{Built \bookdate{} from commit \texttt{\bookcommit}%
+  \ifx\bookboard\empty\else\\Board appendix: \bookboard\fi}
+
+\begin{document}
+\frontmatter
+\maketitle
+\tableofcontents
+
+\chapter{About this book}
+This book is generated from the repository by \texttt{make docs}. Each
+chapter below is still maintained as the Markdown file named in the table and
+converted at build time; a chapter moves into LaTeX source when it is
+rewritten. Part~III is the design record: those documents are archival and
+are reproduced as written, including their struck-through corrections.
+
+\begin{longtable}{@{}ll@{}}
+\toprule
+Chapter & Source \\
+\midrule
+\endhead
+Ontology & \texttt{docs/ONTOLOGY.md} \\
+Roadmap & \texttt{docs/ROADMAP.md} \\
+Boot targets & \texttt{boards/README.md} \\
+v4 justification & \texttt{docs/v4.0.0/JUSTIFICATION.md} \\
+v4 decomposition & \texttt{docs/v4.0.0/DECOMPOSITION.md} \\
+FABRIC-0 \dots{} FABRIC-4 & \texttt{docs/fabric/FABRIC-*.md} \\
+Board appendix (\texttt{TARGET=}) & \texttt{boards/<board>/README.md} \\
+\bottomrule
+\end{longtable}
+
+\mainmatter
+
+\part{The system}
+\input{ontology}
+\input{roadmap}
+\input{boards}
+
+\part{StarForth v4}
+\input{justification}
+\input{decomposition}
+
+\part{Design record (FABRIC series)}
+\input{fabric-0}
+\input{fabric-1}
+\input{fabric-2}
+\input{fabric-3}
+\input{fabric-3-5}
+\input{fabric-3-6}
+\input{fabric-3-7}
+\input{fabric-4}
+
+\ifdefined\bookboardappendix
+\appendix
+\input{\bookboardappendix}
+\fi
+
+\end{document}
@@ -0,0 +1,29 @@
+-- docs/book/pandoc/code-breaks.lua
+--
+-- Inline code in the design record is full of long identifiers and paths
+-- (g_wirebind_attached_username, kernel/src/arch/riscv64/...), which LaTeX
+-- cannot break, so they run into the margin. Emit inline code as \texttt
+-- ourselves, escaped the same way pandoc does, with a break opportunity
+-- after each separator character.
+
+local BREAK_AFTER = { ["_"] = true, ["/"] = true, ["."] = true, ["-"] = true,
+                      [":"] = true, [","] = true, ["("] = true, ["="] = true }
+
+local ESCAPE = {
+  ["\\"] = "\\textbackslash{}", ["{"] = "\\{", ["}"] = "\\}",
+  ["$"] = "\\$", ["&"] = "\\&", ["#"] = "\\#", ["%"] = "\\%", ["_"] = "\\_",
+  ["^"] = "\\textasciicircum{}", ["~"] = "\\textasciitilde{}",
+  ["'"] = "\\textquotesingle{}", ["`"] = "\\textasciigrave{}",
+  [" "] = "\\ ",
+}
+
+function Code(el)
+  if not FORMAT:match("latex") then return nil end
+  local out = {}
+  for _, cp in utf8.codes(el.text) do
+    local ch = utf8.char(cp)
+    out[#out + 1] = ESCAPE[ch] or ch
+    if BREAK_AFTER[ch] then out[#out + 1] = "\\allowbreak{}" end
+  end
+  return pandoc.RawInline("latex", "\\texttt{" .. table.concat(out) .. "}")
+end
@@ -0,0 +1,3 @@
+$if(highlighting-macros)$
+$highlighting-macros$
+$endif$
@@ -0,0 +1,55 @@
+-- docs/book/pandoc/table-widths.lua
+--
+-- pandoc's gfm reader leaves every column width unset, so the LaTeX writer
+-- emits `l` columns and a table with long cells runs off the page. For any
+-- table wider than WIDE characters, give each column a width proportional
+-- to its longest cell (clamped to [MIN_COL, MAX_COL] characters); the
+-- writer then emits wrapping p{} columns that together fill \linewidth.
+
+local WIDE    = 72
+local MIN_COL = 6
+local MAX_COL = 60
+
+local stringify = pandoc.utils.stringify
+
+local function measure(rows, widths)
+  for _, row in ipairs(rows) do
+    local col = 1
+    for _, cell in ipairs(row.cells) do
+      local span = cell.col_span or 1
+      local len = utf8.len(stringify(cell.contents)) or #stringify(cell.contents)
+      local per = len / span
+      for c = col, col + span - 1 do
+        if per > (widths[c] or 0) then widths[c] = per end
+      end
+      col = col + span
+    end
+  end
+end
+
+function Table(tbl)
+  local n = #tbl.colspecs
+  local widths = {}
+  for i = 1, n do widths[i] = 0 end
+
+  measure(tbl.head.rows, widths)
+  for _, body in ipairs(tbl.bodies) do
+    measure(body.head, widths)
+    measure(body.body, widths)
+  end
+  measure(tbl.foot.rows, widths)
+
+  local total = 0
+  for i = 1, n do total = total + widths[i] end
+  if total <= WIDE then return nil end
+
+  local sum = 0
+  for i = 1, n do
+    widths[i] = math.max(MIN_COL, math.min(MAX_COL, widths[i]))
+    sum = sum + widths[i]
+  end
+  for i = 1, n do
+    tbl.colspecs[i] = { tbl.colspecs[i][1], widths[i] / sum }
+  end
+  return tbl
+end