#!/usr/bin/env python3 # SPDX-License-Identifier: LicenseRef-Starship-1.0 # Copyright (c) 2023-2026 Robert A. James """mkdictionary.py -- the dictionary of v4 (#157): every word the system boots with, alphabetical, with its stack notation, a description, and where it is defined. v4/tools/mkdictionary.py words the live word list, one per line v4/tools/mkdictionary.py draft TSV a table for every live word not in TSV yet, appended, its stack and description taken from the source where the source has them (to be read and finished by hand) v4/tools/mkdictionary.py check TSV every live word has an entry with a stack and a description, and every entry is a live word: 0, or 1 with what is wrong v4/tools/mkdictionary.py md TSV OUT.md the dictionary as Markdown, alphabetical, for the book The live word list is the system's own: the hosted v4 (`make -C v4 hosted`) booted and asked `WORDS`, plus the kernel's words of the lone node (`sk_v4_words[]` in kernel/src/v4/sk_v4.c) -- the words a booted StarshipOS has. The table (docs/reference/v4-dictionary.tsv) holds the stack notation and the description, which only a person can write; the names are never typed by hand. """ import os, re, subprocess, sys ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) HOSTED = os.path.join(ROOT, "v4", "build", "starforth4-amd64") SOURCES = ([os.path.join("capsules/v4/sys", f) for f in sorted(os.listdir(os.path.join(ROOT, "capsules/v4/sys"))) if f.endswith(".f18")] + [os.path.join("capsules/v4", f) for f in sorted(os.listdir(os.path.join(ROOT, "capsules/v4"))) if f.endswith(".4th")]) KERNEL_WORDS_SRC = "kernel/src/v4/sk_v4.c" def live_words(): """The hosted system's WORDS, then the kernel's own words not among them.""" if not os.path.exists(HOSTED): sys.exit(f"mkdictionary: {HOSTED} is missing: make -C v4 hosted") out = subprocess.run([HOSTED], input="WORDS\n", capture_output=True, text=True, timeout=120).stdout lines = out.split("\n") try: start = next(k for k, l in enumerate(lines) if l.startswith("ok> ") and l[4:].strip()) except StopIteration: sys.exit("mkdictionary: the hosted system printed no WORDS") got = [] for l in lines[start:]: if l.startswith("ok> ") and got: break got.append(l[4:] if l.startswith("ok> ") else l) text = " ".join(got) if " ok" in text: text = text[:text.rfind(" ok")] words = text.split() src = open(os.path.join(ROOT, KERNEL_WORDS_SRC)).read() m = re.search(r"sk_v4_words\[\]\s*=\s*\{([^}]*)\}", src) for k in re.findall(r'"([^"]+)"', m.group(1) if m else ""): if k not in words: words.append(k) return words def definitions(): """Where each name is defined, and what its source says of it: {name: (file, stack, text)}.""" found = {} for rel in SOURCES: lines = open(os.path.join(ROOT, rel), errors="replace").read().split("\n") for i, l in enumerate(lines): names = [] if rel.endswith(".f18"): names = re.findall(r'",\s+(\S+)\s+#,', l) else: names = re.findall(r"(?:^|\s):\s+(\S+)", l) names += re.findall(r"\b(?:CONSTANT|VARIABLE|DEFER|CREATE|VOCABULARY)\s+(\S+)", l) for n in names: if n in found: continue stack, text = "", "" inline = re.search(r":\s+" + re.escape(n) + r"\s+(\([^)]*\))", l) if inline: stack = inline.group(1) # the comment block just above j, block = i - 1, [] while j >= 0 and lines[j].startswith("\\"): block.insert(0, lines[j].lstrip("\\").strip()) j -= 1 comment = " ".join(b for b in block if b) cm = re.match(r"(\([^)]*--[^)]*\))\s*(.*)", comment) if cm and not stack: stack, comment = cm.group(1), cm.group(2) text = comment found[n] = (rel, stack, text) for k in re.findall(r'"([^"]+)"', re.search(r"sk_v4_words\[\]\s*=\s*\{([^}]*)\}", open(os.path.join(ROOT, KERNEL_WORDS_SRC)).read()).group(1)): found.setdefault(k, (KERNEL_WORDS_SRC, "", "")) return found def read_table(path): table = {} if os.path.exists(path): for l in open(path).read().split("\n"): # a comment is a line beginning with # that has no tab: a row always has tabs, and # some words begin with # (#, #>, #S) if not l.strip() or (l.startswith("#") and "\t" not in l): continue f = l.split("\t") while len(f) < 3: f.append("") table[f[0]] = (f[1].strip(), f[2].strip()) return table def main(): if len(sys.argv) < 2: sys.exit(__doc__) mode = sys.argv[1] if mode == "words": print("\n".join(live_words())) return path = sys.argv[2] table = read_table(path) words = live_words() if mode == "draft": defs = definitions() new = [w for w in words if w not in table] with open(path, "a") as f: if not table: f.write("# docs/reference/v4-dictionary.tsv -- the dictionary of v4 (#157): name, stack notation,\n" "# description, tab-separated. The names are the live system's (v4/tools/mkdictionary.py\n" "# check refuses a missing or a stale one). A row whose description starts TODO is a draft.\n") for w in sorted(new): rel, stack, text = defs.get(w, ("", "", "")) desc = ("TODO " + text) if text else "TODO" f.write(f"{w}\t{stack}\t{desc}\n") print(f"mkdictionary: {len(words)} live words; {len(new)} drafted into {path}") return if mode == "check": bad = [] for w in words: if w not in table: bad.append(f"no entry: {w}") elif not table[w][0] or not table[w][1] or table[w][1].startswith("TODO"): bad.append(f"unfinished: {w}") for w in table: if w not in words: bad.append(f"not a live word: {w}") if bad: print("\n".join(bad)) print(f"mkdictionary: {len(bad)} problem(s) in {path} ({len(words)} live words)") sys.exit(1) print(f"mkdictionary: {path} covers all {len(words)} live words") return if mode == "md": defs = definitions() out = sys.argv[3] with open(out, "w") as f: f.write("# The dictionary of v4\n\n") f.write(f"Every word StarForth v4 boots with -- {len(words)} of them -- in alphabetical order: its " "stack notation (what it takes and leaves: `( before -- after )`), what it does, and " "where it is defined.\n\n") f.write("| Word | Stack | Description | Defined in |\n|---|---|---|---|\n") for w in sorted(words, key=lambda x: (x.upper(), x)): stack, desc = table.get(w, ("", "")) rel = defs.get(w, ("", "", ""))[0] cell = lambda x: x.replace("|", "\\|") f.write(f"| `{cell(w)}` | `{cell(stack)}` | {cell(desc)} | `{rel}` |\n") print(f"mkdictionary: {out}, {len(words)} words") return sys.exit(__doc__) if __name__ == "__main__": main()