stack/scripts/generate-specification.py
tim 1a6f915f04 Add paired specification.json and specification.html generator
Single SPEC source regenerates machine-readable JSON for agents and
dark HTML for humans so architecture survives lost chat sessions.
2026-07-22 19:54:42 -07:00

640 lines
24 KiB
Python
Executable file
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""
Single source of truth for project specification.
Writes:
specification.json — for coding agents / tools
specification.html — for humans
Run after architecture changes:
./scripts/generate-specification.py
Why JSON for agents (not only HTML):
- Stable keys, easy to grep/parse
- No layout noise
- Matches how tools ingest structured context
HTML is a readable projection of the same data — never edit HTML by hand.
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SPEC_VERSION = "1.2.0"
GENERATED_AT = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
# ---------------------------------------------------------------------------
# Canonical specification (edit THIS, then re-run the generator)
# ---------------------------------------------------------------------------
SPEC = {
"meta": {
"name": "arr-stack",
"title": "Homelab media + local AI stack",
"version": SPEC_VERSION,
"generated_at": GENERATED_AT,
"generator": "scripts/generate-specification.py",
"repository": {
"forgejo": "http://192.168.8.123:3002/tim/stack",
"clone_ssh": "ssh://git@192.168.8.123:2222/tim/stack.git",
"default_branch": "main",
},
"primary_host": {
"hostname": "stack",
"lan_ip": "192.168.8.123",
"user": "tim",
"os": "Ubuntu Server 24.04",
"hardware": "AMD Ryzen 7 5800H + Radeon Graphics (shared RAM), ~2832 GB RAM",
"stack_root": "/opt/stack",
"media_root": "/storage",
},
"demo_credentials": {
"username": "tim",
"password": "asdfasdf",
"note": "Tech demo only; used for local UIs when configured that way",
},
"docs_for_humans": [
"specification.html",
"quickstart.md",
"README.md",
],
"docs_for_agents": [
"specification.json",
"AGENTS.md",
"quickstart.md",
"profiles/*.env",
"docker-compose.yml",
],
"never_commit": [
".env",
"secrets/.env",
"wireguard/wg0.conf",
"config/",
"models/**/*.gguf",
"data/",
],
},
"architecture": {
"summary": (
"Docker Compose stack combining VPN-locked torrents (*arr + qBittorrent via gluetun), "
"Usenet (SABnzbd), media (Jellyfin), local LLM inference (Prism llama-server + Open WebUI), "
"Forgejo git, OpenHands agent, Caddy LAN landing page, and Cloudflare DDNS."
),
"networks": {
"arr-net": "Bridge network for most services",
"gluetun_namespace": "qbittorrent uses network_mode: service:gluetun (VPN killswitch)",
"host_network": ["cloudflare-ddns"],
},
"data_layout": {
"CONFIG_DIR": "App configs (default /opt/stack/config on server)",
"DATA_DIR": "Shared media tree (default /storage on server)",
"paths": {
"media": "/data/media/{movies,tv,music}",
"torrents": "/data/torrents/{movies,tv,music}",
"usenet": "/data/usenet/incomplete and /data/usenet/complete/{movies,tv,music}",
"models": "models/bonsai/*.gguf mounted read-only at /models in bonsai",
"workspace": "AI_WORKSPACE_DIR for Open WebUI + OpenHands",
},
},
"security_model": {
"torrents": "Only via WireGuard through gluetun; no BitTorrent port on host",
"usenet": "Direct SSL to provider (not through gluetun)",
"lan_firewall": "UFW allow service ports from LAN_SUBNET only",
"landing_and_uis": "Plain HTTP on LAN (no TLS certs yet)",
},
},
"hardware_profiles": {
"how_to_apply": "./scripts/apply-profile.sh mini|mid|gaming",
"files": {
"mini": "profiles/mini.env",
"mid": "profiles/mid.env",
"gaming": "profiles/gaming.env",
},
"profiles": {
"mini": {
"hardware": "Beelink / SER5-class mini PC",
"inference": "cpu",
"ngl": 0,
"default_ctx": 32768,
"threads": 12,
"default_model": "Ternary-Bonsai-27B-Q2_0.gguf",
"model_repo": "prism-ml/Ternary-Bonsai-27B-gguf",
"notes": "CPU preferred on APU; do not force NGL in docker-compose.override.yml",
},
"mid": {
"hardware": "~8GB AMD discrete GPU, ~16GB system RAM",
"inference": "vulkan",
"ngl": 99,
"default_ctx": 8192,
"threads": 8,
"kv4": True,
"default_model": "Qwen3.5-9B-abliterated-v2-MAX.Q4_K_M.gguf",
"model_repo": "mradermacher/Qwen3.5-9B-abliterated-v2-MAX-GGUF",
"notes": "Fits VRAM; use KV4 for memory",
},
"gaming": {
"hardware": "~16GB AMD GPU, ~64GB system RAM",
"inference": "vulkan",
"ngl": 99,
"default_ctx": 32768,
"threads": 16,
"default_model": "Ternary-Bonsai-27B-Q2_0.gguf",
"also_download": ["Qwen3.5-9B Q4_K_M"],
"notes": "Full Vulkan + large context",
},
},
"models": {
"bonsai27": {
"file": "Ternary-Bonsai-27B-Q2_0.gguf",
"repo": "prism-ml/Ternary-Bonsai-27B-gguf",
"runtime": "PrismML llama.cpp fork (custom Q2_0)",
"size_approx": "~7.2 GB",
},
"qwen9_abliterated_q4km": {
"file": "Qwen3.5-9B-abliterated-v2-MAX.Q4_K_M.gguf",
"repo": "mradermacher/Qwen3.5-9B-abliterated-v2-MAX-GGUF",
"runtime": "Same Prism llama-server image (standard GGUF)",
"size_approx": "~5.6 GB",
},
},
},
"services": [
{
"id": "landing",
"name": "Landing (Caddy)",
"image": "caddy:2-alpine",
"port": 80,
"url": "http://HOST/",
"purpose": "Dark-mode LAN resource map",
"network": "arr-net",
"category": "gateway",
},
{
"id": "gluetun",
"name": "gluetun",
"image": "qmcgaw/gluetun:v3",
"port": None,
"published_ports": ["8080:8080 (qBittorrent WebUI)"],
"purpose": "WireGuard VPN + killswitch",
"network": "arr-net",
"category": "vpn",
"config": "wireguard/wg0.conf",
},
{
"id": "qbittorrent",
"name": "qBittorrent",
"image": "lscr.io/linuxserver/qbittorrent:latest",
"port": 8080,
"url": "http://HOST:8080/",
"purpose": "Torrent client (VPN-only via gluetun)",
"network": "service:gluetun",
"category": "download",
},
{
"id": "sabnzbd",
"name": "SABnzbd",
"image": "lscr.io/linuxserver/sabnzbd:latest",
"port": 8085,
"url": "http://HOST:8085/",
"purpose": "Usenet NZB downloader",
"network": "arr-net",
"category": "download",
},
{
"id": "prowlarr",
"name": "Prowlarr",
"image": "lscr.io/linuxserver/prowlarr:latest",
"port": 9696,
"url": "http://HOST:9696/",
"purpose": "Indexer manager (torrents + NZB)",
"network": "arr-net",
"category": "download",
},
{
"id": "flaresolverr",
"name": "FlareSolverr",
"image": "ghcr.io/flaresolverr/flaresolverr:latest",
"port": 8191,
"url": "http://flaresolverr:8191 (internal)",
"purpose": "Cloudflare JS challenge proxy for Prowlarr",
"network": "arr-net",
"category": "download",
"published": False,
},
{
"id": "sonarr",
"name": "Sonarr",
"image": "lscr.io/linuxserver/sonarr:latest",
"port": 8989,
"url": "http://HOST:8989/",
"purpose": "TV automation",
"network": "arr-net",
"category": "media",
},
{
"id": "radarr",
"name": "Radarr",
"image": "lscr.io/linuxserver/radarr:latest",
"port": 7878,
"url": "http://HOST:7878/",
"purpose": "Movie automation",
"network": "arr-net",
"category": "media",
},
{
"id": "lidarr",
"name": "Lidarr",
"image": "lscr.io/linuxserver/lidarr:latest",
"port": 8686,
"url": "http://HOST:8686/",
"purpose": "Music automation",
"network": "arr-net",
"category": "media",
},
{
"id": "bazarr",
"name": "Bazarr",
"image": "lscr.io/linuxserver/bazarr:latest",
"port": 6767,
"url": "http://HOST:6767/",
"purpose": "Subtitles",
"network": "arr-net",
"category": "media",
},
{
"id": "jellyfin",
"name": "Jellyfin",
"image": "lscr.io/linuxserver/jellyfin:latest",
"port": 8096,
"url": "http://HOST:8096/",
"purpose": "Media server / playback",
"network": "arr-net",
"category": "media",
},
{
"id": "bonsai",
"name": "Bonsai (llama-server)",
"image": "arr-stack/bonsai:local (ai/Dockerfile)",
"port": 8081,
"url": "http://HOST:8081/v1",
"purpose": "Local LLM OpenAI-compatible API (Prism build)",
"network": "arr-net",
"category": "ai",
},
{
"id": "open-webui",
"name": "Open WebUI",
"image": "ghcr.io/open-webui/open-webui:main",
"port": 3000,
"url": "http://HOST:3000/",
"purpose": "Chat UI + RAG + tools",
"network": "arr-net",
"category": "ai",
"llm_base": "http://bonsai:8080/v1",
},
{
"id": "searxng",
"name": "SearXNG",
"image": "docker.io/searxng/searxng:latest",
"port": 8888,
"url": "http://127.0.0.1:8888/ (localhost only)",
"purpose": "Private search for Open WebUI RAG",
"network": "arr-net",
"category": "ai",
"published": "loopback",
},
{
"id": "openhands",
"name": "OpenHands",
"image": "ghcr.io/all-hands-ai/openhands:0.54",
"port": 3001,
"url": "http://HOST:3001/",
"purpose": "AI coding agent (always deployed)",
"network": "arr-net",
"category": "ai",
"note": "No Forgejo integration in UI — clone repos into workspace",
},
{
"id": "forgejo",
"name": "Forgejo",
"image": "codeberg.org/forgejo/forgejo:11",
"port": 3002,
"url": "http://HOST:3002/",
"ssh_port": 2222,
"purpose": "Self-hosted Git",
"network": "arr-net",
"category": "code",
},
{
"id": "cloudflare-ddns",
"name": "Cloudflare DDNS",
"image": "timothyjmiller/cloudflare-ddns:latest",
"port": None,
"url": None,
"purpose": "Update public IPv4 A records for domain + subdomains",
"network": "host",
"category": "dns",
"domain": "devopshomelab.com",
"subdomains": ["mc", "minecraft", "vpn", "www", "forgejo", "jellyfin"],
},
],
"scripts": {
"generate-specification.py": "Regenerate specification.json + specification.html (this file)",
"apply-profile.sh": "Merge profiles/{mini,mid,gaming}.env into .env",
"bootstrap-stack.sh": "Fresh bring-up: dirs, models, compose up all, heal gluetun, usenet, cloudflare",
"download-models.sh": "Download Bonsai 27B and/or Qwen 9B GGUF into models/bonsai",
"download-bonsai-model.sh": "Wrapper → download-models.sh",
"configure-usenet.sh": "SABnzbd folders + Newshosting/Tweaknews from secrets/.env",
"render-cloudflare-ddns-env.sh": "Build config/cloudflare-ddns/ddns.env from secrets",
"gluetun-endpoint-watch.sh": "If gluetun unhealthy: dig WG host, update Endpoint, recreate gluetun+qbit",
},
"ansible": {
"entry": "ansible/site.yml",
"inventory": "ansible/inventory/hosts.ini",
"roles": ["common", "docker", "amd_vulkan", "firewall", "stack", "users"],
"stack_deploy_method": "git (default) or rsync",
"server_repo": "/opt/stack tracks Forgejo via git pull",
"timers": ["arr-stack.service", "gluetun-endpoint-watch.timer"],
},
"workflows": {
"fresh_install": [
"cp .env.example .env; fill STACK_HOST, LAN_SUBNET, usenet, cloudflare keys",
"Place wireguard/wg0.conf (Endpoint IP + # EndpointHost = hostname)",
"./scripts/apply-profile.sh mini|mid|gaming",
"./scripts/bootstrap-stack.sh",
"One-time UI: Prowlarr apps, Jellyfin libraries, Forgejo/Open WebUI admin (demo user/pass)",
],
"day_to_day_server": [
"cd /opt/stack && git pull",
"docker compose up -d",
"rebuild bonsai if Dockerfile/profile backend changed",
],
"after_secrets_change": {
"cloudflare": [
"./scripts/render-cloudflare-ddns-env.sh",
"docker compose up -d cloudflare-ddns --force-recreate",
],
"usenet_servers": ["./scripts/configure-usenet.sh", "Test servers in SABnzbd UI"],
"nzbgeek": ["Update API key in Prowlarr UI"],
"wireguard": [
"Edit wireguard/wg0.conf",
"docker compose up -d gluetun --force-recreate",
"or ./scripts/gluetun-endpoint-watch.sh",
],
"stack_env": ["Edit .env", "docker compose up -d (+ build bonsai if backend changes)"],
},
"switch_profile": [
"./scripts/apply-profile.sh mid",
"docker compose build bonsai && docker compose up -d",
],
"agent_git": [
"Commit meaningful changes with complete sentences",
"Push to origin main (Forgejo)",
"Never commit secrets",
"After architecture changes: run generate-specification.py and commit both outputs",
],
},
"compose": {
"file": "docker-compose.yml",
"override": "docker-compose.override.yml",
"always_deploy_all": True,
"note": "OpenHands is not behind a compose profile; full stack always starts",
"override_rules": [
"Do not put group_add names (video/render) — use host GIDs or 0666 /dev/dri",
"Do not force bonsai NGL in override on mini (overrides .env)",
"mini: jellyfin /dev/dri only; mid/gaming: add bonsai devices for Vulkan",
],
},
"env_and_secrets": {
"stack_env": ".env (from .env.example) — ports, paths, AI, bootstrap flags, demo login",
"secrets_env": "secrets/.env — usenet + cloudflare API (gitignored)",
"approx_env_keys": 80,
"demo_user": "tim",
"demo_pass": "asdfasdf",
},
}
def write_json(path: Path) -> None:
path.write_text(json.dumps(SPEC, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
def _esc(s: object) -> str:
return (
str(s)
.replace("&", "&")
.replace("<", "&lt;")
.replace(">", "&gt;")
.replace('"', "&quot;")
)
def write_html(path: Path) -> None:
m = SPEC["meta"]
host = m["primary_host"]["lan_ip"]
services = SPEC["services"]
profiles = SPEC["hardware_profiles"]["profiles"]
scripts = SPEC["scripts"]
workflows = SPEC["workflows"]
def svc_rows() -> str:
rows = []
for s in services:
url = (s.get("url") or "").replace("HOST", host)
port = s.get("port")
port_s = str(port) if port is not None else ""
rows.append(
"<tr>"
f"<td><code>{_esc(s['id'])}</code></td>"
f"<td>{_esc(s['name'])}</td>"
f"<td>{_esc(s.get('category', ''))}</td>"
f"<td>{_esc(port_s)}</td>"
f"<td><a href=\"{_esc(url) if url.startswith('http') else '#'}\">{_esc(url)}</a></td>"
f"<td>{_esc(s['purpose'])}</td>"
"</tr>"
)
return "\n".join(rows)
def profile_cards() -> str:
cards = []
for name, p in profiles.items():
cards.append(
f"""
<article class="card">
<h3>{_esc(name)}</h3>
<p class="muted">{_esc(p['hardware'])}</p>
<ul>
<li><strong>Inference:</strong> {_esc(p['inference'])} (ngl={_esc(p['ngl'])})</li>
<li><strong>Context:</strong> {_esc(p['default_ctx'])}</li>
<li><strong>Model:</strong> <code>{_esc(p['default_model'])}</code></li>
<li>{_esc(p.get('notes', ''))}</li>
</ul>
</article>"""
)
return "\n".join(cards)
def script_list() -> str:
return "\n".join(
f"<li><code>{_esc(k)}</code> — {_esc(v)}</li>" for k, v in scripts.items()
)
def workflow_secrets() -> str:
parts = []
for k, steps in workflows["after_secrets_change"].items():
items = "".join(f"<li><code>{_esc(s)}</code></li>" for s in steps)
parts.append(f"<h4>{_esc(k)}</h4><ol>{items}</ol>")
return "\n".join(parts)
html = f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>{_esc(m['title'])} — specification</title>
<meta name="color-scheme" content="dark" />
<style>
:root {{
--bg: #0c0e12; --elev: #141820; --card: #1a1f2a; --border: #2a3344;
--text: #e8ecf4; --muted: #8b95a8; --accent: #6ea8fe; --ok: #7dd3a0;
--font: "Segoe UI", system-ui, sans-serif;
--mono: ui-monospace, "Cascadia Code", Menlo, monospace;
}}
* {{ box-sizing: border-box; }}
body {{
margin: 0; font-family: var(--font); background: var(--bg); color: var(--text);
line-height: 1.55; padding: 2rem 1.25rem 4rem;
background-image: radial-gradient(ellipse 80% 40% at 50% -10%, rgba(110,168,254,.12), transparent);
}}
.wrap {{ max-width: 980px; margin: 0 auto; }}
h1 {{ font-size: 1.75rem; letter-spacing: -0.02em; margin: 0 0 .35rem; }}
h2 {{ font-size: 1.2rem; margin: 2rem 0 .75rem; border-bottom: 1px solid var(--border); padding-bottom: .4rem; }}
h3 {{ font-size: 1.05rem; margin: 0 0 .4rem; }}
h4 {{ font-size: .95rem; margin: 1rem 0 .35rem; color: var(--accent); }}
p, li {{ color: var(--muted); }}
.lead {{ color: var(--text); font-size: 1.05rem; }}
.meta {{
display: flex; flex-wrap: wrap; gap: .5rem; margin: 1rem 0 1.5rem;
}}
.pill {{
font-family: var(--mono); font-size: .78rem; padding: .3rem .65rem;
border: 1px solid var(--border); border-radius: 999px; background: var(--elev); color: var(--accent);
}}
.banner {{
background: var(--elev); border: 1px solid var(--border); border-radius: 12px;
padding: 1rem 1.15rem; margin-bottom: 1.5rem;
}}
.banner strong {{ color: var(--ok); }}
code {{ font-family: var(--mono); font-size: .88em; background: var(--card); padding: .1rem .35rem; border-radius: 4px; color: var(--accent); }}
a {{ color: var(--accent); }}
table {{ width: 100%; border-collapse: collapse; font-size: .88rem; margin: .75rem 0 1.25rem; }}
th, td {{ text-align: left; padding: .5rem .55rem; border-bottom: 1px solid var(--border); vertical-align: top; }}
th {{ color: var(--text); font-weight: 600; background: var(--elev); }}
.grid {{ display: grid; grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); gap: .75rem; }}
.card {{
background: var(--card); border: 1px solid var(--border); border-radius: 12px; padding: 1rem;
}}
.card ul {{ margin: .4rem 0 0; padding-left: 1.1rem; }}
.card li {{ margin: .25rem 0; }}
.muted {{ color: var(--muted); font-size: .9rem; }}
footer {{ margin-top: 2.5rem; font-size: .8rem; color: #5c667a; text-align: center; }}
</style>
</head>
<body>
<div class="wrap">
<h1>{_esc(m['title'])}</h1>
<p class="lead">{_esc(SPEC['architecture']['summary'])}</p>
<div class="meta">
<span class="pill">spec v{_esc(m['version'])}</span>
<span class="pill">generated {_esc(m['generated_at'])}</span>
<span class="pill">host {_esc(host)}</span>
<span class="pill">stack {_esc(m['primary_host']['stack_root'])}</span>
</div>
<div class="banner">
<strong>Source of truth:</strong> edit <code>scripts/generate-specification.py</code> (the <code>SPEC</code> dict),
then run <code>./scripts/generate-specification.py</code>. That rewrites both
<code>specification.json</code> (agents) and <code>specification.html</code> (you) so they never drift.
Agents should prefer <code>specification.json</code> + <code>AGENTS.md</code>; humans use this HTML and <code>quickstart.md</code>.
</div>
<h2>Primary host</h2>
<ul>
<li>LAN: <code>http://{_esc(host)}/</code> (Caddy landing)</li>
<li>User: <code>{_esc(m['primary_host']['user'])}</code> · OS: {_esc(m['primary_host']['os'])}</li>
<li>Hardware: {_esc(m['primary_host']['hardware'])}</li>
<li>Git: <a href="{_esc(m['repository']['forgejo'])}">{_esc(m['repository']['forgejo'])}</a></li>
<li>Demo login: <code>{_esc(m['demo_credentials']['username'])}</code> / <code>{_esc(m['demo_credentials']['password'])}</code></li>
</ul>
<h2>Hardware profiles</h2>
<p class="muted">Apply with <code>./scripts/apply-profile.sh mini|mid|gaming</code></p>
<div class="grid">
{profile_cards()}
</div>
<h2>Services</h2>
<p class="muted">All services deploy by default (<code>docker compose up -d</code>). Replace HOST with {_esc(host)}. Links use plain <code>http://</code> (no TLS).</p>
<table>
<thead>
<tr><th>ID</th><th>Name</th><th>Category</th><th>Port</th><th>URL</th><th>Purpose</th></tr>
</thead>
<tbody>
{svc_rows()}
</tbody>
</table>
<h2>Scripts</h2>
<ul>
{script_list()}
</ul>
<h2>After changing secrets</h2>
<p class="muted">Edit <code>secrets/.env</code> first, then:</p>
{workflow_secrets()}
<h2>Fresh install</h2>
<ol>
{''.join(f'<li><code>{_esc(s)}</code></li>' for s in workflows['fresh_install'])}
</ol>
<h2>Day-to-day on server</h2>
<ol>
{''.join(f'<li><code>{_esc(s)}</code></li>' for s in workflows['day_to_day_server'])}
</ol>
<h2>Ansible</h2>
<ul>
<li>Entry: <code>{_esc(SPEC['ansible']['entry'])}</code></li>
<li>Roles: {', '.join(f'<code>{_esc(r)}</code>' for r in SPEC['ansible']['roles'])}</li>
<li>Deploy method: {_esc(SPEC['ansible']['stack_deploy_method'])}</li>
</ul>
<h2>Never commit</h2>
<ul>
{''.join(f'<li><code>{_esc(x)}</code></li>' for x in m['never_commit'])}
</ul>
<footer>
Generated by scripts/generate-specification.py · paired with specification.json · arr-stack
</footer>
</div>
</body>
</html>
"""
path.write_text(html, encoding="utf-8")
def main() -> None:
json_path = ROOT / "specification.json"
html_path = ROOT / "specification.html"
write_json(json_path)
write_html(html_path)
print(f"Wrote {json_path.relative_to(ROOT)}")
print(f"Wrote {html_path.relative_to(ROOT)}")
print(f"spec version {SPEC_VERSION} @ {GENERATED_AT}")
if __name__ == "__main__":
main()