wren-linux / tools / site.py
# tools/site.py: renders Wren Linux's website (tools/build-site.sh runs it).
# Usage: python3 -I tools/site.py PAGES SOURCE LOG OUT
# PAGES the site's pages: layout.html, style.css and pages/*.html, each
# page its title on the first line and its body after it
# SOURCE the files of the commit, for the source browser
# LOG the commit log, one "HASH<tab>DATE<tab>SUBJECT" a line
# OUT the folder to write the site into (made, or emptied of nothing)
# The values come from the environment, checked here: SITE_FPR (40 hex),
# SITE_GIT_URL (https), SITE_COMMIT (40 hex), SITE_ONION (optional, a v3 .onion
# name), SITE_SIGNED (yes once build-site.sh checked the commit's signature). Everything taken from the repository is escaped: no file in it, nor
# its name, can become markup or script in the site. A Markdown file (README.md, docs/) is
# also shown as a page, by tools/site_md.py, which escapes it the same way and writes only
# its own few tags; the Docs page lists the ones for readers.
import html
import importlib.util
import os
import posixpath
import re
import shutil
import sys
import urllib.parse
# (python3 -I: this file's folder isn't where modules are looked for, so by its path; and no
# __pycache__ left in the copy it is built from)
sys.dont_write_bytecode = True
_spec = importlib.util.spec_from_file_location("site_md", os.path.join(os.path.dirname(os.path.abspath(__file__)), "site_md.py"))
site_md = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(site_md)
_spec = importlib.util.spec_from_file_location("site_hl", os.path.join(os.path.dirname(os.path.abspath(__file__)), "site_hl.py"))
site_hl = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(site_hl)
pages_dir, source, log_file, out = sys.argv[1:5]
def fail(msg):
sys.stderr.write("site.py: %s\n" % msg)
sys.exit(1)
def value(name, pattern, optional=False):
v = os.environ.get(name, "")
if not v and optional:
return ""
if not re.fullmatch(pattern, v):
fail("%s isn't what it should be: %r" % (name, v))
return v
FPR = value("SITE_FPR", r"[0-9A-F]{40}")
GIT_URL = value("SITE_GIT_URL", r"https://[A-Za-z0-9.-]+(/[A-Za-z0-9._-]+)+")
COMMIT = value("SITE_COMMIT", r"[0-9a-f]{40}")
ONION = value("SITE_ONION", r"[a-z2-7]{56}\.onion", optional=True)
e = html.escape
SUBS = {
"@GIT_URL@": e(GIT_URL),
"@FPR_SPACED@": " ".join(FPR[i:i + 4] for i in range(0, 40, 4)),
"@ONION_COMPARE@": (", and this site's onion address, <code>%s</code>, in Tor Browser" % e(ONION)) if ONION else "",
"@ONION_LINE@": (" Also at <code>%s</code> in Tor Browser." % e(ONION)) if ONION else "",
"@COMMIT@": COMMIT[:12],
# (a link: its depth is filled in with @ROOT@ afterwards)
"@SIGNED@": ', signed by the key on <a href="@ROOT@verify.html">Verify</a>' if os.environ.get("SITE_SIGNED") == "yes"
else " (a preview: its signature wasn't checked)",
}
layout = open(os.path.join(pages_dir, "layout.html"), encoding="utf-8").read()
def fill(text):
for k, v in SUBS.items():
text = text.replace(k, v)
return text
def page(rel, title, body):
"""Writes OUT/rel: body (markup made here or a page of the site) in the layout."""
depth = rel.count("/")
root = "../" * depth
frame = fill(layout).replace("@TITLE@", e(title)).replace("@ROOT@", root)
left = re.findall(r"@[A-Z_]+@", frame.replace("@BODY@", ""))
if left:
fail("layout.html: nothing fills %s" % ", ".join(sorted(set(left))))
text = frame.replace("@BODY@", body)
path = os.path.join(out, rel)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(text)
os.makedirs(out, exist_ok=True)
shutil.copyfile(os.path.join(pages_dir, "style.css"), os.path.join(out, "style.css"))
# Images: PNG only, and only the picture. A drawing program leaves its history,
# dates with the time zone, and EXIF in text chunks: none of that is copied.
def clean_png(src, dst):
import struct
import zlib
d = open(src, "rb").read()
if d[:8] != b"\x89PNG\r\n\x1a\n":
fail("%s isn't a PNG" % src)
keep, i = [d[:8]], 8
while i < len(d):
n = struct.unpack(">I", d[i : i + 4])[0]
chunk = d[i : i + 12 + n]
if len(chunk) != 12 + n or struct.unpack(">I", chunk[-4:])[0] != zlib.crc32(chunk[4:-4]) & 0xFFFFFFFF:
fail("%s is damaged" % src)
if chunk[4:8] in (b"IHDR", b"PLTE", b"tRNS", b"IDAT", b"IEND"):
keep.append(chunk)
i += 12 + n
os.makedirs(os.path.dirname(dst), exist_ok=True)
with open(dst, "wb") as f:
f.write(b"".join(keep))
images = os.path.join(pages_dir, "image")
if os.path.isdir(images):
for name in sorted(os.listdir(images)):
if not re.fullmatch(r"[a-z0-9-]+\.png", name):
fail("image/%s: only PNG files, named in small letters, digits and dashes" % name)
clean_png(os.path.join(images, name), os.path.join(out, "image", name))
# The site's own pages
for name in sorted(os.listdir(os.path.join(pages_dir, "pages"))):
if not re.fullmatch(r"[a-z0-9-]+\.html", name):
continue
title, _, body = open(os.path.join(pages_dir, "pages", name), encoding="utf-8").read().partition("\n")
body = fill(body)
left = re.findall(r"@[A-Z_]+@", body)
if left:
fail("pages/%s: nothing fills %s" % (name, ", ".join(sorted(set(left)))))
page(name, title.strip(), body)
# The source browser: one page a folder, one a file, every name and line escaped
SAFE = re.compile(r"[A-Za-z0-9._+-]+")
skipped = []
docs = {} # a Markdown file's path in the commit: its title
unlinked = []
def resolver(rel):
"""For the Markdown file at REL: a link's path (from that file's folder, or from the top
with /) to the href of its page in the source browser, or None (out of the commit, not a
file or folder of it, a name the browser doesn't show)."""
here = posixpath.dirname("source/" + rel)
def resolve(path, anchor):
if path == "":
target = rel
elif path.startswith("/"):
target = posixpath.normpath(path.lstrip("/") or ".")
else:
target = posixpath.normpath(posixpath.join(posixpath.dirname(rel), path))
if target.startswith("..") or target.startswith("/"):
return None
parts = [] if target == "." else target.split("/")
if any(p == ".git" or not SAFE.fullmatch(p) for p in parts):
return None
full = os.path.join(source, *parts)
if not os.path.realpath(full).startswith(os.path.realpath(source) + os.sep) and parts:
return None
if os.path.isdir(full) and not os.path.islink(full):
page_rel = "source/" + "/".join(parts + ["index.html"])
elif os.path.isfile(full) or os.path.islink(full):
page_rel = "source/" + target + ".html"
else:
return None
return posixpath.relpath(page_rel, here) + ("#" + anchor if anchor else "")
return resolve
def href(*parts):
return "/".join(urllib.parse.quote(p) for p in parts)
def breadcrumb(parts, depth):
up = "../" * depth
crumbs = ['<a href="%sindex.html">wren-linux</a>' % up]
for i, p in enumerate(parts):
crumbs.append('<a href="%s%sindex.html">%s</a>' % (up, href(*parts[: i + 1]) + "/", e(p)))
return " / ".join(crumbs)
# Prose (wraps, no numbers: its page is the way to read it); everything else is code: its lines
# kept whole (they scroll sideways, as programmers expect), each numbered, a link to itself.
# The numbers are drawn by the style sheet, not written in the text, so a copy of the code
# is the code alone.
PROSE = re.compile(r".*\.(md|txt)|LICENSE|COPYING|CREDITS")
# (each line coloured as vim colours it, for bash and Python: tools/site_hl.py, escaped there)
def numbered(name, text):
return "".join('<span class="l" id="L%d"><a class="ln" href="#L%d"></a>%s\n</span>' % (i, i, line)
for i, line in enumerate(site_hl.lines(name, text), 1))
def text_of(path):
if os.path.getsize(path) > 512 * 1024:
return None
data = open(path, "rb").read()
if b"\0" in data:
return None
try:
return data.decode("utf-8")
except UnicodeDecodeError:
return None
for top, dirs, files in os.walk(source):
rel_dir = os.path.relpath(top, source)
parts = [] if rel_dir == "." else rel_dir.split(os.sep)
dirs[:] = sorted(d for d in dirs if d != ".git" and SAFE.fullmatch(d))
depth = len(parts) + 1 # pages under OUT/source/
items = []
for d in dirs:
items.append('<li class="dir"><a href="%s/index.html">%s/</a></li>' % (href(d), e(d)))
for name in sorted(files):
full = os.path.join(top, name)
if not SAFE.fullmatch(name):
skipped.append(os.path.join(rel_dir, name))
continue
items.append('<li><a href="%s.html">%s</a></li>' % (href(name), e(name)))
if os.path.islink(full):
body = "<p>A link to <code>%s</code>.</p>" % e(os.readlink(full))
else:
text = text_of(full)
if text is None:
body = '<p>Not shown here (not text, or too long): it is in the <a href="%swren-linux.tar.gz">download</a>.</p>' % ("../" * depth)
elif PROSE.fullmatch(name):
body = '<div class="file prose"><pre><code>%s</code></pre></div>' % e(text)
else:
body = '<div class="file"><pre><code>%s</code></pre></div>' % numbered(name, text)
file_parts = parts + [name]
rel = "/".join(file_parts)
path_line = '<p class="path">%s / %s' % (breadcrumb(parts, len(parts)), e(name))
if name.endswith(".md") and not os.path.islink(full) and text is not None:
# the page: the file as a document; and the file itself, as plain text, beside it
doc, title, missing = site_md.render(text, resolver(rel))
docs[rel] = title or name
unlinked += ["%s: %s" % (rel, m) for m in missing]
page("source/" + rel + ".plain.html", rel + " (plain text) - Wren Linux",
'%s (<a href="%s.html">as a page</a>)</p>\n%s' % (path_line, href(name), body))
page("source/" + rel + ".html", (title or rel) + " - Wren Linux",
'%s (<a href="%s.plain.html">as plain text</a>)</p>\n<div class="doc">\n%s\n</div>' % (path_line, href(name), doc))
continue
page("source/" + rel + ".html", rel + " - Wren Linux", "%s</p>\n%s" % (path_line, body))
head = ""
if not parts:
head = (
"<h1>Source</h1>\n<p>Every file of Wren Linux at commit <code>%s</code>, the one the key on "
'<a href="../verify.html">Verify</a> signed. Download it all: <a href="../wren-linux.tar.gz">wren-linux.tar.gz</a> '
'(<a href="../wren-linux.tar.gz.sha256">SHA-256</a>), or <code>git clone %s</code>. '
'The latest changes: <a href="log.html">the log</a>.</p>\n' % (COMMIT[:12], e(GIT_URL))
)
else:
head = '<p class="path">%s</p>\n' % breadcrumb(parts[:-1], len(parts)) + "<h1>%s/</h1>\n" % e(parts[-1])
page(
"source/" + "/".join(parts + ["index.html"]),
("/".join(parts) + "/ - " if parts else "Source - ") + "Wren Linux",
head + '<ul class="files">\n%s\n</ul>' % "\n".join(items),
)
# The log
rows = []
for line in open(log_file, encoding="utf-8", errors="replace"):
h, _, rest = line.rstrip("\n").partition("\t")
date, _, subject = rest.partition("\t")
if re.fullmatch(r"[0-9a-f]{40}", h):
rows.append("<tr><td>%s</td><td>%s</td><td>%s</td></tr>" % (h[:12], e(date), e(subject)))
page(
"source/log.html",
"Log - Wren Linux",
"<h1>Log</h1>\n<p>The latest commits, newest first.</p>\n<table class=\"log\">\n%s\n</table>" % "\n".join(rows),
)
# Docs: the ones for readers, each a page of the source browser (the notes for whoever works
# on Wren Linux itself, CLAUDE.md and tools/, and the vendored programs' are there too, unlisted)
def doc_list(paths):
return "<ul>\n%s\n</ul>" % "\n".join(
'<li><a href="source/%s.html">%s</a></li>' % ("/".join(urllib.parse.quote(x) for x in p.split("/")), e(docs[p])) for p in paths)
guides = sorted((p for p in docs if re.fullmatch(r"docs/[^/]+\.md", p)), key=lambda p: docs[p].lower())
extras = sorted((p for p in docs if re.fullmatch(r"docs/extras/[^/]+\.md", p) and p != "docs/extras/README.md"), key=lambda p: docs[p].lower())
project = [p for p in ("SECURITY.md", "THREAT-MODEL.md", "CONTRIBUTING.md") if p in docs]
sections = []
if "README.md" in docs:
sections.append("<h2>Wren Linux</h2>\n" + doc_list(["README.md"]))
if guides:
sections.append("<h2>Guides</h2>\n" + doc_list(guides))
if extras:
sections.append("<h2>The extras</h2>\n" + doc_list((["docs/extras/README.md"] if "docs/extras/README.md" in docs else []) + extras))
if project:
sections.append("<h2>The project</h2>\n" + doc_list(project))
page("docs.html", "Docs - Wren Linux",
"<h1>Docs</h1>\n<p>Wren Linux's documents, as pages, from the same commit as the rest of this site. Each one is also there "
"as plain text, from its page.</p>\n" + "\n".join(sections))
if skipped:
sys.stderr.write("site.py: not shown (names with unusual characters): %s\n" % ", ".join(skipped))
if unlinked:
sys.stderr.write("site.py: links shown as text (an address it doesn't take, or nothing there): %s\n" % "; ".join(unlinked))