#!/usr/bin/env python3 # SPDX-License-Identifier: AGPL-3.0-or-later from __future__ import annotations import argparse import base64 import hashlib import io import json import os import re import shutil import statistics import sys import tarfile import urllib.parse import urllib.request import zipfile from concurrent.futures import ProcessPoolExecutor from dataclasses import dataclass, field from pathlib import Path from typing import Union from fontTools import unicodedata as ftunicodedata from fontTools.pens.recordingPen import DecomposingRecordingPen from fontTools.subset import Options as SubsetOptions from fontTools.subset import Subsetter from fontTools.ttLib import TTFont REPO_ROOT = Path(__file__).resolve().parents[2] PACKAGE_ROOT = REPO_ROOT / "packages" / "fonts" FILES_DIR = PACKAGE_ROOT / "files" CSS_DIR = PACKAGE_ROOT / "css" MANIFEST_PATH = PACKAGE_ROOT / "manifest.json" TOOL_ROOT = Path(__file__).resolve().parent LOCK_PATH = TOOL_ROOT / "fonts.lock.json" SPDX_CSS_HEADER = "/* SPDX-License-Identifier: AGPL-3.0-or-later */" GENERATED_CSS_HEADER = "/* Generated by tools/fonts/build_fonts.py. Do not edit by hand. */" CHUNK_TARGET_BYTES = 64 * 1024 MIN_STANDALONE_BLOCK = 256 PLEX_GASP = {8: 10, 16: 7, 65535: 15} CJK_GASP = {7: 10, 65535: 15} IDENTITY_NAME_IDS = (1, 3, 4, 6, 16, 17, 18, 20, 21, 22, 25) RENAME_NAME_IDS = IDENTITY_NAME_IDS NO_PLEX_NAME_IDS = IDENTITY_NAME_IDS MUTABLE_TABLES = frozenset({"gasp", "name", "head"}) CRITICAL_TABLES = ( "glyf", "loca", "fpgm", "prep", "cvt ", "cmap", "OS/2", "hhea", "hmtx", "maxp", "post", "GDEF", "GPOS", "GSUB", ) HEAD_CHECKSUM_SLICE = slice(8, 12) HINTING_TABLES = ("fpgm", "prep", "cvt ") METRIC_FIELDS: tuple[tuple[str, str], ...] = ( ("head", "unitsPerEm"), ("head", "macStyle"), ("hhea", "ascender"), ("hhea", "descender"), ("hhea", "lineGap"), ("hhea", "caretSlopeRise"), ("hhea", "caretSlopeRun"), ("OS/2", "sTypoAscender"), ("OS/2", "sTypoDescender"), ("OS/2", "sTypoLineGap"), ("OS/2", "usWinAscent"), ("OS/2", "usWinDescent"), ("OS/2", "sxHeight"), ("OS/2", "sCapHeight"), ("OS/2", "usWeightClass"), ("OS/2", "fsSelection"), ("vhea", "ascent"), ("vhea", "descent"), ("vhea", "lineGap"), ) UPRIGHT_WEIGHTS = ( "Thin", "ExtraLight", "Light", "Regular", "Text", "Medium", "SemiBold", "Bold", ) ITALIC_WEIGHTS = ( "ThinItalic", "ExtraLightItalic", "LightItalic", "Italic", "TextItalic", "MediumItalic", "SemiBoldItalic", "BoldItalic", ) CSS_WEIGHTS: dict[str, int] = { "Thin": 100, "ExtraLight": 200, "Light": 300, "Regular": 400, "Text": 450, "Medium": 500, "SemiBold": 600, "Bold": 700, "ExtraBold": 800, } def css_weight(weight: str) -> int: upright = ( weight[: -len("Italic")] if weight.endswith("Italic") and weight != "Italic" else weight ) if upright == "Italic": upright = "Regular" return CSS_WEIGHTS[upright] def css_style(weight: str) -> str: return "italic" if weight.endswith("Italic") else "normal" @dataclass(frozen=True) class NpmSource: package: str version: str @property def label(self) -> str: return f"{self.package}@{self.version}" def provenance(self) -> dict[str, str]: return {"type": "npm", "package": self.package, "version": self.version} def fetch(self, cache: Path) -> Path: root = cache / f"npm-{self.package.replace('/', '-').lstrip('@')}-{self.version}" marker = root / ".complete" meta_url = f"https://registry.npmjs.org/{self.package}/{self.version}" if not marker.exists(): meta = json.loads(_http_get(meta_url)) tarball = _http_get(meta["dist"]["tarball"]) _check_integrity(meta["dist"]["integrity"], tarball, self.label) _extract_tar(tarball, root) marker.write_text(hashlib.sha256(tarball).hexdigest() + "\n") return root / "package" def artifact_sha256(self, cache: Path) -> str: root = cache / f"npm-{self.package.replace('/', '-').lstrip('@')}-{self.version}" return (root / ".complete").read_text().strip() @dataclass(frozen=True) class GithubReleaseSource: repo: str tag: str asset: str @property def label(self) -> str: return f"{self.repo}@{self.tag}:{self.asset}" def provenance(self) -> dict[str, str]: return { "type": "github-release", "repo": self.repo, "tag": self.tag, "asset": self.asset, } def _root(self, cache: Path) -> Path: slug = re.sub(r"[^A-Za-z0-9]+", "-", f"{self.repo}-{self.tag}").strip("-") return cache / f"gh-{slug}" def fetch(self, cache: Path) -> Path: root = self._root(cache) marker = root / ".complete" if not marker.exists(): url = ( f"https://github.com/{self.repo}/releases/download/" f"{urllib.parse.quote(self.tag, safe='/')}/{self.asset}" ) blob = _http_get(url) _extract_zip(blob, root) marker.write_text(hashlib.sha256(blob).hexdigest() + "\n") return root def artifact_sha256(self, cache: Path) -> str: return (self._root(cache) / ".complete").read_text().strip() Source = Union[NpmSource, GithubReleaseSource] SOURCES: dict[str, Source] = { "plex-sans": NpmSource("@ibm/plex-sans", "1.1.0"), "plex-mono": NpmSource("@ibm/plex-mono", "2.5.0"), "plex-sans-arabic": NpmSource("@ibm/plex-sans-arabic", "1.1.0"), "plex-sans-devanagari": NpmSource("@ibm/plex-sans-devanagari", "1.1.0"), "plex-sans-hebrew": NpmSource("@ibm/plex-sans-hebrew", "1.1.0"), "plex-sans-thai": NpmSource("@ibm/plex-sans-thai", "1.1.0"), "plex-sans-thai-looped": NpmSource("@ibm/plex-sans-thai-looped", "1.1.0"), "plex-sans-jp": NpmSource("@ibm/plex-sans-jp", "3.0.0"), "plex-sans-sc": NpmSource("@ibm/plex-sans-sc", "1.1.0"), "plex-sans-tc": NpmSource("@ibm/plex-sans-tc", "1.1.1"), "plex-sans-kr": GithubReleaseSource( "IBM/plex", "@ibm/plex-sans-kr@1.1.0", "ibm-plex-sans-kr.zip" ), } REBUILD = "rebuild" SUBSET = "subset" @dataclass(frozen=True) class Family: out_dir: str prefix: str weights: tuple[str, ...] source: str src_subdir: str mode: str gasp: dict[int, int] css_family: str stylesheet: str def source_files(self) -> list[str]: return [f"{self.prefix}-{w}.woff2" for w in self.weights] LATIN_WEIGHTS = UPRIGHT_WEIGHTS + ITALIC_WEIGHTS PLEX_WOFF2 = "fonts/complete/woff2" LATIN_CORE_STYLESHEETS = ("fluxer-sans.css", "fluxer-mono.css") ALWAYS_BUNDLED_LATIN_FAMILIES: tuple[str, ...] = ("Fluxer Sans", "Fluxer Mono") FAMILIES: tuple[Family, ...] = ( Family( "FluxerSans", "IBMPlexSans", LATIN_WEIGHTS, "plex-sans", PLEX_WOFF2, REBUILD, PLEX_GASP, "Fluxer Sans", "fluxer-sans.css", ), Family( "FluxerMono", "IBMPlexMono", LATIN_WEIGHTS, "plex-mono", PLEX_WOFF2, REBUILD, PLEX_GASP, "Fluxer Mono", "fluxer-mono.css", ), Family( "FluxerSansArabic", "IBMPlexSansArabic", UPRIGHT_WEIGHTS, "plex-sans-arabic", PLEX_WOFF2, REBUILD, PLEX_GASP, "Fluxer Sans Arabic", "script-arabic.css", ), Family( "FluxerSansDevanagari", "IBMPlexSansDevanagari", UPRIGHT_WEIGHTS, "plex-sans-devanagari", PLEX_WOFF2, REBUILD, PLEX_GASP, "Fluxer Sans Devanagari", "script-devanagari.css", ), Family( "FluxerSansHebrew", "IBMPlexSansHebrew", UPRIGHT_WEIGHTS, "plex-sans-hebrew", PLEX_WOFF2, REBUILD, PLEX_GASP, "Fluxer Sans Hebrew", "script-hebrew.css", ), Family( "FluxerSansThai", "IBMPlexSansThai", UPRIGHT_WEIGHTS, "plex-sans-thai", PLEX_WOFF2, REBUILD, PLEX_GASP, "Fluxer Sans Thai", "script-thai.css", ), Family( "FluxerSansThaiLooped", "IBMPlexSansThaiLooped", UPRIGHT_WEIGHTS, "plex-sans-thai-looped", PLEX_WOFF2, REBUILD, PLEX_GASP, "Fluxer Sans Thai Looped", "script-thai-looped.css", ), Family( "FluxerSansJP", "IBMPlexSansJP", UPRIGHT_WEIGHTS, "plex-sans-jp", f"{PLEX_WOFF2}/hinted", SUBSET, CJK_GASP, "Fluxer Sans JP", "script-jp.css", ), Family( "FluxerSansSC", "IBMPlexSansSC", UPRIGHT_WEIGHTS, "plex-sans-sc", f"{PLEX_WOFF2}/hinted", SUBSET, CJK_GASP, "Fluxer Sans SC", "script-sc.css", ), Family( "FluxerSansTC", "IBMPlexSansTC", UPRIGHT_WEIGHTS, "plex-sans-tc", f"{PLEX_WOFF2}/hinted", SUBSET, CJK_GASP, "Fluxer Sans TC", "script-tc.css", ), Family( "FluxerSansKR", "IBMPlexSansKR", UPRIGHT_WEIGHTS, "plex-sans-kr", "ibm-plex-sans-kr/fonts/complete/woff2/unhinted", SUBSET, PLEX_GASP, "Fluxer Sans KR", "script-kr.css", ), ) FAMILY_BY_DIR = {family.out_dir: family for family in FAMILIES} KANA_OWNED_BY_JP = frozenset(range(0x3040, 0x3100)) WITHHELD_CODEPOINTS: dict[str, frozenset[int]] = { "FluxerSansSC": KANA_OWNED_BY_JP, "FluxerSansTC": KANA_OWNED_BY_JP, } SOURCE_EXPECTATIONS: dict[str, dict[str, object]] = { "FluxerSansKR": { "sfntVersion": "\x00\x01\x00\x00", "tables": {"fpgm": 682, "prep": 329, "cvt ": 46}, "numGlyphs": 12283, "reference_file": "IBMPlexSansKR-Regular.woff2", "reference_glyf": 2161193, }, } ROOT_VARIABLES_TEMPLATE = """:root {{ \t--font-sans: \t\t{sans}; \t--font-mono: 'Fluxer Mono', 'Menlo', 'Monaco', 'Courier New', monospace; }} """ LOCALE_FALLBACKS_COMMENT = "" SCRIPT_FAMILY_ORDER: tuple[str, ...] = ( "Fluxer Sans Arabic", "Fluxer Sans Hebrew", "Fluxer Sans Devanagari", "Fluxer Sans Thai Looped", "Fluxer Sans SC", "Fluxer Sans TC", "Fluxer Sans JP", "Fluxer Sans KR", ) GENERIC_UI_FACES: tuple[str, ...] = ("system-ui", "-apple-system", "BlinkMacSystemFont") UNCHAINED_SCRIPT_FAMILIES: tuple[str, ...] = ("Fluxer Sans Thai",) LOCALE_FALLBACKS: tuple[tuple[tuple[str, ...], str, tuple[str, ...]], ...] = ( ( ("ja",), "Fluxer Sans JP", ("Hiragino Sans", "Hiragino Kaku Gothic ProN", "Yu Gothic", "Meiryo"), ), (("ko",), "Fluxer Sans KR", ("Malgun Gothic", "Apple SD Gothic Neo")), (("zh-CN", "zh-Hans"), "Fluxer Sans SC", ("PingFang SC", "Microsoft YaHei")), (("zh-Hant-TW", "zh-TW"), "Fluxer Sans TC", ("PingFang TC", "Microsoft JhengHei")), (("zh-HK", "zh-Hant-HK"), "Fluxer Sans TC", ("PingFang HK", "Microsoft JhengHei")), (("ar",), "Fluxer Sans Arabic", ("Segoe UI", "Geeza Pro", "Arabic Typesetting")), (("he",), "Fluxer Sans Hebrew", ("Segoe UI", "Arial Hebrew")), (("hi", "mr", "ne", "sa"), "Fluxer Sans Devanagari", ("Nirmala UI", "Noto Sans Devanagari")), (("th",), "Fluxer Sans Thai Looped", ("Leelawadee UI", "Thonburi")), ) def font_stack(preferred: str | None, os_faces: tuple[str, ...]) -> list[str]: scripts = [preferred] if preferred else [] scripts += [family for family in SCRIPT_FAMILY_ORDER if family != preferred] return [ quote_family("Fluxer Sans"), *(quote_family(family) for family in scripts), *GENERIC_UI_FACES, *(quote_family(face) for face in os_faces), "sans-serif", ] def quote_family(name: str) -> str: return f"'{name}'" def wrap_font_stack(entries: list[str], indent: str, width: int = 118) -> str: lines: list[str] = [] current = "" for index, entry in enumerate(entries): piece = entry if index == len(entries) - 1 else f"{entry}," candidate = f"{current} {piece}" if current else piece if current and len(indent) + len(candidate) > width: lines.append(current) current = piece else: current = candidate if current: lines.append(current) return f"\n{indent}".join(lines) BASE_UI_FACES: tuple[str, ...] = ("Segoe UI", "Roboto") def render_variables_stylesheet() -> str: stack = wrap_font_stack(font_stack(None, BASE_UI_FACES), "\t\t") body = ROOT_VARIABLES_TEMPLATE.format(sans=stack) return "\n".join([SPDX_CSS_HEADER, GENERATED_CSS_HEADER, "", body]) def render_locale_fallbacks_stylesheet() -> str: lines = [SPDX_CSS_HEADER, GENERATED_CSS_HEADER, "", LOCALE_FALLBACKS_COMMENT] for langs, preferred, os_faces in LOCALE_FALLBACKS: selector = ",\n".join(f":root:lang({lang})" for lang in langs) stack = wrap_font_stack(font_stack(preferred, os_faces), "\t\t") lines.append(f"{selector} {{\n\t--font-sans:\n\t\t{stack};\n}}") return "\n".join([*lines, ""]) def _http_get(url: str) -> bytes: request = urllib.request.Request(url, headers={"User-Agent": "fluxer-font-build"}) with urllib.request.urlopen(request) as response: # noqa: S310 - pinned https URLs return response.read() def _check_integrity(integrity: str, blob: bytes, label: str) -> None: algo, _, encoded = integrity.partition("-") actual = base64.b64encode(hashlib.new(algo, blob).digest()).decode() if actual != encoded: raise SystemExit(f"integrity mismatch for {label}: expected {encoded}, got {actual}") def _extract_tar(blob: bytes, root: Path) -> None: if root.exists(): shutil.rmtree(root) root.mkdir(parents=True) with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as archive: for member in archive.getmembers(): _extract_member(archive, member, member.name, root, member.isdir()) def _extract_zip(blob: bytes, root: Path) -> None: if root.exists(): shutil.rmtree(root) root.mkdir(parents=True) with zipfile.ZipFile(io.BytesIO(blob)) as archive: for info in archive.infolist(): _extract_member(archive, info, info.filename, root, info.is_dir()) def _extract_member(archive: object, member: object, name: str, root: Path, is_dir: bool) -> None: dest = (root / name).resolve() if not str(dest).startswith(str(root.resolve())): raise SystemExit(f"refusing to extract outside the cache: {name}") if is_dir: dest.mkdir(parents=True, exist_ok=True) return dest.parent.mkdir(parents=True, exist_ok=True) reader = ( archive.extractfile(member) if isinstance(archive, tarfile.TarFile) else archive.open(member) ) if reader is None: return with reader: dest.write_bytes(reader.read()) def fluxer_name(value: str) -> str: return value.replace("IBM Plex ", "Fluxer ").replace("IBMPlex", "Fluxer") def _name_map(font: TTFont) -> dict[tuple[int, int, int, int], str]: return { (r.nameID, r.platformID, r.platEncID, r.langID): r.toUnicode() for r in font["name"].names } def build_font(src: Path, gasp: dict[int, int]) -> bytes: font = TTFont(src, lazy=True, recalcTimestamp=False, recalcBBoxes=False) font["gasp"].gaspRange = dict(gasp) for record in font["name"].names: if record.nameID in RENAME_NAME_IDS: record.string = fluxer_name(record.toUnicode()) font.flavor = "woff2" buffer = io.BytesIO() font.save(buffer) font.close() return buffer.getvalue() def check_rebuild(src: Path, produced: bytes, gasp: dict[int, int]) -> None: upstream = TTFont(src, lazy=True) built = TTFont(io.BytesIO(produced), lazy=True) try: up_tags = {t for t in upstream.keys() if t != "GlyphOrder"} new_tags = {t for t in built.keys() if t != "GlyphOrder"} if up_tags != new_tags: raise SystemExit( f"{src.name}: table set changed: " f"+{sorted(new_tags - up_tags)} -{sorted(up_tags - new_tags)}" ) for tag in sorted(up_tags - MUTABLE_TABLES): if upstream.reader[tag] != built.reader[tag]: raise SystemExit(f"{src.name}: table {tag!r} changed but must not") for tag in CRITICAL_TABLES: if tag in up_tags and upstream.reader[tag] != built.reader[tag]: raise SystemExit(f"{src.name}: critical table {tag!r} changed") up_head = upstream.reader["head"] new_head = built.reader["head"] if len(up_head) != len(new_head): raise SystemExit(f"{src.name}: head length changed") if ( up_head[: HEAD_CHECKSUM_SLICE.start] != new_head[: HEAD_CHECKSUM_SLICE.start] or up_head[HEAD_CHECKSUM_SLICE.stop :] != new_head[HEAD_CHECKSUM_SLICE.stop :] ): raise SystemExit(f"{src.name}: head changed outside checkSumAdjustment") if built["gasp"].gaspRange != gasp: raise SystemExit(f"{src.name}: gasp is {built['gasp'].gaspRange}, expected {gasp}") if built["gasp"].version != 1: raise SystemExit(f"{src.name}: gasp version is {built['gasp'].version}, expected 1") up_names = _name_map(upstream) new_names = _name_map(built) if set(up_names) != set(new_names): raise SystemExit(f"{src.name}: name record set changed") for key, value in up_names.items(): expected = fluxer_name(value) if key[0] in RENAME_NAME_IDS else value if new_names[key] != expected: raise SystemExit( f"{src.name}: name {key} is {new_names[key]!r}, expected {expected!r}" ) check_no_reserved_name(built, src.name) finally: upstream.close() built.close() def check_no_reserved_name(font: TTFont, label: str) -> None: for record in font["name"].names: if record.nameID in NO_PLEX_NAME_IDS and "Plex" in record.toUnicode(): raise SystemExit( f"{label}: reserved font name 'Plex' still present in name ID {record.nameID}" ) def subset_options() -> SubsetOptions: options = SubsetOptions() options.hinting = True # keep glyph instructions; fpgm/prep/cvt pass through options.legacy_kern = True options.layout_features = ["*"] # CJK needs vert/vrt2/halt/palt/trad/smpl options.layout_scripts = ["*"] options.name_IDs = ["*"] # the OFL rename lives in the name table options.name_legacy = True options.name_languages = ["*"] options.notdef_outline = True options.glyph_names = False # post 3.0; names are not rendering data options.recalc_bounds = False # keep upstream head/glyf bounding boxes options.recalc_timestamp = False # keep upstream head.modified options.recalc_max_context = False options.recalc_average_width = False # keep OS/2.xAvgCharWidth options.prune_unicode_ranges = False # keep OS/2.ulUnicodeRange options.prune_codepage_ranges = False options.drop_tables = [] options.passthrough_tables = True # e.g. `meta`, which the subsetter cannot slice options.ignore_missing_unicodes = True return options def subset_face(sfnt: bytes, codepoints: tuple[int, ...]) -> bytes: font = TTFont(io.BytesIO(sfnt), lazy=True, recalcTimestamp=False, recalcBBoxes=False) subsetter = Subsetter(options=subset_options()) subsetter.populate(unicodes=codepoints) subsetter.subset(font) font.flavor = "woff2" buffer = io.BytesIO() font.save(buffer) font.close() return buffer.getvalue() def glyph_signature(glyph_set: object, glyf: object, name: str) -> tuple: pen = DecomposingRecordingPen(glyph_set) glyph_set[name].draw(pen) glyph = glyf[name] program = getattr(glyph, "program", None) bytecode = bytes(program.getBytecode()) if program is not None else b"" box = (glyph.xMin, glyph.yMin, glyph.xMax, glyph.yMax) if glyph.numberOfContours != 0 else () return (tuple(pen.value), box, bytecode) def metric_signature(font: TTFont) -> dict[str, int]: values: dict[str, int] = {} for table, field_name in METRIC_FIELDS: if table not in font: continue value = getattr(font[table], field_name, None) if value is None: continue values[f"{table.strip()}.{field_name}"] = int(value) return values def check_subset( parent: TTFont, chunk_bytes: bytes, requested: tuple[int, ...], label: str ) -> dict: chunk = TTFont(io.BytesIO(chunk_bytes), lazy=True) try: parent_tags = {t for t in parent.keys() if t != "GlyphOrder"} chunk_tags = {t for t in chunk.keys() if t != "GlyphOrder"} if parent_tags != chunk_tags: raise SystemExit( f"{label}: table set changed: " f"+{sorted(chunk_tags - parent_tags)} -{sorted(parent_tags - chunk_tags)}" ) for tag in HINTING_TABLES + ("gasp",): if parent.reader[tag] != chunk.reader[tag]: raise SystemExit(f"{label}: table {tag!r} must pass through unchanged") if metric_signature(parent) != metric_signature(chunk): raise SystemExit(f"{label}: vertical metrics changed") check_no_reserved_name(chunk, label) if not (chunk["name"].getDebugName(6) or "").startswith("Fluxer"): raise SystemExit(f"{label}: PostScript name is not a Fluxer name") chunk_cmap = chunk.getBestCmap() expected = {cp for cp in requested if cp in parent.getBestCmap()} if set(chunk_cmap) != expected: missing = sorted(expected - set(chunk_cmap))[:4] extra = sorted(set(chunk_cmap) - expected)[:4] raise SystemExit(f"{label}: cmap drifted from the plan (-{missing} +{extra})") parent_cmap = parent.getBestCmap() parent_glyf = parent["glyf"] chunk_glyf = chunk["glyf"] parent_set = parent.getGlyphSet() chunk_set = chunk.getGlyphSet() parent_hmtx = parent["hmtx"] chunk_hmtx = chunk["hmtx"] parent_vmtx = parent["vmtx"] if "vmtx" in parent else None chunk_vmtx = chunk["vmtx"] if "vmtx" in chunk else None for cp in sorted(expected): name = parent_cmap[cp] chunk_name = chunk_cmap[cp] if glyph_signature(parent_set, parent_glyf, name) != glyph_signature( chunk_set, chunk_glyf, chunk_name ): raise SystemExit(f"{label}: outline changed for U+{cp:04X} ({name})") if parent_hmtx[name] != chunk_hmtx[chunk_name]: raise SystemExit(f"{label}: horizontal metrics changed for U+{cp:04X} ({name})") if parent_vmtx is not None and parent_vmtx[name] != chunk_vmtx[chunk_name]: raise SystemExit(f"{label}: vertical metrics changed for U+{cp:04X} ({name})") return { "codepoints": sorted(chunk_cmap), "hinting": {tag: len(chunk.reader[tag]) for tag in HINTING_TABLES}, "metrics": metric_signature(chunk), } finally: chunk.close() def plan_chunks(codepoints: set[int], target: int) -> list[tuple[int, ...]]: if target < 1: raise SystemExit(f"chunk target must be positive, got {target}") by_block: dict[str, list[int]] = {} for cp in sorted(codepoints): by_block.setdefault(ftunicodedata.block(cp), []).append(cp) chunks: list[tuple[int, ...]] = [] pending: list[int] = [] def flush() -> None: if pending: chunks.append(tuple(pending)) pending.clear() for _, members in sorted(by_block.items(), key=lambda item: item[1][0]): if len(members) < MIN_STANDALONE_BLOCK: pending.extend(members) if len(pending) >= target: flush() continue flush() parts = max(1, round(len(members) / target)) if len(members) > target else 1 size = -(-len(members) // parts) for start in range(0, len(members), size): chunks.append(tuple(members[start : start + size])) flush() if not chunks: raise SystemExit("chunk plan is empty") return chunks TRANSFER_SAMPLES: dict[str, dict[str, str]] = { "FluxerSansJP": { "mixed script": "語", "display name": "田中太郎", "sentence": "これは日本語のサンプルテキストです", }, "FluxerSansSC": { "mixed script": "文", "display name": "小明", "sentence": "这是简体中文的示例文本", }, "FluxerSansTC": { "mixed script": "文", "display name": "小明", "sentence": "這是繁體中文的示例文本", }, "FluxerSansKR": { "mixed script": "한", "display name": "김민준", "sentence": "이것은 한국어 샘플 텍스트입니다", }, } SLICING_DECISION: dict[str, dict[str, float]] = { "FluxerSansJP": {"minMixedScriptRatio": 20.0, "maxFullCoverageRatio": 1.25}, "FluxerSansSC": {"minMixedScriptRatio": 20.0, "maxFullCoverageRatio": 1.30}, "FluxerSansTC": {"minMixedScriptRatio": 20.0, "maxFullCoverageRatio": 1.50}, "FluxerSansKR": {"minMixedScriptRatio": 2.0, "maxFullCoverageRatio": 1.25}, } def transfer_table(chunks: list[tuple[int, set[int]]], unsliced: int, family_dir: str) -> dict: scenarios = {} for label, text in TRANSFER_SAMPLES[family_dir].items(): wanted = {ord(c) for c in text} hit = [(size, covered) for size, covered in chunks if covered & wanted] fetched = sum(size for size, _ in hit) scenarios[label] = { "bytes": fetched, "chunks": len(hit), "ratio": round(unsliced / fetched, 3) if fetched else None, } full = sum(size for size, _ in chunks) scenarios["full coverage"] = { "bytes": full, "chunks": len(chunks), "ratio": round(unsliced / full, 3) if full else None, } return {"unslicedBytes": unsliced, "scenarios": scenarios} def check_slicing_decision(family_dir: str, transfer: dict) -> list[str]: bars = SLICING_DECISION.get(family_dir) if bars is None: return [] failures = [] mixed = transfer["scenarios"]["mixed script"]["ratio"] if mixed is None or mixed < bars["minMixedScriptRatio"]: failures.append( f"{family_dir}: slicing wins only {mixed}x on the mixed-script case, below the " f"{bars['minMixedScriptRatio']}x bar that justifies it." ) full_ratio = transfer["scenarios"]["full coverage"]["ratio"] regression = 1 / full_ratio if full_ratio else float("inf") if regression > bars["maxFullCoverageRatio"]: failures.append( f"{family_dir}: full coverage now costs {regression:.3f}x the unsliced face, past the " f"{bars['maxFullCoverageRatio']}x bar" ) return failures def codepoint_ranges(codepoints: list[int]) -> list[tuple[int, int]]: ranges: list[tuple[int, int]] = [] for cp in codepoints: if ranges and cp == ranges[-1][1] + 1: ranges[-1] = (ranges[-1][0], cp) else: ranges.append((cp, cp)) return ranges def format_unicode_range(codepoints: list[int]) -> str: parts = [] for start, end in codepoint_ranges(sorted(codepoints)): parts.append(f"U+{start:04X}" if start == end else f"U+{start:04X}-{end:04X}") return ", ".join(parts) def parse_unicode_range(value: str) -> set[int]: result: set[int] = set() for token in value.split(","): token = token.strip() if not token: continue match = re.fullmatch(r"[Uu]\+([0-9A-Fa-f]+)(?:-([0-9A-Fa-f]+))?", token) if not match: raise ValueError(f"unparseable unicode-range token {token!r}") start = int(match.group(1), 16) end = int(match.group(2), 16) if match.group(2) else start result.update(range(start, end + 1)) return result def calibrate_chunk_size(sfnt: bytes, codepoints: list[int], target_bytes: int) -> tuple[int, str]: small, large = 300, 900 if len(codepoints) <= large: return len(codepoints), "single chunk (family smaller than one probe)" middle = len(codepoints) // 2 probe_small = tuple(codepoints[middle : middle + small]) probe_large = tuple(codepoints[middle : middle + large]) size_small = len(subset_face(sfnt, probe_small)) size_large = len(subset_face(sfnt, probe_large)) rate = (size_large - size_small) / (large - small) overhead = size_small - rate * small if rate <= 0: raise SystemExit("calibration produced a non-positive bytes-per-codepoint rate") budget = int((target_bytes - overhead) / rate) budget = max(MIN_STANDALONE_BLOCK, budget) note = ( f"{rate:.1f} B/codepoint marginal, {overhead / 1024:.1f} KiB intercept " f"-> {budget} codepoints per chunk" ) return budget, note def sha256_bytes(blob: bytes) -> str: return hashlib.sha256(blob).hexdigest() @dataclass class FaceResult: family: str weight: str source_sha256: str hinting: dict[str, int] metrics: dict[str, int] files: list[dict] = field(default_factory=list) def check_source_expectations(family: Family, src_dir: Path) -> None: expectations = SOURCE_EXPECTATIONS.get(family.out_dir) if expectations is None: return path = src_dir / str(expectations["reference_file"]) font = TTFont(path, lazy=True) try: if font.sfntVersion != expectations["sfntVersion"]: raise SystemExit( f"{path.name}: sfntVersion {font.sfntVersion!r}, " f"expected {expectations['sfntVersion']!r} (TrueType outlines)" ) for tag, size in expectations["tables"].items(): # type: ignore[union-attr] if tag not in font: raise SystemExit(f"{path.name}: hinting table {tag!r} is missing upstream") if len(font.reader[tag]) != size: raise SystemExit( f"{path.name}: {tag!r} is {len(font.reader[tag])} B upstream, expected {size} B" ) if font["maxp"].numGlyphs != expectations["numGlyphs"]: raise SystemExit( f"{path.name}: numGlyphs {font['maxp'].numGlyphs}, expected {expectations['numGlyphs']}" ) if "glyf" in font and len(font.reader["glyf"]) != expectations["reference_glyf"]: raise SystemExit( f"{path.name}: glyf is {len(font.reader['glyf'])} B, " f"expected {expectations['reference_glyf']} B" ) finally: font.close() def face_cmap(path: Path) -> set[int]: font = TTFont(path, lazy=True) try: return set(font.getBestCmap()) finally: font.close() def renamed_sfnt(src: Path, gasp: dict[int, int]) -> bytes: produced = build_font(src, gasp) check_rebuild(src, produced, gasp) font = TTFont(io.BytesIO(produced), lazy=True, recalcTimestamp=False, recalcBBoxes=False) font.flavor = None buffer = io.BytesIO() font.save(buffer) font.close() return buffer.getvalue() def build_face(job: dict) -> FaceResult: family = FAMILY_BY_DIR[job["family"]] src = Path(job["src"]) weight = job["weight"] source_bytes = src.read_bytes() result = FaceResult( family=family.out_dir, weight=weight, source_sha256=sha256_bytes(source_bytes), hinting={}, metrics={}, ) stage = Path(job["stage"]) / family.out_dir stage.mkdir(parents=True, exist_ok=True) if family.mode == REBUILD: produced = build_font(src, family.gasp) check_rebuild(src, produced, family.gasp) font = TTFont(io.BytesIO(produced), lazy=True) try: result.hinting = {tag: len(font.reader[tag]) for tag in HINTING_TABLES} result.metrics = metric_signature(font) covered = sorted(font.getBestCmap()) finally: font.close() name = f"{family.out_dir}-{weight}.woff2" (stage / name).write_bytes(produced) result.files.append( { "path": f"{family.out_dir}/{name}", "sha256": sha256_bytes(produced), "bytes": len(produced), "codepoints": len(covered), "unicodeRange": None if family.stylesheet in LATIN_CORE_STYLESHEETS else format_unicode_range(covered), "chunk": None, } ) return result sfnt = renamed_sfnt(src, family.gasp) parent = TTFont(io.BytesIO(sfnt), lazy=True) try: result.hinting = {tag: len(parent.reader[tag]) for tag in HINTING_TABLES} result.metrics = metric_signature(parent) parent_cmap = set(parent.getBestCmap()) for index, planned in enumerate(job["plan"]): requested = tuple(cp for cp in planned if cp in parent_cmap) if not requested: continue name = f"{family.out_dir}-{weight}.{index:03d}.woff2" label = f"{family.out_dir}/{name}" produced = subset_face(sfnt, requested) info = check_subset(parent, produced, requested, label) if info["hinting"] != result.hinting: raise SystemExit(f"{label}: hinting table sizes drifted from the parent face") (stage / name).write_bytes(produced) result.files.append( { "path": label, "sha256": sha256_bytes(produced), "bytes": len(produced), "codepoints": len(info["codepoints"]), "unicodeRange": format_unicode_range(info["codepoints"]), "chunk": index, } ) finally: parent.close() return result def resolve_sources(cache: Path, families: tuple[Family, ...]) -> dict[str, Path]: roots: dict[str, Path] = {} for key in dict.fromkeys(f.source for f in families): source = SOURCES[key] print(f" fetching {source.label}", flush=True) roots[key] = source.fetch(cache) return roots def render_face_rule(family: Family, entry: dict) -> str: weight = entry["weight"] lines = [ "@font-face {", f"\tfont-family: '{family.css_family}';", f"\tsrc: url('../files/{entry['path']}') format('woff2');", f"\tfont-weight: {css_weight(weight)};", f"\tfont-style: {css_style(weight)};", "\tfont-display: swap;", ] if entry["unicodeRange"]: lines.append(f"\tunicode-range: {entry['unicodeRange']};") lines.append("}") return "\n".join(lines) def chunk_major(family: Family, entries: list[dict]) -> list[dict]: order = {weight: index for index, weight in enumerate(family.weights)} return sorted( entries, key=lambda entry: ( entry["chunk"] if entry["chunk"] is not None else -1, order[entry["weight"]], ), ) def render_stylesheets(entries: list[dict]) -> dict[str, str]: by_family: dict[str, list[dict]] = {} for entry in entries: by_family.setdefault(entry["family"], []).append(entry) by_stylesheet: dict[str, list[str]] = {} for family in FAMILIES: for entry in chunk_major(family, by_family[family.out_dir]): by_stylesheet.setdefault(family.stylesheet, []).append(render_face_rule(family, entry)) sheets: dict[str, str] = {} for name, rules in by_stylesheet.items(): sheets[name] = "\n".join([SPDX_CSS_HEADER, GENERATED_CSS_HEADER, "", *rules, ""]) sheets["variables.css"] = render_variables_stylesheet() sheets["locale-fallbacks.css"] = render_locale_fallbacks_stylesheet() return sheets def render_manifest(entries: list[dict], chunking: dict) -> str: families = [] for family in FAMILIES: faces = [ { "file": entry["path"], "weight": css_weight(entry["weight"]), "style": css_style(entry["weight"]), "unicodeRange": entry["unicodeRange"], "bytes": entry["bytes"], } for entry in chunk_major(family, [e for e in entries if e["family"] == family.out_dir]) ] families.append( { "id": family.out_dir, "cssFamily": family.css_family, "stylesheet": f"css/{family.stylesheet}", "mode": family.mode, "latinCore": family.stylesheet in LATIN_CORE_STYLESHEETS, "chunks": chunking.get(family.out_dir, {}).get("chunks", 1), "faces": faces, } ) payload = { "_comment": ( "Generated by tools/fonts/build_fonts.py. Do not edit by hand. " "Paths are relative to packages/fonts/." ), "entrypoints": { "latinCore": { family.css_family: f"css/{family.stylesheet}" for family in FAMILIES if family.stylesheet in LATIN_CORE_STYLESHEETS }, "variables": "css/variables.css", "localeFallbacks": "css/locale-fallbacks.css", "scripts": { family.css_family: f"css/{family.stylesheet}" for family in FAMILIES if family.stylesheet not in LATIN_CORE_STYLESHEETS }, }, "families": families, } return json.dumps(payload, indent="\t", sort_keys=False) + "\n" def size_distribution(sizes: list[int]) -> str: if not sizes: return "no files" ordered = sorted(sizes) return ( f"min {ordered[0] / 1024:.0f} KiB, median {statistics.median(ordered) / 1024:.0f} KiB, " f"p95 {ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))] / 1024:.0f} KiB, " f"max {ordered[-1] / 1024:.0f} KiB" ) def build(cache: Path, dry_run: bool, workers: int) -> int: print(f"cache: {cache}") cache.mkdir(parents=True, exist_ok=True) roots = resolve_sources(cache, FAMILIES) stage = cache / "stage" if stage.exists(): shutil.rmtree(stage) stage.mkdir(parents=True) print("\nplanning chunks") plans: dict[str, list[tuple[int, ...]]] = {} chunking: dict[str, dict] = {} family_codepoints: dict[str, int] = {} for family in FAMILIES: src_dir = roots[family.source] / family.src_subdir for name in family.source_files(): if not (src_dir / name).is_file(): raise SystemExit(f"missing upstream file: {src_dir / name}") check_source_expectations(family, src_dir) union: set[int] = set() for name in family.source_files(): union |= face_cmap(src_dir / name) union -= WITHHELD_CODEPOINTS.get(family.out_dir, frozenset()) family_codepoints[family.out_dir] = len(union) if family.mode != SUBSET: continue reference = src_dir / f"{family.prefix}-Regular.woff2" sfnt = renamed_sfnt(reference, family.gasp) budget, note = calibrate_chunk_size(sfnt, sorted(union), CHUNK_TARGET_BYTES) plan = plan_chunks(union, budget) plans[family.out_dir] = plan chunking[family.out_dir] = { "targetBytes": CHUNK_TARGET_BYTES, "codepointsPerChunk": budget, "chunks": len(plan), "codepoints": len(union), } print(f" {family.out_dir}: {len(union)} codepoints, {note}, {len(plan)} chunks") jobs = [ { "family": family.out_dir, "weight": weight, "src": str( roots[family.source] / family.src_subdir / f"{family.prefix}-{weight}.woff2" ), "stage": str(stage), "plan": plans.get(family.out_dir, []), } for family in FAMILIES for weight in family.weights ] print(f"\nbuilding {len(jobs)} faces on {workers} worker(s)") results: list[FaceResult] = [] if workers <= 1: for job in jobs: results.append(build_face(job)) print(f" {job['family']}-{job['weight']}", flush=True) else: with ProcessPoolExecutor(max_workers=workers) as pool: for result in pool.map(build_face, jobs): results.append(result) print(f" {result.family}-{result.weight}: {len(result.files)} file(s)", flush=True) order = { (family.out_dir, weight): (i, j) for i, family in enumerate(FAMILIES) for j, weight in enumerate(family.weights) } results.sort(key=lambda r: order[(r.family, r.weight)]) entries: list[dict] = [] for result in results: for item in result.files: entries.append({**item, "family": result.family, "weight": result.weight}) sheets = render_stylesheets(entries) manifest = render_manifest(entries, chunking) lock: dict[str, object] = { "_comment": ( "Generated by tools/fonts/build_fonts.py. Do not edit by hand. " "Run `python3 tools/fonts/build_fonts.py --verify` to check the tree against it." ), "gasp": { "plex": {str(k): v for k, v in PLEX_GASP.items()}, "cjk": {str(k): v for k, v in CJK_GASP.items()}, }, "chunking": {"targetBytes": CHUNK_TARGET_BYTES, "families": chunking}, "sources": {}, "families": {}, "stylesheets": {}, "files": {}, } for key in dict.fromkeys(f.source for f in FAMILIES): source = SOURCES[key] lock["sources"][key] = {**source.provenance(), "sha256": source.artifact_sha256(cache)} for family in FAMILIES: faces = {} for result in results: if result.family != family.out_dir: continue faces[result.weight] = { "sourceFile": f"{family.prefix}-{result.weight}.woff2", "sourceSha256": result.source_sha256, "hinting": result.hinting, "metrics": result.metrics, "files": len(result.files), "codepoints": sum(item["codepoints"] for item in result.files), } lock["families"][family.out_dir] = { "mode": family.mode, "source": family.source, "srcSubdir": family.src_subdir, "cssFamily": family.css_family, "stylesheet": f"css/{family.stylesheet}", "latinCore": family.stylesheet in LATIN_CORE_STYLESHEETS, "gasp": {str(k): v for k, v in family.gasp.items()}, "codepoints": family_codepoints[family.out_dir], "faces": faces, } if family.out_dir in TRANSFER_SAMPLES: unsliced = ( (roots[family.source] / family.src_subdir / f"{family.prefix}-Regular.woff2") .stat() .st_size ) regular = [ (item["bytes"], parse_unicode_range(item["unicodeRange"])) for result in results if result.family == family.out_dir and result.weight == "Regular" for item in result.files ] transfer = transfer_table(regular, unsliced, family.out_dir) for failure in check_slicing_decision(family.out_dir, transfer): raise SystemExit(failure) lock["families"][family.out_dir]["transfer"] = transfer for name, text in sorted(sheets.items()): blob = text.encode() lock["stylesheets"][f"css/{name}"] = {"sha256": sha256_bytes(blob), "bytes": len(blob)} manifest_blob = manifest.encode() lock["stylesheets"]["manifest.json"] = { "sha256": sha256_bytes(manifest_blob), "bytes": len(manifest_blob), } for entry in entries: lock["files"][f"files/{entry['path']}"] = { "sha256": entry["sha256"], "bytes": entry["bytes"], } print("\nper-family results") print(f" {'family':24} {'files':>6} {'upstream':>10} {'shipped':>10} distribution") total_before = 0 total_after = 0 for family in FAMILIES: src_dir = roots[family.source] / family.src_subdir before = sum((src_dir / name).stat().st_size for name in family.source_files()) sizes = [entry["bytes"] for entry in entries if entry["family"] == family.out_dir] after = sum(sizes) total_before += before total_after += after print( f" {family.out_dir:24} {len(sizes):>6} {before / 1048576:>9.2f}M {after / 1048576:>9.2f}M " f"{size_distribution(sizes)}" ) print( f" {'TOTAL':24} {len(entries):>6} {total_before / 1048576:>9.2f}M {total_after / 1048576:>9.2f}M" ) changed, unchanged, created, removed = _diff_tree(entries, sheets, manifest) print(f"\nsummary: {created} new, {changed} changed, {unchanged} unchanged, {removed} removed") if dry_run: print("dry run: nothing written") return 0 _write_tree(entries, sheets, manifest, stage) LOCK_PATH.write_text(json.dumps(lock, indent="\t", sort_keys=False) + "\n") print(f"wrote {PACKAGE_ROOT.relative_to(REPO_ROOT)}/ and {LOCK_PATH.relative_to(REPO_ROOT)}") return 0 def _tracked_outputs(sheets: dict[str, str], manifest: str) -> dict[str, bytes]: outputs: dict[str, bytes] = {} for name, text in sheets.items(): outputs[f"css/{name}"] = text.encode() outputs["manifest.json"] = manifest.encode() return outputs def _diff_tree( entries: list[dict], sheets: dict[str, str], manifest: str ) -> tuple[int, int, int, int]: changed = unchanged = created = 0 expected: set[str] = set() for entry in entries: rel = f"files/{entry['path']}" expected.add(rel) dest = PACKAGE_ROOT / rel if not dest.is_file(): created += 1 elif sha256_bytes(dest.read_bytes()) == entry["sha256"]: unchanged += 1 else: changed += 1 for rel, blob in _tracked_outputs(sheets, manifest).items(): expected.add(rel) dest = PACKAGE_ROOT / rel if not dest.is_file(): created += 1 elif dest.read_bytes() == blob: unchanged += 1 else: changed += 1 removed = len(_existing_generated() - expected) return changed, unchanged, created, removed def _existing_generated() -> set[str]: found: set[str] = set() if FILES_DIR.is_dir(): found |= {str(p.relative_to(PACKAGE_ROOT)) for p in FILES_DIR.rglob("*.woff2")} if CSS_DIR.is_dir(): found |= {str(p.relative_to(PACKAGE_ROOT)) for p in CSS_DIR.rglob("*.css")} if MANIFEST_PATH.is_file(): found.add("manifest.json") return found def _write_tree(entries: list[dict], sheets: dict[str, str], manifest: str, stage: Path) -> None: expected: set[str] = set() for entry in entries: rel = f"files/{entry['path']}" expected.add(rel) dest = PACKAGE_ROOT / rel dest.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(stage / entry["path"], dest) for rel, blob in _tracked_outputs(sheets, manifest).items(): expected.add(rel) dest = PACKAGE_ROOT / rel dest.parent.mkdir(parents=True, exist_ok=True) dest.write_bytes(blob) for rel in sorted(_existing_generated() - expected): (PACKAGE_ROOT / rel).unlink() for directory in sorted(FILES_DIR.rglob("*"), reverse=True): if directory.is_dir() and not any(directory.iterdir()): directory.rmdir() FACE_RULE = re.compile( r"@font-face\s*\{(?P[^}]*)\}", re.MULTILINE, ) def parse_stylesheet(text: str) -> list[dict[str, str]]: faces = [] for match in FACE_RULE.finditer(text): body = match.group("body") declarations = {} for declaration in body.split(";"): name, _, value = declaration.partition(":") name = name.strip() if name: declarations[name] = value.strip() url = re.search(r"url\('([^']+)'\)", declarations.get("src", "")) faces.append( { "family": declarations.get("font-family", "").strip("'"), "url": url.group(1) if url else "", "weight": declarations.get("font-weight", ""), "style": declarations.get("font-style", ""), "display": declarations.get("font-display", ""), "unicodeRange": declarations.get("unicode-range", ""), } ) return faces def check_variable_families(text: str) -> list[str]: allowed = set(ALWAYS_BUNDLED_LATIN_FAMILIES) allowed |= {f.css_family for f in FAMILIES if f.stylesheet not in LATIN_CORE_STYLESHEETS} shipped = {f.css_family for f in FAMILIES} failures: list[str] = [] for match in re.finditer(r"(--font-[a-z0-9-]+)\s*:(?P[^;]*);", text): variable = match.group(1) for name in re.findall(r"'([^']+)'", match.group("value")): if name in shipped and name not in allowed: failures.append( f"packages/fonts/css/variables.css: {variable} names {name!r}, but importing " f"variables.css does not bring that family's @font-face rules with it. Declare " f"the variable in the consumer that actually imports its stylesheet instead." ) return failures def verify() -> int: if not LOCK_PATH.is_file(): raise SystemExit(f"missing {LOCK_PATH}; run the build first") lock = json.loads(LOCK_PATH.read_text()) failures: list[str] = [] locked_files = set(lock["files"]) on_disk = ( {f"files/{p.relative_to(FILES_DIR)}" for p in FILES_DIR.rglob("*.woff2")} if FILES_DIR.is_dir() else set() ) for extra in sorted(on_disk - locked_files): failures.append(f"font present in the tree but absent from fonts.lock.json: {extra}") for missing in sorted(locked_files - on_disk): failures.append(f"font recorded in fonts.lock.json but missing from the tree: {missing}") strays = ( sorted(p.relative_to(PACKAGE_ROOT) for p in PACKAGE_ROOT.rglob("*.otf")) if PACKAGE_ROOT.is_dir() else [] ) if strays: failures.append(f"{len(strays)} unreferenced .otf file(s) present, e.g. {strays[0]}") referenced: dict[str, list[str]] = {} declared_ranges: dict[str, str] = {} for rel, recorded in lock["stylesheets"].items(): path = PACKAGE_ROOT / rel if not path.is_file(): failures.append(f"missing generated file: packages/fonts/{rel}") continue blob = path.read_bytes() if sha256_bytes(blob) != recorded["sha256"]: failures.append(f"packages/fonts/{rel}: sha256 does not match fonts.lock.json") if not rel.endswith(".css"): continue text = blob.decode() if "?v=" in text: failures.append(f"packages/fonts/{rel}: contains a ?v= cache-bust token") for face in parse_stylesheet(text): if not face["url"].startswith("../files/"): failures.append( f"packages/fonts/{rel}: url {face['url']!r} is not a relative ../files/ path" ) continue target = f"files/{face['url'][len('../files/') :]}" if not (PACKAGE_ROOT / target).is_file(): failures.append( f"packages/fonts/{rel} references a file that does not exist: {target}" ) referenced.setdefault(target, []).append(rel) if face["display"] != "swap": failures.append( f"packages/fonts/{rel}: {target} declares font-display {face['display']!r}" ) previous = declared_ranges.get(target) if previous is not None and previous != face["unicodeRange"]: failures.append(f"{target}: declared with two different unicode-range values") declared_ranges[target] = face["unicodeRange"] for orphan in sorted(locked_files - set(referenced)): failures.append(f"font is shipped but no stylesheet claims it: {orphan}") for rel, rendered in ( ("css/variables.css", render_variables_stylesheet()), ("css/locale-fallbacks.css", render_locale_fallbacks_stylesheet()), ): path = PACKAGE_ROOT / rel if path.is_file() and path.read_text() != rendered: failures.append(f"packages/fonts/{rel}: does not match what build_fonts.py would emit") variables_css = PACKAGE_ROOT / "css" / "variables.css" if variables_css.is_file(): base_chain = re.search(r"--font-sans:(?P[^;]*);", variables_css.read_text()) if base_chain is None: failures.append("packages/fonts/css/variables.css: no --font-sans declaration") else: value = base_chain.group("value") for family in FAMILIES: if family.stylesheet in LATIN_CORE_STYLESHEETS: continue if family.css_family in UNCHAINED_SCRIPT_FAMILIES: continue if f"'{family.css_family}'" not in value: failures.append( f"packages/fonts/css/variables.css: base --font-sans does not name " f"{family.css_family!r}, so its unicode-range gate can never fire" ) failures.extend(check_variable_families(variables_css.read_text())) if MANIFEST_PATH.is_file(): manifest = json.loads(MANIFEST_PATH.read_text()) manifest_sheets = [manifest["entrypoints"]["variables"]] manifest_sheets.append(manifest["entrypoints"]["localeFallbacks"]) manifest_sheets.extend(manifest["entrypoints"]["latinCore"].values()) manifest_sheets.extend(manifest["entrypoints"]["scripts"].values()) manifest_files: set[str] = set() for entry in manifest["families"]: manifest_sheets.append(entry["stylesheet"]) manifest_files |= {f"files/{face['file']}" for face in entry["faces"]} for sheet in sorted(set(manifest_sheets)): if not (PACKAGE_ROOT / sheet).is_file(): failures.append(f"manifest.json names a stylesheet that does not exist: {sheet}") if manifest_files != locked_files: failures.append( f"manifest.json lists {len(manifest_files)} faces but fonts.lock.json lists " f"{len(locked_files)}" ) per_family_ranges: dict[str, list[tuple[str, set[int]]]] = {} for rel in sorted(locked_files): path = PACKAGE_ROOT / rel if not path.is_file(): continue blob = path.read_bytes() recorded = lock["files"][rel] if recorded["sha256"] != sha256_bytes(blob): failures.append(f"{rel}: sha256 does not match fonts.lock.json") continue if recorded["bytes"] != len(blob): failures.append(f"{rel}: byte count does not match fonts.lock.json") family_dir = rel.split("/")[1] family = FAMILY_BY_DIR.get(family_dir) if family is None: failures.append(f"{rel}: no Family entry claims this directory") continue locked_family = lock["families"][family.out_dir] weight = _weight_from_filename(family, Path(rel).name) if weight is None: failures.append(f"{rel}: filename does not match any weight of {family.out_dir}") continue locked_face = locked_family["faces"][weight] font = TTFont(io.BytesIO(blob), lazy=True) try: gasp = font["gasp"].gaspRange if "gasp" in font else None expected_gasp = {int(k): v for k, v in locked_family["gasp"].items()} if gasp != expected_gasp: failures.append(f"{rel}: gasp is {gasp}, expected {expected_gasp}") if "gasp" in font and font["gasp"].version != 1: failures.append(f"{rel}: gasp version is {font['gasp'].version}, expected 1") for tag, size in locked_face["hinting"].items(): if tag not in font: failures.append(f"{rel}: hinting table {tag!r} is missing") elif len(font.reader[tag]) != size: failures.append( f"{rel}: {tag!r} is {len(font.reader[tag])} B, expected {size} B" ) if metric_signature(font) != locked_face["metrics"]: failures.append(f"{rel}: vertical metrics do not match fonts.lock.json") try: check_no_reserved_name(font, rel) except SystemExit as exc: failures.append(str(exc)) if not (font["name"].getDebugName(6) or "").startswith("Fluxer"): failures.append(f"{rel}: PostScript name is not a Fluxer name") covered = set(font.getBestCmap()) finally: font.close() declared = declared_ranges.get(rel) if declared is None: continue if declared == "": if family.stylesheet not in LATIN_CORE_STYLESHEETS: failures.append( f"{rel}: no unicode-range declared, but {family.stylesheet} is lazy" ) continue try: claimed = parse_unicode_range(declared) except ValueError as exc: failures.append(f"{rel}: {exc}") continue if claimed != covered: failures.append( f"{rel}: unicode-range claims {len(claimed)} codepoints but the font encodes " f"{len(covered)} (+{len(claimed - covered)} phantom, -{len(covered - claimed)} unreachable)" ) per_family_ranges.setdefault(f"{family.out_dir}/{weight}", []).append((rel, claimed)) for key, chunks in sorted(per_family_ranges.items()): family_dir, weight = key.rsplit("/", 1) seen: dict[int, str] = {} overlapped = False for rel, claimed in chunks: for cp in claimed: if cp in seen: failures.append(f"U+{cp:04X} is claimed by both {seen[cp]} and {rel}") overlapped = True break seen[cp] = rel if overlapped: break locked_total = lock["families"][family_dir]["faces"][weight]["codepoints"] if not overlapped and len(seen) != locked_total: failures.append( f"{family_dir}/{weight}: the {len(chunks)} chunk(s) cover {len(seen)} codepoints, " f"but fonts.lock.json records {locked_total}" ) transfer_report: list[str] = [] for family_dir in sorted(TRANSFER_SAMPLES): recorded = lock["families"].get(family_dir, {}).get("transfer") if recorded is None: failures.append(f"{family_dir}: fonts.lock.json records no transfer table") continue chunks = [ (lock["files"][rel]["bytes"], claimed) for rel, claimed in per_family_ranges.get(f"{family_dir}/Regular", []) ] if not chunks: failures.append(f"{family_dir}: no Regular chunks to measure the transfer table on") continue measured = transfer_table(chunks, recorded["unslicedBytes"], family_dir) if measured != recorded: failures.append( f"{family_dir}: the transfer table recomputed from the shipped tree does not match " f"fonts.lock.json ({measured} vs {recorded})" ) failures.extend(check_slicing_decision(family_dir, measured)) row = " ".join( f"{label} {case['bytes']:,} B/{case['chunks']}c ({case['ratio']}x)" for label, case in measured["scenarios"].items() ) transfer_report.append(f" {family_dir:18} unsliced {recorded['unslicedBytes']:,} B {row}") if failures: print(f"FAIL ({len(failures)} problem(s)):") for line in failures: print(f" - {line}") return 1 total = sum(entry["bytes"] for entry in lock["files"].values()) print( f"OK: {len(locked_files)} WOFF2 files ({total / 1048576:.1f} MiB) and " f"{len(lock['stylesheets'])} generated stylesheets match fonts.lock.json; " "gasp, rename, hinting, metric and unicode-range invariants hold." ) print("\nbytes fetched per sliced family (ratio = unsliced face / fetched):") for line in transfer_report: print(line) return 0 def _weight_from_filename(family: Family, filename: str) -> str | None: stem = filename[: -len(".woff2")] if not stem.startswith(f"{family.out_dir}-"): return None remainder = stem[len(family.out_dir) + 1 :] weight = remainder.split(".")[0] return weight if weight in family.weights else None def main() -> int: parser = argparse.ArgumentParser(description="Build the font package in packages/fonts/.") parser.add_argument( "--verify", action="store_true", help="offline: check the checked-in tree against fonts.lock.json and re-assert invariants", ) parser.add_argument( "--dry-run", action="store_true", help="fetch and build, report the diff, but write nothing", ) parser.add_argument( "--cache-dir", type=Path, default=Path(__file__).resolve().parent / ".cache", help="where upstream downloads are kept (default: tools/fonts/.cache, git-ignored)", ) parser.add_argument( "--workers", type=int, default=min(8, os.cpu_count() or 1), help="parallel face builds (default: min(8, cpu count); 1 disables the pool)", ) args = parser.parse_args() if args.verify: if args.dry_run: parser.error("--verify and --dry-run are mutually exclusive") return verify() return build(args.cache_dir.resolve(), args.dry_run, max(1, args.workers)) if __name__ == "__main__": sys.exit(main())