pepepper/fluxer
1
0 Fork 0
mirror of https://github.com/fluxerapp/fluxer synced 2026-10-17 13:05:32 +09:00
Code Issues Packages Projects Releases Wiki Activity
Files
e7b13fd19f130b8d1eafc76b51c4bcc2fc98f08e
fluxer/tools/fonts/build_fonts.py
T

1731 lines
61 KiB
Python
Raw Blame History

#!/usr/bin/env python3
# SPDX-License-Identifier: AGPL-3.0-or-later
from __future__ import annotations
import argparse
import base64
import hashlib
import io
import json
import os
import re
import shutil
import statistics
import sys
import tarfile
import urllib.parse
import urllib.request
import zipfile
from concurrent.futures import ProcessPoolExecutor
from dataclasses import dataclass, field
from pathlib import Path
from typing import Union
from fontTools import unicodedata as ftunicodedata
from fontTools.pens.recordingPen import DecomposingRecordingPen
from fontTools.subset import Options as SubsetOptions
from fontTools.subset import Subsetter
from fontTools.ttLib import TTFont
REPO_ROOT = Path(__file__).resolve().parents[2]
PACKAGE_ROOT = REPO_ROOT / "packages" / "fonts"
FILES_DIR = PACKAGE_ROOT / "files"
CSS_DIR = PACKAGE_ROOT / "css"
MANIFEST_PATH = PACKAGE_ROOT / "manifest.json"
TOOL_ROOT = Path(__file__).resolve().parent
LOCK_PATH = TOOL_ROOT / "fonts.lock.json"
SPDX_CSS_HEADER = "/* SPDX-License-Identifier: AGPL-3.0-or-later */"
GENERATED_CSS_HEADER = "/* Generated by tools/fonts/build_fonts.py. Do not edit by hand. */"
CHUNK_TARGET_BYTES = 64 * 1024
MIN_STANDALONE_BLOCK = 256
PLEX_GASP = {8: 10, 65535: 15}
CJK_GASP = {7: 10, 65535: 15}
IDENTITY_NAME_IDS = (1, 3, 4, 6, 16, 17, 18, 20, 21, 22, 25)
RENAME_NAME_IDS = IDENTITY_NAME_IDS
NO_PLEX_NAME_IDS = IDENTITY_NAME_IDS
MUTABLE_TABLES = frozenset({"gasp", "name", "head"})
CRITICAL_TABLES = (
"glyf",
"loca",
"fpgm",
"prep",
"cvt ",
"cmap",
"OS/2",
"hhea",
"hmtx",
"maxp",
"post",
"GDEF",
"GPOS",
"GSUB",
)
HEAD_CHECKSUM_SLICE = slice(8, 12)
HINTING_TABLES = ("fpgm", "prep", "cvt ")
METRIC_FIELDS: tuple[tuple[str, str], ...] = (
("head", "unitsPerEm"),
("head", "macStyle"),
("hhea", "ascender"),
("hhea", "descender"),
("hhea", "lineGap"),
("hhea", "caretSlopeRise"),
("hhea", "caretSlopeRun"),
("OS/2", "sTypoAscender"),
("OS/2", "sTypoDescender"),
("OS/2", "sTypoLineGap"),
("OS/2", "usWinAscent"),
("OS/2", "usWinDescent"),
("OS/2", "sxHeight"),
("OS/2", "sCapHeight"),
("OS/2", "usWeightClass"),
("OS/2", "fsSelection"),
("vhea", "ascent"),
("vhea", "descent"),
("vhea", "lineGap"),
)
UPRIGHT_WEIGHTS = (
"Thin",
"ExtraLight",
"Light",
"Regular",
"Text",
"Medium",
"SemiBold",
"Bold",
)
ITALIC_WEIGHTS = (
"ThinItalic",
"ExtraLightItalic",
"LightItalic",
"Italic",
"TextItalic",
"MediumItalic",
"SemiBoldItalic",
"BoldItalic",
)
CSS_WEIGHTS: dict[str, int] = {
"Thin": 100,
"ExtraLight": 200,
"Light": 300,
"Regular": 400,
"Text": 450,
"Medium": 500,
"SemiBold": 600,
"Bold": 700,
"ExtraBold": 800,
}
def css_weight(weight: str) -> int:
upright = (
weight[: -len("Italic")] if weight.endswith("Italic") and weight != "Italic" else weight
)
if upright == "Italic":
upright = "Regular"
return CSS_WEIGHTS[upright]
def css_style(weight: str) -> str:
return "italic" if weight.endswith("Italic") else "normal"
@dataclass(frozen=True)
class NpmSource:
package: str
version: str
@property
def label(self) -> str:
return f"{self.package}@{self.version}"
def provenance(self) -> dict[str, str]:
return {"type": "npm", "package": self.package, "version": self.version}
def fetch(self, cache: Path) -> Path:
root = cache / f"npm-{self.package.replace('/', '-').lstrip('@')}-{self.version}"
marker = root / ".complete"
meta_url = f"https://registry.npmjs.org/{self.package}/{self.version}"
if not marker.exists():
meta = json.loads(_http_get(meta_url))
tarball = _http_get(meta["dist"]["tarball"])
_check_integrity(meta["dist"]["integrity"], tarball, self.label)
_extract_tar(tarball, root)
marker.write_text(hashlib.sha256(tarball).hexdigest() + "\n")
return root / "package"
def artifact_sha256(self, cache: Path) -> str:
root = cache / f"npm-{self.package.replace('/', '-').lstrip('@')}-{self.version}"
return (root / ".complete").read_text().strip()
@dataclass(frozen=True)
class GithubReleaseSource:
repo: str
tag: str
asset: str
@property
def label(self) -> str:
return f"{self.repo}@{self.tag}:{self.asset}"
def provenance(self) -> dict[str, str]:
return {
"type": "github-release",
"repo": self.repo,
"tag": self.tag,
"asset": self.asset,
}
def _root(self, cache: Path) -> Path:
slug = re.sub(r"[^A-Za-z0-9]+", "-", f"{self.repo}-{self.tag}").strip("-")
return cache / f"gh-{slug}"
def fetch(self, cache: Path) -> Path:
root = self._root(cache)
marker = root / ".complete"
if not marker.exists():
url = (
f"https://github.com/{self.repo}/releases/download/"
f"{urllib.parse.quote(self.tag, safe='/')}/{self.asset}"
)
blob = _http_get(url)
_extract_zip(blob, root)
marker.write_text(hashlib.sha256(blob).hexdigest() + "\n")
return root
def artifact_sha256(self, cache: Path) -> str:
return (self._root(cache) / ".complete").read_text().strip()
Source = Union[NpmSource, GithubReleaseSource]
SOURCES: dict[str, Source] = {
"plex-sans": NpmSource("@ibm/plex-sans", "1.1.0"),
"plex-mono": NpmSource("@ibm/plex-mono", "2.5.0"),
"plex-sans-arabic": NpmSource("@ibm/plex-sans-arabic", "1.1.0"),
"plex-sans-devanagari": NpmSource("@ibm/plex-sans-devanagari", "1.1.0"),
"plex-sans-hebrew": NpmSource("@ibm/plex-sans-hebrew", "1.1.0"),
"plex-sans-thai": NpmSource("@ibm/plex-sans-thai", "1.1.0"),
"plex-sans-thai-looped": NpmSource("@ibm/plex-sans-thai-looped", "1.1.0"),
"plex-sans-jp": NpmSource("@ibm/plex-sans-jp", "3.0.0"),
"plex-sans-sc": NpmSource("@ibm/plex-sans-sc", "1.1.0"),
"plex-sans-tc": NpmSource("@ibm/plex-sans-tc", "1.1.1"),
"plex-sans-kr": GithubReleaseSource(
"IBM/plex", "@ibm/[email protected]", "ibm-plex-sans-kr.zip"
),
}
REBUILD = "rebuild"
SUBSET = "subset"
@dataclass(frozen=True)
class Family:
out_dir: str
prefix: str
weights: tuple[str, ...]
source: str
src_subdir: str
mode: str
gasp: dict[int, int]
css_family: str
stylesheet: str
def source_files(self) -> list[str]:
return [f"{self.prefix}-{w}.woff2" for w in self.weights]
LATIN_WEIGHTS = UPRIGHT_WEIGHTS + ITALIC_WEIGHTS
PLEX_WOFF2 = "fonts/complete/woff2"
LATIN_CORE_STYLESHEETS = ("fluxer-sans.css", "fluxer-mono.css")
ALWAYS_BUNDLED_LATIN_FAMILIES: tuple[str, ...] = ("Fluxer Sans", "Fluxer Mono")
FAMILIES: tuple[Family, ...] = (
Family(
"FluxerSans",
"IBMPlexSans",
LATIN_WEIGHTS,
"plex-sans",
PLEX_WOFF2,
REBUILD,
PLEX_GASP,
"Fluxer Sans",
"fluxer-sans.css",
),
Family(
"FluxerMono",
"IBMPlexMono",
LATIN_WEIGHTS,
"plex-mono",
PLEX_WOFF2,
REBUILD,
PLEX_GASP,
"Fluxer Mono",
"fluxer-mono.css",
),
Family(
"FluxerSansArabic",
"IBMPlexSansArabic",
UPRIGHT_WEIGHTS,
"plex-sans-arabic",
PLEX_WOFF2,
REBUILD,
PLEX_GASP,
"Fluxer Sans Arabic",
"script-arabic.css",
),
Family(
"FluxerSansDevanagari",
"IBMPlexSansDevanagari",
UPRIGHT_WEIGHTS,
"plex-sans-devanagari",
PLEX_WOFF2,
REBUILD,
PLEX_GASP,
"Fluxer Sans Devanagari",
"script-devanagari.css",
),
Family(
"FluxerSansHebrew",
"IBMPlexSansHebrew",
UPRIGHT_WEIGHTS,
"plex-sans-hebrew",
PLEX_WOFF2,
REBUILD,
PLEX_GASP,
"Fluxer Sans Hebrew",
"script-hebrew.css",
),
Family(
"FluxerSansThai",
"IBMPlexSansThai",
UPRIGHT_WEIGHTS,
"plex-sans-thai",
PLEX_WOFF2,
REBUILD,
PLEX_GASP,
"Fluxer Sans Thai",
"script-thai.css",
),
Family(
"FluxerSansThaiLooped",
"IBMPlexSansThaiLooped",
UPRIGHT_WEIGHTS,
"plex-sans-thai-looped",
PLEX_WOFF2,
REBUILD,
PLEX_GASP,
"Fluxer Sans Thai Looped",
"script-thai-looped.css",
),
Family(
"FluxerSansJP",
"IBMPlexSansJP",
UPRIGHT_WEIGHTS,
"plex-sans-jp",
f"{PLEX_WOFF2}/hinted",
SUBSET,
CJK_GASP,
"Fluxer Sans JP",
"script-jp.css",
),
Family(
"FluxerSansSC",
"IBMPlexSansSC",
UPRIGHT_WEIGHTS,
"plex-sans-sc",
f"{PLEX_WOFF2}/hinted",
SUBSET,
CJK_GASP,
"Fluxer Sans SC",
"script-sc.css",
),
Family(
"FluxerSansTC",
"IBMPlexSansTC",
UPRIGHT_WEIGHTS,
"plex-sans-tc",
f"{PLEX_WOFF2}/hinted",
SUBSET,
CJK_GASP,
"Fluxer Sans TC",
"script-tc.css",
),
Family(
"FluxerSansKR",
"IBMPlexSansKR",
UPRIGHT_WEIGHTS,
"plex-sans-kr",
"ibm-plex-sans-kr/fonts/complete/woff2/unhinted",
SUBSET,
PLEX_GASP,
"Fluxer Sans KR",
"script-kr.css",
),
)
FAMILY_BY_DIR = {family.out_dir: family for family in FAMILIES}
KANA_OWNED_BY_JP = frozenset(range(0x3040, 0x3100))
WITHHELD_CODEPOINTS: dict[str, frozenset[int]] = {
"FluxerSansSC": KANA_OWNED_BY_JP,
"FluxerSansTC": KANA_OWNED_BY_JP,
}
SOURCE_EXPECTATIONS: dict[str, dict[str, object]] = {
"FluxerSansKR": {
"sfntVersion": "\x00\x01\x00\x00",
"tables": {"fpgm": 682, "prep": 329, "cvt ": 46},
"numGlyphs": 12283,
"reference_file": "IBMPlexSansKR-Regular.woff2",
"reference_glyf": 2161193,
},
}
ROOT_VARIABLES_TEMPLATE = """:root {{
\t--font-sans:
\t\t{sans};
\t--font-mono: 'Fluxer Mono', 'Menlo', 'Monaco', 'Courier New', monospace;
}}
"""
LOCALE_FALLBACKS_COMMENT = ""
SCRIPT_FAMILY_ORDER: tuple[str, ...] = (
"Fluxer Sans Arabic",
"Fluxer Sans Hebrew",
"Fluxer Sans Devanagari",
"Fluxer Sans Thai Looped",
"Fluxer Sans SC",
"Fluxer Sans TC",
"Fluxer Sans JP",
"Fluxer Sans KR",
)
GENERIC_UI_FACES: tuple[str, ...] = ("system-ui", "-apple-system", "BlinkMacSystemFont")
UNCHAINED_SCRIPT_FAMILIES: tuple[str, ...] = ("Fluxer Sans Thai",)
LOCALE_FALLBACKS: tuple[tuple[tuple[str, ...], str, tuple[str, ...]], ...] = (
(
("ja",),
"Fluxer Sans JP",
("Hiragino Sans", "Hiragino Kaku Gothic ProN", "Yu Gothic", "Meiryo"),
),
(("ko",), "Fluxer Sans KR", ("Malgun Gothic", "Apple SD Gothic Neo")),
(("zh-CN", "zh-Hans"), "Fluxer Sans SC", ("PingFang SC", "Microsoft YaHei")),
(("zh-Hant-TW", "zh-TW"), "Fluxer Sans TC", ("PingFang TC", "Microsoft JhengHei")),
(("zh-HK", "zh-Hant-HK"), "Fluxer Sans TC", ("PingFang HK", "Microsoft JhengHei")),
(("ar",), "Fluxer Sans Arabic", ("Segoe UI", "Geeza Pro", "Arabic Typesetting")),
(("he",), "Fluxer Sans Hebrew", ("Segoe UI", "Arial Hebrew")),
(("hi", "mr", "ne", "sa"), "Fluxer Sans Devanagari", ("Nirmala UI", "Noto Sans Devanagari")),
(("th",), "Fluxer Sans Thai Looped", ("Leelawadee UI", "Thonburi")),
)
def font_stack(preferred: str | None, os_faces: tuple[str, ...]) -> list[str]:
scripts = [preferred] if preferred else []
scripts += [family for family in SCRIPT_FAMILY_ORDER if family != preferred]
return [
quote_family("Fluxer Sans"),
*(quote_family(family) for family in scripts),
*GENERIC_UI_FACES,
*(quote_family(face) for face in os_faces),
"sans-serif",
]
def quote_family(name: str) -> str:
return f"'{name}'"
def wrap_font_stack(entries: list[str], indent: str, width: int = 118) -> str:
lines: list[str] = []
current = ""
for index, entry in enumerate(entries):
piece = entry if index == len(entries) - 1 else f"{entry},"
candidate = f"{current} {piece}" if current else piece
if current and len(indent) + len(candidate) > width:
lines.append(current)
current = piece
else:
current = candidate
if current:
lines.append(current)
return f"\n{indent}".join(lines)
BASE_UI_FACES: tuple[str, ...] = ("Segoe UI", "Roboto")
def render_variables_stylesheet() -> str:
stack = wrap_font_stack(font_stack(None, BASE_UI_FACES), "\t\t")
body = ROOT_VARIABLES_TEMPLATE.format(sans=stack)
return "\n".join([SPDX_CSS_HEADER, GENERATED_CSS_HEADER, "", body])
def render_locale_fallbacks_stylesheet() -> str:
lines = [SPDX_CSS_HEADER, GENERATED_CSS_HEADER, "", LOCALE_FALLBACKS_COMMENT]
for langs, preferred, os_faces in LOCALE_FALLBACKS:
selector = ",\n".join(f":root:lang({lang})" for lang in langs)
stack = wrap_font_stack(font_stack(preferred, os_faces), "\t\t")
lines.append(f"{selector} {{\n\t--font-sans:\n\t\t{stack};\n}}")
return "\n".join([*lines, ""])
def _http_get(url: str) -> bytes:
request = urllib.request.Request(url, headers={"User-Agent": "fluxer-font-build"})
with urllib.request.urlopen(request) as response: # noqa: S310 - pinned https URLs
return response.read()
def _check_integrity(integrity: str, blob: bytes, label: str) -> None:
algo, _, encoded = integrity.partition("-")
actual = base64.b64encode(hashlib.new(algo, blob).digest()).decode()
if actual != encoded:
raise SystemExit(f"integrity mismatch for {label}: expected {encoded}, got {actual}")
def _extract_tar(blob: bytes, root: Path) -> None:
if root.exists():
shutil.rmtree(root)
root.mkdir(parents=True)
with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as archive:
for member in archive.getmembers():
_extract_member(archive, member, member.name, root, member.isdir())
def _extract_zip(blob: bytes, root: Path) -> None:
if root.exists():
shutil.rmtree(root)
root.mkdir(parents=True)
with zipfile.ZipFile(io.BytesIO(blob)) as archive:
for info in archive.infolist():
_extract_member(archive, info, info.filename, root, info.is_dir())
def _extract_member(archive: object, member: object, name: str, root: Path, is_dir: bool) -> None:
dest = (root / name).resolve()
if not str(dest).startswith(str(root.resolve())):
raise SystemExit(f"refusing to extract outside the cache: {name}")
if is_dir:
dest.mkdir(parents=True, exist_ok=True)
return
dest.parent.mkdir(parents=True, exist_ok=True)
reader = (
archive.extractfile(member)
if isinstance(archive, tarfile.TarFile)
else archive.open(member)
)
if reader is None:
return
with reader:
dest.write_bytes(reader.read())
def fluxer_name(value: str) -> str:
return value.replace("IBM Plex ", "Fluxer ").replace("IBMPlex", "Fluxer")
def _name_map(font: TTFont) -> dict[tuple[int, int, int, int], str]:
return {
(r.nameID, r.platformID, r.platEncID, r.langID): r.toUnicode() for r in font["name"].names
}
def build_font(src: Path, gasp: dict[int, int]) -> bytes:
font = TTFont(src, lazy=True, recalcTimestamp=False, recalcBBoxes=False)
font["gasp"].gaspRange = dict(gasp)
for record in font["name"].names:
if record.nameID in RENAME_NAME_IDS:
record.string = fluxer_name(record.toUnicode())
font.flavor = "woff2"
buffer = io.BytesIO()
font.save(buffer)
font.close()
return buffer.getvalue()
def check_rebuild(src: Path, produced: bytes, gasp: dict[int, int]) -> None:
upstream = TTFont(src, lazy=True)
built = TTFont(io.BytesIO(produced), lazy=True)
try:
up_tags = {t for t in upstream.keys() if t != "GlyphOrder"}
new_tags = {t for t in built.keys() if t != "GlyphOrder"}
if up_tags != new_tags:
raise SystemExit(
f"{src.name}: table set changed: "
f"+{sorted(new_tags - up_tags)} -{sorted(up_tags - new_tags)}"
)
for tag in sorted(up_tags - MUTABLE_TABLES):
if upstream.reader[tag] != built.reader[tag]:
raise SystemExit(f"{src.name}: table {tag!r} changed but must not")
for tag in CRITICAL_TABLES:
if tag in up_tags and upstream.reader[tag] != built.reader[tag]:
raise SystemExit(f"{src.name}: critical table {tag!r} changed")
up_head = upstream.reader["head"]
new_head = built.reader["head"]
if len(up_head) != len(new_head):
raise SystemExit(f"{src.name}: head length changed")
if (
up_head[: HEAD_CHECKSUM_SLICE.start] != new_head[: HEAD_CHECKSUM_SLICE.start]
or up_head[HEAD_CHECKSUM_SLICE.stop :] != new_head[HEAD_CHECKSUM_SLICE.stop :]
):
raise SystemExit(f"{src.name}: head changed outside checkSumAdjustment")
if built["gasp"].gaspRange != gasp:
raise SystemExit(f"{src.name}: gasp is {built['gasp'].gaspRange}, expected {gasp}")
if built["gasp"].version != 1:
raise SystemExit(f"{src.name}: gasp version is {built['gasp'].version}, expected 1")
up_names = _name_map(upstream)
new_names = _name_map(built)
if set(up_names) != set(new_names):
raise SystemExit(f"{src.name}: name record set changed")
for key, value in up_names.items():
expected = fluxer_name(value) if key[0] in RENAME_NAME_IDS else value
if new_names[key] != expected:
raise SystemExit(
f"{src.name}: name {key} is {new_names[key]!r}, expected {expected!r}"
)
check_no_reserved_name(built, src.name)
finally:
upstream.close()
built.close()
def check_no_reserved_name(font: TTFont, label: str) -> None:
for record in font["name"].names:
if record.nameID in NO_PLEX_NAME_IDS and "Plex" in record.toUnicode():
raise SystemExit(
f"{label}: reserved font name 'Plex' still present in name ID {record.nameID}"
)
def subset_options() -> SubsetOptions:
options = SubsetOptions()
options.hinting = True # keep glyph instructions; fpgm/prep/cvt pass through
options.legacy_kern = True
options.layout_features = ["*"] # CJK needs vert/vrt2/halt/palt/trad/smpl
options.layout_scripts = ["*"]
options.name_IDs = ["*"] # the OFL rename lives in the name table
options.name_legacy = True
options.name_languages = ["*"]
options.notdef_outline = True
options.glyph_names = False # post 3.0; names are not rendering data
options.recalc_bounds = False # keep upstream head/glyf bounding boxes
options.recalc_timestamp = False # keep upstream head.modified
options.recalc_max_context = False
options.recalc_average_width = False # keep OS/2.xAvgCharWidth
options.prune_unicode_ranges = False # keep OS/2.ulUnicodeRange
options.prune_codepage_ranges = False
options.drop_tables = []
options.passthrough_tables = True # e.g. `meta`, which the subsetter cannot slice
options.ignore_missing_unicodes = True
return options
def subset_face(sfnt: bytes, codepoints: tuple[int, ...]) -> bytes:
font = TTFont(io.BytesIO(sfnt), lazy=True, recalcTimestamp=False, recalcBBoxes=False)
subsetter = Subsetter(options=subset_options())
subsetter.populate(unicodes=codepoints)
subsetter.subset(font)
font.flavor = "woff2"
buffer = io.BytesIO()
font.save(buffer)
font.close()
return buffer.getvalue()
def glyph_signature(glyph_set: object, glyf: object, name: str) -> tuple:
pen = DecomposingRecordingPen(glyph_set)
glyph_set[name].draw(pen)
glyph = glyf[name]
program = getattr(glyph, "program", None)
bytecode = bytes(program.getBytecode()) if program is not None else b""
box = (glyph.xMin, glyph.yMin, glyph.xMax, glyph.yMax) if glyph.numberOfContours != 0 else ()
return (tuple(pen.value), box, bytecode)
def metric_signature(font: TTFont) -> dict[str, int]:
values: dict[str, int] = {}
for table, field_name in METRIC_FIELDS:
if table not in font:
continue
value = getattr(font[table], field_name, None)
if value is None:
continue
values[f"{table.strip()}.{field_name}"] = int(value)
return values
def check_subset(
parent: TTFont, chunk_bytes: bytes, requested: tuple[int, ...], label: str
) -> dict:
chunk = TTFont(io.BytesIO(chunk_bytes), lazy=True)
try:
parent_tags = {t for t in parent.keys() if t != "GlyphOrder"}
chunk_tags = {t for t in chunk.keys() if t != "GlyphOrder"}
if parent_tags != chunk_tags:
raise SystemExit(
f"{label}: table set changed: "
f"+{sorted(chunk_tags - parent_tags)} -{sorted(parent_tags - chunk_tags)}"
)
for tag in HINTING_TABLES + ("gasp",):
if parent.reader[tag] != chunk.reader[tag]:
raise SystemExit(f"{label}: table {tag!r} must pass through unchanged")
if metric_signature(parent) != metric_signature(chunk):
raise SystemExit(f"{label}: vertical metrics changed")
check_no_reserved_name(chunk, label)
if not (chunk["name"].getDebugName(6) or "").startswith("Fluxer"):
raise SystemExit(f"{label}: PostScript name is not a Fluxer name")
chunk_cmap = chunk.getBestCmap()
expected = {cp for cp in requested if cp in parent.getBestCmap()}
if set(chunk_cmap) != expected:
missing = sorted(expected - set(chunk_cmap))[:4]
extra = sorted(set(chunk_cmap) - expected)[:4]
raise SystemExit(f"{label}: cmap drifted from the plan (-{missing} +{extra})")
parent_cmap = parent.getBestCmap()
parent_glyf = parent["glyf"]
chunk_glyf = chunk["glyf"]
parent_set = parent.getGlyphSet()
chunk_set = chunk.getGlyphSet()
parent_hmtx = parent["hmtx"]
chunk_hmtx = chunk["hmtx"]
parent_vmtx = parent["vmtx"] if "vmtx" in parent else None
chunk_vmtx = chunk["vmtx"] if "vmtx" in chunk else None
for cp in sorted(expected):
name = parent_cmap[cp]
chunk_name = chunk_cmap[cp]
if glyph_signature(parent_set, parent_glyf, name) != glyph_signature(
chunk_set, chunk_glyf, chunk_name
):
raise SystemExit(f"{label}: outline changed for U+{cp:04X} ({name})")
if parent_hmtx[name] != chunk_hmtx[chunk_name]:
raise SystemExit(f"{label}: horizontal metrics changed for U+{cp:04X} ({name})")
if parent_vmtx is not None and parent_vmtx[name] != chunk_vmtx[chunk_name]:
raise SystemExit(f"{label}: vertical metrics changed for U+{cp:04X} ({name})")
return {
"codepoints": sorted(chunk_cmap),
"hinting": {tag: len(chunk.reader[tag]) for tag in HINTING_TABLES},
"metrics": metric_signature(chunk),
}
finally:
chunk.close()
def plan_chunks(codepoints: set[int], target: int) -> list[tuple[int, ...]]:
if target < 1:
raise SystemExit(f"chunk target must be positive, got {target}")
by_block: dict[str, list[int]] = {}
for cp in sorted(codepoints):
by_block.setdefault(ftunicodedata.block(cp), []).append(cp)
chunks: list[tuple[int, ...]] = []
pending: list[int] = []
def flush() -> None:
if pending:
chunks.append(tuple(pending))
pending.clear()
for _, members in sorted(by_block.items(), key=lambda item: item[1][0]):
if len(members) < MIN_STANDALONE_BLOCK:
pending.extend(members)
if len(pending) >= target:
flush()
continue
flush()
parts = max(1, round(len(members) / target)) if len(members) > target else 1
size = -(-len(members) // parts)
for start in range(0, len(members), size):
chunks.append(tuple(members[start : start + size]))
flush()
if not chunks:
raise SystemExit("chunk plan is empty")
return chunks
TRANSFER_SAMPLES: dict[str, dict[str, str]] = {
"FluxerSansJP": {
"mixed script": "語",
"display name": "田中太郎",
"sentence": "これは日本語のサンプルテキストです",
},
"FluxerSansSC": {
"mixed script": "文",
"display name": "小明",
"sentence": "这是简体中文的示例文本",
},
"FluxerSansTC": {
"mixed script": "文",
"display name": "小明",
"sentence": "這是繁體中文的示例文本",
},
"FluxerSansKR": {
"mixed script": "한",
"display name": "김민준",
"sentence": "이것은 한국어 샘플 텍스트입니다",
},
}
SLICING_DECISION: dict[str, dict[str, float]] = {
"FluxerSansJP": {"minMixedScriptRatio": 20.0, "maxFullCoverageRatio": 1.25},
"FluxerSansSC": {"minMixedScriptRatio": 20.0, "maxFullCoverageRatio": 1.30},
"FluxerSansTC": {"minMixedScriptRatio": 20.0, "maxFullCoverageRatio": 1.50},
"FluxerSansKR": {"minMixedScriptRatio": 2.0, "maxFullCoverageRatio": 1.25},
}
def transfer_table(chunks: list[tuple[int, set[int]]], unsliced: int, family_dir: str) -> dict:
scenarios = {}
for label, text in TRANSFER_SAMPLES[family_dir].items():
wanted = {ord(c) for c in text}
hit = [(size, covered) for size, covered in chunks if covered & wanted]
fetched = sum(size for size, _ in hit)
scenarios[label] = {
"bytes": fetched,
"chunks": len(hit),
"ratio": round(unsliced / fetched, 3) if fetched else None,
}
full = sum(size for size, _ in chunks)
scenarios["full coverage"] = {
"bytes": full,
"chunks": len(chunks),
"ratio": round(unsliced / full, 3) if full else None,
}
return {"unslicedBytes": unsliced, "scenarios": scenarios}
def check_slicing_decision(family_dir: str, transfer: dict) -> list[str]:
bars = SLICING_DECISION.get(family_dir)
if bars is None:
return []
failures = []
mixed = transfer["scenarios"]["mixed script"]["ratio"]
if mixed is None or mixed < bars["minMixedScriptRatio"]:
failures.append(
f"{family_dir}: slicing wins only {mixed}x on the mixed-script case, below the "
f"{bars['minMixedScriptRatio']}x bar that justifies it."
)
full_ratio = transfer["scenarios"]["full coverage"]["ratio"]
regression = 1 / full_ratio if full_ratio else float("inf")
if regression > bars["maxFullCoverageRatio"]:
failures.append(
f"{family_dir}: full coverage now costs {regression:.3f}x the unsliced face, past the "
f"{bars['maxFullCoverageRatio']}x bar"
)
return failures
def codepoint_ranges(codepoints: list[int]) -> list[tuple[int, int]]:
ranges: list[tuple[int, int]] = []
for cp in codepoints:
if ranges and cp == ranges[-1][1] + 1:
ranges[-1] = (ranges[-1][0], cp)
else:
ranges.append((cp, cp))
return ranges
def format_unicode_range(codepoints: list[int]) -> str:
parts = []
for start, end in codepoint_ranges(sorted(codepoints)):
parts.append(f"U+{start:04X}" if start == end else f"U+{start:04X}-{end:04X}")
return ", ".join(parts)
def parse_unicode_range(value: str) -> set[int]:
result: set[int] = set()
for token in value.split(","):
token = token.strip()
if not token:
continue
match = re.fullmatch(r"[Uu]\+([0-9A-Fa-f]+)(?:-([0-9A-Fa-f]+))?", token)
if not match:
raise ValueError(f"unparseable unicode-range token {token!r}")
start = int(match.group(1), 16)
end = int(match.group(2), 16) if match.group(2) else start
result.update(range(start, end + 1))
return result
def calibrate_chunk_size(sfnt: bytes, codepoints: list[int], target_bytes: int) -> tuple[int, str]:
small, large = 300, 900
if len(codepoints) <= large:
return len(codepoints), "single chunk (family smaller than one probe)"
middle = len(codepoints) // 2
probe_small = tuple(codepoints[middle : middle + small])
probe_large = tuple(codepoints[middle : middle + large])
size_small = len(subset_face(sfnt, probe_small))
size_large = len(subset_face(sfnt, probe_large))
rate = (size_large - size_small) / (large - small)
overhead = size_small - rate * small
if rate <= 0:
raise SystemExit("calibration produced a non-positive bytes-per-codepoint rate")
budget = int((target_bytes - overhead) / rate)
budget = max(MIN_STANDALONE_BLOCK, budget)
note = (
f"{rate:.1f} B/codepoint marginal, {overhead / 1024:.1f} KiB intercept "
f"-> {budget} codepoints per chunk"
)
return budget, note
def sha256_bytes(blob: bytes) -> str:
return hashlib.sha256(blob).hexdigest()
@dataclass
class FaceResult:
family: str
weight: str
source_sha256: str
hinting: dict[str, int]
metrics: dict[str, int]
files: list[dict] = field(default_factory=list)
def check_source_expectations(family: Family, src_dir: Path) -> None:
expectations = SOURCE_EXPECTATIONS.get(family.out_dir)
if expectations is None:
return
path = src_dir / str(expectations["reference_file"])
font = TTFont(path, lazy=True)
try:
if font.sfntVersion != expectations["sfntVersion"]:
raise SystemExit(
f"{path.name}: sfntVersion {font.sfntVersion!r}, "
f"expected {expectations['sfntVersion']!r} (TrueType outlines)"
)
for tag, size in expectations["tables"].items(): # type: ignore[union-attr]
if tag not in font:
raise SystemExit(f"{path.name}: hinting table {tag!r} is missing upstream")
if len(font.reader[tag]) != size:
raise SystemExit(
f"{path.name}: {tag!r} is {len(font.reader[tag])} B upstream, expected {size} B"
)
if font["maxp"].numGlyphs != expectations["numGlyphs"]:
raise SystemExit(
f"{path.name}: numGlyphs {font['maxp'].numGlyphs}, expected {expectations['numGlyphs']}"
)
if "glyf" in font and len(font.reader["glyf"]) != expectations["reference_glyf"]:
raise SystemExit(
f"{path.name}: glyf is {len(font.reader['glyf'])} B, "
f"expected {expectations['reference_glyf']} B"
)
finally:
font.close()
def face_cmap(path: Path) -> set[int]:
font = TTFont(path, lazy=True)
try:
return set(font.getBestCmap())
finally:
font.close()
def renamed_sfnt(src: Path, gasp: dict[int, int]) -> bytes:
produced = build_font(src, gasp)
check_rebuild(src, produced, gasp)
font = TTFont(io.BytesIO(produced), lazy=True, recalcTimestamp=False, recalcBBoxes=False)
font.flavor = None
buffer = io.BytesIO()
font.save(buffer)
font.close()
return buffer.getvalue()
def build_face(job: dict) -> FaceResult:
family = FAMILY_BY_DIR[job["family"]]
src = Path(job["src"])
weight = job["weight"]
source_bytes = src.read_bytes()
result = FaceResult(
family=family.out_dir,
weight=weight,
source_sha256=sha256_bytes(source_bytes),
hinting={},
metrics={},
)
stage = Path(job["stage"]) / family.out_dir
stage.mkdir(parents=True, exist_ok=True)
if family.mode == REBUILD:
produced = build_font(src, family.gasp)
check_rebuild(src, produced, family.gasp)
font = TTFont(io.BytesIO(produced), lazy=True)
try:
result.hinting = {tag: len(font.reader[tag]) for tag in HINTING_TABLES}
result.metrics = metric_signature(font)
covered = sorted(font.getBestCmap())
finally:
font.close()
name = f"{family.out_dir}-{weight}.woff2"
(stage / name).write_bytes(produced)
result.files.append(
{
"path": f"{family.out_dir}/{name}",
"sha256": sha256_bytes(produced),
"bytes": len(produced),
"codepoints": len(covered),
"unicodeRange": None
if family.stylesheet in LATIN_CORE_STYLESHEETS
else format_unicode_range(covered),
"chunk": None,
}
)
return result
sfnt = renamed_sfnt(src, family.gasp)
parent = TTFont(io.BytesIO(sfnt), lazy=True)
try:
result.hinting = {tag: len(parent.reader[tag]) for tag in HINTING_TABLES}
result.metrics = metric_signature(parent)
parent_cmap = set(parent.getBestCmap())
for index, planned in enumerate(job["plan"]):
requested = tuple(cp for cp in planned if cp in parent_cmap)
if not requested:
continue
name = f"{family.out_dir}-{weight}.{index:03d}.woff2"
label = f"{family.out_dir}/{name}"
produced = subset_face(sfnt, requested)
info = check_subset(parent, produced, requested, label)
if info["hinting"] != result.hinting:
raise SystemExit(f"{label}: hinting table sizes drifted from the parent face")
(stage / name).write_bytes(produced)
result.files.append(
{
"path": label,
"sha256": sha256_bytes(produced),
"bytes": len(produced),
"codepoints": len(info["codepoints"]),
"unicodeRange": format_unicode_range(info["codepoints"]),
"chunk": index,
}
)
finally:
parent.close()
return result
def resolve_sources(cache: Path, families: tuple[Family, ...]) -> dict[str, Path]:
roots: dict[str, Path] = {}
for key in dict.fromkeys(f.source for f in families):
source = SOURCES[key]
print(f" fetching {source.label}", flush=True)
roots[key] = source.fetch(cache)
return roots
def render_face_rule(family: Family, entry: dict) -> str:
weight = entry["weight"]
lines = [
"@font-face {",
f"\tfont-family: '{family.css_family}';",
f"\tsrc: url('../files/{entry['path']}') format('woff2');",
f"\tfont-weight: {css_weight(weight)};",
f"\tfont-style: {css_style(weight)};",
"\tfont-display: swap;",
]
if entry["unicodeRange"]:
lines.append(f"\tunicode-range: {entry['unicodeRange']};")
lines.append("}")
return "\n".join(lines)
def chunk_major(family: Family, entries: list[dict]) -> list[dict]:
order = {weight: index for index, weight in enumerate(family.weights)}
return sorted(
entries,
key=lambda entry: (
entry["chunk"] if entry["chunk"] is not None else -1,
order[entry["weight"]],
),
)
def render_stylesheets(entries: list[dict]) -> dict[str, str]:
by_family: dict[str, list[dict]] = {}
for entry in entries:
by_family.setdefault(entry["family"], []).append(entry)
by_stylesheet: dict[str, list[str]] = {}
for family in FAMILIES:
for entry in chunk_major(family, by_family[family.out_dir]):
by_stylesheet.setdefault(family.stylesheet, []).append(render_face_rule(family, entry))
sheets: dict[str, str] = {}
for name, rules in by_stylesheet.items():
sheets[name] = "\n".join([SPDX_CSS_HEADER, GENERATED_CSS_HEADER, "", *rules, ""])
sheets["variables.css"] = render_variables_stylesheet()
sheets["locale-fallbacks.css"] = render_locale_fallbacks_stylesheet()
return sheets
def render_manifest(entries: list[dict], chunking: dict) -> str:
families = []
for family in FAMILIES:
faces = [
{
"file": entry["path"],
"weight": css_weight(entry["weight"]),
"style": css_style(entry["weight"]),
"unicodeRange": entry["unicodeRange"],
"bytes": entry["bytes"],
}
for entry in chunk_major(family, [e for e in entries if e["family"] == family.out_dir])
]
families.append(
{
"id": family.out_dir,
"cssFamily": family.css_family,
"stylesheet": f"css/{family.stylesheet}",
"mode": family.mode,
"latinCore": family.stylesheet in LATIN_CORE_STYLESHEETS,
"chunks": chunking.get(family.out_dir, {}).get("chunks", 1),
"faces": faces,
}
)
payload = {
"_comment": (
"Generated by tools/fonts/build_fonts.py. Do not edit by hand. "
"Paths are relative to packages/fonts/."
),
"entrypoints": {
"latinCore": {
family.css_family: f"css/{family.stylesheet}"
for family in FAMILIES
if family.stylesheet in LATIN_CORE_STYLESHEETS
},
"variables": "css/variables.css",
"localeFallbacks": "css/locale-fallbacks.css",
"scripts": {
family.css_family: f"css/{family.stylesheet}"
for family in FAMILIES
if family.stylesheet not in LATIN_CORE_STYLESHEETS
},
},
"families": families,
}
return json.dumps(payload, indent="\t", sort_keys=False) + "\n"
def size_distribution(sizes: list[int]) -> str:
if not sizes:
return "no files"
ordered = sorted(sizes)
return (
f"min {ordered[0] / 1024:.0f} KiB, median {statistics.median(ordered) / 1024:.0f} KiB, "
f"p95 {ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))] / 1024:.0f} KiB, "
f"max {ordered[-1] / 1024:.0f} KiB"
)
def build(cache: Path, dry_run: bool, workers: int) -> int:
print(f"cache: {cache}")
cache.mkdir(parents=True, exist_ok=True)
roots = resolve_sources(cache, FAMILIES)
stage = cache / "stage"
if stage.exists():
shutil.rmtree(stage)
stage.mkdir(parents=True)
print("\nplanning chunks")
plans: dict[str, list[tuple[int, ...]]] = {}
chunking: dict[str, dict] = {}
family_codepoints: dict[str, int] = {}
for family in FAMILIES:
src_dir = roots[family.source] / family.src_subdir
for name in family.source_files():
if not (src_dir / name).is_file():
raise SystemExit(f"missing upstream file: {src_dir / name}")
check_source_expectations(family, src_dir)
union: set[int] = set()
for name in family.source_files():
union |= face_cmap(src_dir / name)
union -= WITHHELD_CODEPOINTS.get(family.out_dir, frozenset())
family_codepoints[family.out_dir] = len(union)
if family.mode != SUBSET:
continue
reference = src_dir / f"{family.prefix}-Regular.woff2"
sfnt = renamed_sfnt(reference, family.gasp)
budget, note = calibrate_chunk_size(sfnt, sorted(union), CHUNK_TARGET_BYTES)
plan = plan_chunks(union, budget)
plans[family.out_dir] = plan
chunking[family.out_dir] = {
"targetBytes": CHUNK_TARGET_BYTES,
"codepointsPerChunk": budget,
"chunks": len(plan),
"codepoints": len(union),
}
print(f" {family.out_dir}: {len(union)} codepoints, {note}, {len(plan)} chunks")
jobs = [
{
"family": family.out_dir,
"weight": weight,
"src": str(
roots[family.source] / family.src_subdir / f"{family.prefix}-{weight}.woff2"
),
"stage": str(stage),
"plan": plans.get(family.out_dir, []),
}
for family in FAMILIES
for weight in family.weights
]
print(f"\nbuilding {len(jobs)} faces on {workers} worker(s)")
results: list[FaceResult] = []
if workers <= 1:
for job in jobs:
results.append(build_face(job))
print(f" {job['family']}-{job['weight']}", flush=True)
else:
with ProcessPoolExecutor(max_workers=workers) as pool:
for result in pool.map(build_face, jobs):
results.append(result)
print(f" {result.family}-{result.weight}: {len(result.files)} file(s)", flush=True)
order = {
(family.out_dir, weight): (i, j)
for i, family in enumerate(FAMILIES)
for j, weight in enumerate(family.weights)
}
results.sort(key=lambda r: order[(r.family, r.weight)])
entries: list[dict] = []
for result in results:
for item in result.files:
entries.append({**item, "family": result.family, "weight": result.weight})
sheets = render_stylesheets(entries)
manifest = render_manifest(entries, chunking)
lock: dict[str, object] = {
"_comment": (
"Generated by tools/fonts/build_fonts.py. Do not edit by hand. "
"Run `python3 tools/fonts/build_fonts.py --verify` to check the tree against it."
),
"gasp": {
"plex": {str(k): v for k, v in PLEX_GASP.items()},
"cjk": {str(k): v for k, v in CJK_GASP.items()},
},
"chunking": {"targetBytes": CHUNK_TARGET_BYTES, "families": chunking},
"sources": {},
"families": {},
"stylesheets": {},
"files": {},
}
for key in dict.fromkeys(f.source for f in FAMILIES):
source = SOURCES[key]
lock["sources"][key] = {**source.provenance(), "sha256": source.artifact_sha256(cache)}
for family in FAMILIES:
faces = {}
for result in results:
if result.family != family.out_dir:
continue
faces[result.weight] = {
"sourceFile": f"{family.prefix}-{result.weight}.woff2",
"sourceSha256": result.source_sha256,
"hinting": result.hinting,
"metrics": result.metrics,
"files": len(result.files),
"codepoints": sum(item["codepoints"] for item in result.files),
}
lock["families"][family.out_dir] = {
"mode": family.mode,
"source": family.source,
"srcSubdir": family.src_subdir,
"cssFamily": family.css_family,
"stylesheet": f"css/{family.stylesheet}",
"latinCore": family.stylesheet in LATIN_CORE_STYLESHEETS,
"gasp": {str(k): v for k, v in family.gasp.items()},
"codepoints": family_codepoints[family.out_dir],
"faces": faces,
}
if family.out_dir in TRANSFER_SAMPLES:
unsliced = (
(roots[family.source] / family.src_subdir / f"{family.prefix}-Regular.woff2")
.stat()
.st_size
)
regular = [
(item["bytes"], parse_unicode_range(item["unicodeRange"]))
for result in results
if result.family == family.out_dir and result.weight == "Regular"
for item in result.files
]
transfer = transfer_table(regular, unsliced, family.out_dir)
for failure in check_slicing_decision(family.out_dir, transfer):
raise SystemExit(failure)
lock["families"][family.out_dir]["transfer"] = transfer
for name, text in sorted(sheets.items()):
blob = text.encode()
lock["stylesheets"][f"css/{name}"] = {"sha256": sha256_bytes(blob), "bytes": len(blob)}
manifest_blob = manifest.encode()
lock["stylesheets"]["manifest.json"] = {
"sha256": sha256_bytes(manifest_blob),
"bytes": len(manifest_blob),
}
for entry in entries:
lock["files"][f"files/{entry['path']}"] = {
"sha256": entry["sha256"],
"bytes": entry["bytes"],
}
print("\nper-family results")
print(f" {'family':24} {'files':>6} {'upstream':>10} {'shipped':>10} distribution")
total_before = 0
total_after = 0
for family in FAMILIES:
src_dir = roots[family.source] / family.src_subdir
before = sum((src_dir / name).stat().st_size for name in family.source_files())
sizes = [entry["bytes"] for entry in entries if entry["family"] == family.out_dir]
after = sum(sizes)
total_before += before
total_after += after
print(
f" {family.out_dir:24} {len(sizes):>6} {before / 1048576:>9.2f}M {after / 1048576:>9.2f}M "
f"{size_distribution(sizes)}"
)
print(
f" {'TOTAL':24} {len(entries):>6} {total_before / 1048576:>9.2f}M {total_after / 1048576:>9.2f}M"
)
changed, unchanged, created, removed = _diff_tree(entries, sheets, manifest)
print(f"\nsummary: {created} new, {changed} changed, {unchanged} unchanged, {removed} removed")
if dry_run:
print("dry run: nothing written")
return 0
_write_tree(entries, sheets, manifest, stage)
LOCK_PATH.write_text(json.dumps(lock, indent="\t", sort_keys=False) + "\n")
print(f"wrote {PACKAGE_ROOT.relative_to(REPO_ROOT)}/ and {LOCK_PATH.relative_to(REPO_ROOT)}")
return 0
def _tracked_outputs(sheets: dict[str, str], manifest: str) -> dict[str, bytes]:
outputs: dict[str, bytes] = {}
for name, text in sheets.items():
outputs[f"css/{name}"] = text.encode()
outputs["manifest.json"] = manifest.encode()
return outputs
def _diff_tree(
entries: list[dict], sheets: dict[str, str], manifest: str
) -> tuple[int, int, int, int]:
changed = unchanged = created = 0
expected: set[str] = set()
for entry in entries:
rel = f"files/{entry['path']}"
expected.add(rel)
dest = PACKAGE_ROOT / rel
if not dest.is_file():
created += 1
elif sha256_bytes(dest.read_bytes()) == entry["sha256"]:
unchanged += 1
else:
changed += 1
for rel, blob in _tracked_outputs(sheets, manifest).items():
expected.add(rel)
dest = PACKAGE_ROOT / rel
if not dest.is_file():
created += 1
elif dest.read_bytes() == blob:
unchanged += 1
else:
changed += 1
removed = len(_existing_generated() - expected)
return changed, unchanged, created, removed
def _existing_generated() -> set[str]:
found: set[str] = set()
if FILES_DIR.is_dir():
found |= {str(p.relative_to(PACKAGE_ROOT)) for p in FILES_DIR.rglob("*.woff2")}
if CSS_DIR.is_dir():
found |= {str(p.relative_to(PACKAGE_ROOT)) for p in CSS_DIR.rglob("*.css")}
if MANIFEST_PATH.is_file():
found.add("manifest.json")
return found
def _write_tree(entries: list[dict], sheets: dict[str, str], manifest: str, stage: Path) -> None:
expected: set[str] = set()
for entry in entries:
rel = f"files/{entry['path']}"
expected.add(rel)
dest = PACKAGE_ROOT / rel
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(stage / entry["path"], dest)
for rel, blob in _tracked_outputs(sheets, manifest).items():
expected.add(rel)
dest = PACKAGE_ROOT / rel
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(blob)
for rel in sorted(_existing_generated() - expected):
(PACKAGE_ROOT / rel).unlink()
for directory in sorted(FILES_DIR.rglob("*"), reverse=True):
if directory.is_dir() and not any(directory.iterdir()):
directory.rmdir()
FACE_RULE = re.compile(
r"@font-face\s*\{(?P<body>[^}]*)\}",
re.MULTILINE,
)
def parse_stylesheet(text: str) -> list[dict[str, str]]:
faces = []
for match in FACE_RULE.finditer(text):
body = match.group("body")
declarations = {}
for declaration in body.split(";"):
name, _, value = declaration.partition(":")
name = name.strip()
if name:
declarations[name] = value.strip()
url = re.search(r"url\('([^']+)'\)", declarations.get("src", ""))
faces.append(
{
"family": declarations.get("font-family", "").strip("'"),
"url": url.group(1) if url else "",
"weight": declarations.get("font-weight", ""),
"style": declarations.get("font-style", ""),
"display": declarations.get("font-display", ""),
"unicodeRange": declarations.get("unicode-range", ""),
}
)
return faces
def check_variable_families(text: str) -> list[str]:
allowed = set(ALWAYS_BUNDLED_LATIN_FAMILIES)
allowed |= {f.css_family for f in FAMILIES if f.stylesheet not in LATIN_CORE_STYLESHEETS}
shipped = {f.css_family for f in FAMILIES}
failures: list[str] = []
for match in re.finditer(r"(--font-[a-z0-9-]+)\s*:(?P<value>[^;]*);", text):
variable = match.group(1)
for name in re.findall(r"'([^']+)'", match.group("value")):
if name in shipped and name not in allowed:
failures.append(
f"packages/fonts/css/variables.css: {variable} names {name!r}, but importing "
f"variables.css does not bring that family's @font-face rules with it. Declare "
f"the variable in the consumer that actually imports its stylesheet instead."
)
return failures
def verify() -> int:
if not LOCK_PATH.is_file():
raise SystemExit(f"missing {LOCK_PATH}; run the build first")
lock = json.loads(LOCK_PATH.read_text())
failures: list[str] = []
locked_files = set(lock["files"])
on_disk = (
{f"files/{p.relative_to(FILES_DIR)}" for p in FILES_DIR.rglob("*.woff2")}
if FILES_DIR.is_dir()
else set()
)
for extra in sorted(on_disk - locked_files):
failures.append(f"font present in the tree but absent from fonts.lock.json: {extra}")
for missing in sorted(locked_files - on_disk):
failures.append(f"font recorded in fonts.lock.json but missing from the tree: {missing}")
strays = (
sorted(p.relative_to(PACKAGE_ROOT) for p in PACKAGE_ROOT.rglob("*.otf"))
if PACKAGE_ROOT.is_dir()
else []
)
if strays:
failures.append(f"{len(strays)} unreferenced .otf file(s) present, e.g. {strays[0]}")
referenced: dict[str, list[str]] = {}
declared_ranges: dict[str, str] = {}
for rel, recorded in lock["stylesheets"].items():
path = PACKAGE_ROOT / rel
if not path.is_file():
failures.append(f"missing generated file: packages/fonts/{rel}")
continue
blob = path.read_bytes()
if sha256_bytes(blob) != recorded["sha256"]:
failures.append(f"packages/fonts/{rel}: sha256 does not match fonts.lock.json")
if not rel.endswith(".css"):
continue
text = blob.decode()
if "?v=" in text:
failures.append(f"packages/fonts/{rel}: contains a ?v= cache-bust token")
for face in parse_stylesheet(text):
if not face["url"].startswith("../files/"):
failures.append(
f"packages/fonts/{rel}: url {face['url']!r} is not a relative ../files/ path"
)
continue
target = f"files/{face['url'][len('../files/') :]}"
if not (PACKAGE_ROOT / target).is_file():
failures.append(
f"packages/fonts/{rel} references a file that does not exist: {target}"
)
referenced.setdefault(target, []).append(rel)
if face["display"] != "swap":
failures.append(
f"packages/fonts/{rel}: {target} declares font-display {face['display']!r}"
)
previous = declared_ranges.get(target)
if previous is not None and previous != face["unicodeRange"]:
failures.append(f"{target}: declared with two different unicode-range values")
declared_ranges[target] = face["unicodeRange"]
for orphan in sorted(locked_files - set(referenced)):
failures.append(f"font is shipped but no stylesheet claims it: {orphan}")
for rel, rendered in (
("css/variables.css", render_variables_stylesheet()),
("css/locale-fallbacks.css", render_locale_fallbacks_stylesheet()),
):
path = PACKAGE_ROOT / rel
if path.is_file() and path.read_text() != rendered:
failures.append(f"packages/fonts/{rel}: does not match what build_fonts.py would emit")
variables_css = PACKAGE_ROOT / "css" / "variables.css"
if variables_css.is_file():
base_chain = re.search(r"--font-sans:(?P<value>[^;]*);", variables_css.read_text())
if base_chain is None:
failures.append("packages/fonts/css/variables.css: no --font-sans declaration")
else:
value = base_chain.group("value")
for family in FAMILIES:
if family.stylesheet in LATIN_CORE_STYLESHEETS:
continue
if family.css_family in UNCHAINED_SCRIPT_FAMILIES:
continue
if f"'{family.css_family}'" not in value:
failures.append(
f"packages/fonts/css/variables.css: base --font-sans does not name "
f"{family.css_family!r}, so its unicode-range gate can never fire"
)
failures.extend(check_variable_families(variables_css.read_text()))
if MANIFEST_PATH.is_file():
manifest = json.loads(MANIFEST_PATH.read_text())
manifest_sheets = [manifest["entrypoints"]["variables"]]
manifest_sheets.append(manifest["entrypoints"]["localeFallbacks"])
manifest_sheets.extend(manifest["entrypoints"]["latinCore"].values())
manifest_sheets.extend(manifest["entrypoints"]["scripts"].values())
manifest_files: set[str] = set()
for entry in manifest["families"]:
manifest_sheets.append(entry["stylesheet"])
manifest_files |= {f"files/{face['file']}" for face in entry["faces"]}
for sheet in sorted(set(manifest_sheets)):
if not (PACKAGE_ROOT / sheet).is_file():
failures.append(f"manifest.json names a stylesheet that does not exist: {sheet}")
if manifest_files != locked_files:
failures.append(
f"manifest.json lists {len(manifest_files)} faces but fonts.lock.json lists "
f"{len(locked_files)}"
)
per_family_ranges: dict[str, list[tuple[str, set[int]]]] = {}
for rel in sorted(locked_files):
path = PACKAGE_ROOT / rel
if not path.is_file():
continue
blob = path.read_bytes()
recorded = lock["files"][rel]
if recorded["sha256"] != sha256_bytes(blob):
failures.append(f"{rel}: sha256 does not match fonts.lock.json")
continue
if recorded["bytes"] != len(blob):
failures.append(f"{rel}: byte count does not match fonts.lock.json")
family_dir = rel.split("/")[1]
family = FAMILY_BY_DIR.get(family_dir)
if family is None:
failures.append(f"{rel}: no Family entry claims this directory")
continue
locked_family = lock["families"][family.out_dir]
weight = _weight_from_filename(family, Path(rel).name)
if weight is None:
failures.append(f"{rel}: filename does not match any weight of {family.out_dir}")
continue
locked_face = locked_family["faces"][weight]
font = TTFont(io.BytesIO(blob), lazy=True)
try:
gasp = font["gasp"].gaspRange if "gasp" in font else None
expected_gasp = {int(k): v for k, v in locked_family["gasp"].items()}
if gasp != expected_gasp:
failures.append(f"{rel}: gasp is {gasp}, expected {expected_gasp}")
if "gasp" in font and font["gasp"].version != 1:
failures.append(f"{rel}: gasp version is {font['gasp'].version}, expected 1")
for tag, size in locked_face["hinting"].items():
if tag not in font:
failures.append(f"{rel}: hinting table {tag!r} is missing")
elif len(font.reader[tag]) != size:
failures.append(
f"{rel}: {tag!r} is {len(font.reader[tag])} B, expected {size} B"
)
if metric_signature(font) != locked_face["metrics"]:
failures.append(f"{rel}: vertical metrics do not match fonts.lock.json")
try:
check_no_reserved_name(font, rel)
except SystemExit as exc:
failures.append(str(exc))
if not (font["name"].getDebugName(6) or "").startswith("Fluxer"):
failures.append(f"{rel}: PostScript name is not a Fluxer name")
covered = set(font.getBestCmap())
finally:
font.close()
declared = declared_ranges.get(rel)
if declared is None:
continue
if declared == "":
if family.stylesheet not in LATIN_CORE_STYLESHEETS:
failures.append(
f"{rel}: no unicode-range declared, but {family.stylesheet} is lazy"
)
continue
try:
claimed = parse_unicode_range(declared)
except ValueError as exc:
failures.append(f"{rel}: {exc}")
continue
if claimed != covered:
failures.append(
f"{rel}: unicode-range claims {len(claimed)} codepoints but the font encodes "
f"{len(covered)} (+{len(claimed - covered)} phantom, -{len(covered - claimed)} unreachable)"
)
per_family_ranges.setdefault(f"{family.out_dir}/{weight}", []).append((rel, claimed))
for key, chunks in sorted(per_family_ranges.items()):
family_dir, weight = key.rsplit("/", 1)
seen: dict[int, str] = {}
overlapped = False
for rel, claimed in chunks:
for cp in claimed:
if cp in seen:
failures.append(f"U+{cp:04X} is claimed by both {seen[cp]} and {rel}")
overlapped = True
break
seen[cp] = rel
if overlapped:
break
locked_total = lock["families"][family_dir]["faces"][weight]["codepoints"]
if not overlapped and len(seen) != locked_total:
failures.append(
f"{family_dir}/{weight}: the {len(chunks)} chunk(s) cover {len(seen)} codepoints, "
f"but fonts.lock.json records {locked_total}"
)
transfer_report: list[str] = []
for family_dir in sorted(TRANSFER_SAMPLES):
recorded = lock["families"].get(family_dir, {}).get("transfer")
if recorded is None:
failures.append(f"{family_dir}: fonts.lock.json records no transfer table")
continue
chunks = [
(lock["files"][rel]["bytes"], claimed)
for rel, claimed in per_family_ranges.get(f"{family_dir}/Regular", [])
]
if not chunks:
failures.append(f"{family_dir}: no Regular chunks to measure the transfer table on")
continue
measured = transfer_table(chunks, recorded["unslicedBytes"], family_dir)
if measured != recorded:
failures.append(
f"{family_dir}: the transfer table recomputed from the shipped tree does not match "
f"fonts.lock.json ({measured} vs {recorded})"
)
failures.extend(check_slicing_decision(family_dir, measured))
row = " ".join(
f"{label} {case['bytes']:,} B/{case['chunks']}c ({case['ratio']}x)"
for label, case in measured["scenarios"].items()
)
transfer_report.append(f" {family_dir:18} unsliced {recorded['unslicedBytes']:,} B {row}")
if failures:
print(f"FAIL ({len(failures)} problem(s)):")
for line in failures:
print(f" - {line}")
return 1
total = sum(entry["bytes"] for entry in lock["files"].values())
print(
f"OK: {len(locked_files)} WOFF2 files ({total / 1048576:.1f} MiB) and "
f"{len(lock['stylesheets'])} generated stylesheets match fonts.lock.json; "
"gasp, rename, hinting, metric and unicode-range invariants hold."
)
print("\nbytes fetched per sliced family (ratio = unsliced face / fetched):")
for line in transfer_report:
print(line)
return 0
def _weight_from_filename(family: Family, filename: str) -> str | None:
stem = filename[: -len(".woff2")]
if not stem.startswith(f"{family.out_dir}-"):
return None
remainder = stem[len(family.out_dir) + 1 :]
weight = remainder.split(".")[0]
return weight if weight in family.weights else None
def main() -> int:
parser = argparse.ArgumentParser(description="Build the font package in packages/fonts/.")
parser.add_argument(
"--verify",
action="store_true",
help="offline: check the checked-in tree against fonts.lock.json and re-assert invariants",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="fetch and build, report the diff, but write nothing",
)
parser.add_argument(
"--cache-dir",
type=Path,
default=Path(__file__).resolve().parent / ".cache",
help="where upstream downloads are kept (default: tools/fonts/.cache, git-ignored)",
)
parser.add_argument(
"--workers",
type=int,
default=min(8, os.cpu_count() or 1),
help="parallel face builds (default: min(8, cpu count); 1 disables the pool)",
)
args = parser.parse_args()
if args.verify:
if args.dry_run:
parser.error("--verify and --dry-run are mutually exclusive")
return verify()
return build(args.cache_dir.resolve(), args.dry_run, max(1, args.workers))
if __name__ == "__main__":
sys.exit(main())
Reference in new issue View Git Blame Copy Permalink