mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-07 19:22:14 +09:00
282 lines
12 KiB
Python
282 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import gzip
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import brotli
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
import build_fonts as bf # noqa: E402
|
|
|
|
CACHE = bf.TOOL_ROOT / ".cache"
|
|
SFNT_CACHE = CACHE / "measure"
|
|
|
|
CORPUS: dict[str, dict[str, str]] = {
|
|
"FluxerSansSC": {
|
|
"one glyph (文)": "文",
|
|
"display name": "小明",
|
|
"three display names": "小明 李伟 王芳",
|
|
"README sentence": "这是简体中文的示例文本",
|
|
"chat message": "我们今天下午三点开个会吧，主要讨论新版本的发布计划和测试进度。",
|
|
"chat paragraph": (
|
|
"我们今天下午三点开个会吧，主要讨论新版本的发布计划和测试进度。"
|
|
"如果有问题，可以随时在这个频道里留言，我会尽快回复大家。谢谢！"
|
|
),
|
|
"UI labels": (
|
|
"设置 通知 频道 消息 好友 语音 视频 屏幕共享 服务器 成员 权限 角色 邀请 退出 "
|
|
"静音 耳机 麦克风 摄像头 主题 语言 隐私 安全 账号 个人资料 状态 在线 离线 忙碌 勿扰"
|
|
),
|
|
},
|
|
"FluxerSansTC": {
|
|
"one glyph (文)": "文",
|
|
"display name": "小明",
|
|
"three display names": "小明 李偉 王芳",
|
|
"README sentence": "這是繁體中文的示例文本",
|
|
"chat message": "我們今天下午三點開個會吧，主要討論新版本的發佈計畫和測試進度。",
|
|
"chat paragraph": (
|
|
"我們今天下午三點開個會吧，主要討論新版本的發佈計畫和測試進度。"
|
|
"如果有問題，可以隨時在這個頻道裡留言，我會盡快回覆大家。謝謝！"
|
|
),
|
|
"UI labels": (
|
|
"設定 通知 頻道 訊息 好友 語音 視訊 螢幕分享 伺服器 成員 權限 角色 邀請 離開 "
|
|
"靜音 耳機 麥克風 攝影機 主題 語言 隱私 安全 帳號 個人資料 狀態 線上 離線 忙碌 勿擾"
|
|
),
|
|
},
|
|
"FluxerSansJP": {
|
|
"one glyph (語)": "語",
|
|
"display name": "田中太郎",
|
|
"three display names": "田中太郎 佐藤花子 鈴木一郎",
|
|
"README sentence": "これは日本語のサンプルテキストです",
|
|
"chat message": "今日の午後三時から会議をしましょう。新しいバージョンの計画を話し合います。",
|
|
"chat paragraph": (
|
|
"今日の午後三時から会議をしましょう。主に新しいバージョンのリリース計画と"
|
|
"テストの進捗について話し合います。質問があれば、いつでもこのチャンネルに"
|
|
"書き込んでください。なるべく早く返信します。よろしくお願いします。"
|
|
),
|
|
"UI labels": (
|
|
"設定 通知 チャンネル メッセージ フレンド ボイス ビデオ 画面共有 サーバー メンバー "
|
|
"権限 ロール 招待 退出 ミュート ヘッドセット マイク カメラ テーマ 言語 "
|
|
"プライバシー セキュリティ アカウント プロフィール ステータス オンライン オフライン 取り込み中 応答不可"
|
|
),
|
|
},
|
|
"FluxerSansKR": {
|
|
"one syllable (한)": "한",
|
|
"display name": "김민준",
|
|
"three display names": "김민준 이서연 박지훈",
|
|
"README sentence": "이것은 한국어 샘플 텍스트입니다",
|
|
"chat message": "오늘 오후 세 시에 회의를 합시다. 새 버전의 출시 계획을 이야기합니다.",
|
|
"chat paragraph": (
|
|
"오늘 오후 세 시에 회의를 합시다. 주로 새 버전의 출시 계획과 테스트 진행 상황에 "
|
|
"대해 이야기할 예정입니다. 질문이 있으면 언제든지 이 채널에 남겨 주세요. "
|
|
"최대한 빨리 답장하겠습니다. 감사합니다."
|
|
),
|
|
"UI labels": (
|
|
"설정 알림 채널 메시지 친구 음성 영상 화면 공유 서버 멤버 권한 역할 초대 나가기 "
|
|
"음소거 헤드셋 마이크 카메라 테마 언어 개인정보 보안 계정 프로필 상태 온라인 오프라인 방해 금지"
|
|
),
|
|
},
|
|
}
|
|
|
|
CJK_FAMILIES = tuple(CORPUS)
|
|
|
|
|
|
def renamed_reference(family: bf.Family, roots: dict[str, Path]) -> bytes:
|
|
"""The renamed Regular SFNT, cached: the WOFF2 round-trip costs ~60 s a face."""
|
|
SFNT_CACHE.mkdir(parents=True, exist_ok=True)
|
|
cached = SFNT_CACHE / f"{family.prefix}-Regular.sfnt"
|
|
if cached.is_file():
|
|
return cached.read_bytes()
|
|
src = roots[family.source] / family.src_subdir / f"{family.prefix}-Regular.woff2"
|
|
print(f" renaming {family.prefix}-Regular (~60 s, cached afterwards)", flush=True)
|
|
sfnt = bf.renamed_sfnt(src, family.gasp)
|
|
cached.write_bytes(sfnt)
|
|
return sfnt
|
|
|
|
|
|
def glyph_id_order(sfnt: bytes) -> list[int]:
|
|
"""This face's codepoints in glyph ID order (its inherited CID tiering)."""
|
|
import io
|
|
|
|
font = bf.TTFont(io.BytesIO(sfnt), lazy=True)
|
|
try:
|
|
cmap = font.getBestCmap()
|
|
order = font.getGlyphOrder()
|
|
rank = {name: index for index, name in enumerate(order)}
|
|
return [cp for _, cp in sorted((rank[name], cp) for cp, name in cmap.items())]
|
|
finally:
|
|
font.close()
|
|
|
|
|
|
def plan_freq000(
|
|
sfnt: bytes, codepoints: set[int], budget: int, head: int
|
|
) -> list[tuple[int, ...]]:
|
|
"""Chunk 000 = the `head` lowest-glyph-ID codepoints; the rest in block order."""
|
|
ordered = [cp for cp in glyph_id_order(sfnt) if cp in codepoints]
|
|
common = ordered[:head]
|
|
if not common:
|
|
raise SystemExit("empty head chunk")
|
|
rest = codepoints - set(common)
|
|
tail = bf.plan_chunks(rest, budget) if rest else []
|
|
return [tuple(sorted(common)), *tail]
|
|
|
|
|
|
def stylesheet_bytes(family: bf.Family, plan: list[tuple[int, ...]]) -> dict[str, int]:
|
|
"""Render the real stylesheet for this plan and compress it as the app would.
|
|
|
|
Chunk-major, every weight, `render_face_rule` verbatim — so the raw/gzip/
|
|
brotli numbers are the ones a browser would actually download, not an
|
|
estimate of the `unicode-range` text alone.
|
|
"""
|
|
rules = [bf.SPDX_CSS_HEADER, bf.GENERATED_CSS_HEADER, ""]
|
|
for index, chunk in enumerate(plan):
|
|
rendered_range = bf.format_unicode_range(list(chunk))
|
|
for weight in family.weights:
|
|
rules.append(
|
|
bf.render_face_rule(
|
|
family,
|
|
{
|
|
"weight": weight,
|
|
"path": f"{family.out_dir}/{family.prefix}-{weight}.{index:03d}.woff2",
|
|
"unicodeRange": rendered_range,
|
|
},
|
|
)
|
|
)
|
|
text = ("\n".join([*rules, ""])).encode()
|
|
return {
|
|
"raw": len(text),
|
|
"gzip": len(gzip.compress(text, 9, mtime=0)),
|
|
"brotli": len(brotli.compress(text, quality=11)),
|
|
}
|
|
|
|
|
|
def build_plan(family: bf.Family, sfnt: bytes, plan: list[tuple[int, ...]]) -> list[dict]:
|
|
"""Really subset and really encode every chunk, so the bytes are measured."""
|
|
chunks = []
|
|
for index, requested in enumerate(plan):
|
|
produced = bf.subset_face(sfnt, tuple(requested))
|
|
chunks.append({"index": index, "codepoints": set(requested), "bytes": len(produced)})
|
|
print(f" chunk {index:03d}: {len(requested)} cp -> {len(produced)} B", flush=True)
|
|
return chunks
|
|
|
|
|
|
def fetched(chunks: list[dict], text: str) -> tuple[int, int]:
|
|
"""Bytes and chunk count a browser fetches to render `text` from this family.
|
|
|
|
A chunk is fetched when its `unicode-range` intersects the text — which is
|
|
exactly the browser's rule, and the reason spaces matter: U+0020 lives in the
|
|
punctuation chunk, so any sentence containing one pays for it.
|
|
"""
|
|
wanted = set(ord(c) for c in text)
|
|
total = 0
|
|
count = 0
|
|
for chunk in chunks:
|
|
if chunk["codepoints"] & wanted:
|
|
total += chunk["bytes"]
|
|
count += 1
|
|
return total, count
|
|
|
|
|
|
def measure_family(family: bf.Family, roots: dict[str, Path], strategies: list[str]) -> dict:
|
|
sfnt = renamed_reference(family, roots)
|
|
union: set[int] = set()
|
|
for name in family.source_files():
|
|
union |= bf.face_cmap(roots[family.source] / family.src_subdir / name)
|
|
budget, note = bf.calibrate_chunk_size(sfnt, sorted(union), bf.CHUNK_TARGET_BYTES)
|
|
print(f"\n{family.out_dir}: {len(union)} codepoints, {note}")
|
|
|
|
result: dict[str, dict] = {}
|
|
for strategy in strategies:
|
|
if strategy == "block":
|
|
plan = bf.plan_chunks(union, budget)
|
|
elif strategy.startswith("freq"):
|
|
plan = plan_freq000(sfnt, union, budget, int(strategy[4:]))
|
|
else:
|
|
raise SystemExit(f"unknown strategy {strategy!r}")
|
|
print(f" {strategy}: {len(plan)} chunks", flush=True)
|
|
chunks = build_plan(family, sfnt, plan)
|
|
result[strategy] = {
|
|
"chunks": len(plan),
|
|
"css": stylesheet_bytes(family, plan),
|
|
"full": sum(c["bytes"] for c in chunks),
|
|
"scenarios": {
|
|
label: fetched(chunks, text) for label, text in CORPUS[family.out_dir].items()
|
|
},
|
|
}
|
|
return result
|
|
|
|
|
|
def report(family_id: str, data: dict[str, dict]) -> None:
|
|
strategies = list(data)
|
|
base = strategies[0]
|
|
print(f"\n=== {family_id} ===")
|
|
header = f"{'scenario':<24}" + "".join(f"{s:>22}" for s in strategies)
|
|
print(header)
|
|
print("-" * len(header))
|
|
for label in CORPUS[family_id]:
|
|
row = f"{label:<24}"
|
|
base_bytes = data[base]["scenarios"][label][0]
|
|
for strategy in strategies:
|
|
got, count = data[strategy]["scenarios"][label]
|
|
ratio = f" ({base_bytes / got:.2f}x)" if strategy != base and got else ""
|
|
row += f"{f'{got:,} B/{count}c{ratio}':>22}"
|
|
print(row)
|
|
for key, label in (("full", "full coverage"), ("chunks", "chunk count")):
|
|
row = f"{label:<24}"
|
|
for strategy in strategies:
|
|
value = data[strategy][key]
|
|
row += f"{f'{value:,}':>22}"
|
|
print(row)
|
|
for field in ("raw", "gzip", "brotli"):
|
|
row = f"{f'stylesheet {field}':<24}"
|
|
for strategy in strategies:
|
|
value = data[strategy]["css"][field]
|
|
row += f"{f'{value:,} B':>22}"
|
|
print(row)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Measure CJK chunking strategies.")
|
|
parser.add_argument("--family", action="append", choices=CJK_FAMILIES)
|
|
parser.add_argument("--strategies", default="block,freq1500,freq3000,freq6000")
|
|
parser.add_argument("--json", type=Path, help="also write the raw numbers here")
|
|
args = parser.parse_args()
|
|
|
|
families = [bf.FAMILY_BY_DIR[f] for f in (args.family or CJK_FAMILIES)]
|
|
strategies = [s.strip() for s in args.strategies.split(",") if s.strip()]
|
|
roots = bf.resolve_sources(CACHE, tuple(families))
|
|
|
|
everything: dict[str, dict] = {}
|
|
for family in families:
|
|
everything[family.out_dir] = measure_family(family, roots, strategies)
|
|
report(family.out_dir, everything[family.out_dir])
|
|
|
|
if args.json:
|
|
args.json.write_text(
|
|
json.dumps(
|
|
{
|
|
fam: {
|
|
s: {
|
|
**{k: v for k, v in d.items() if k != "scenarios"},
|
|
"scenarios": {k: list(v) for k, v in d["scenarios"].items()},
|
|
}
|
|
for s, d in strategies_data.items()
|
|
}
|
|
for fam, strategies_data in everything.items()
|
|
},
|
|
indent="\t",
|
|
)
|
|
+ "\n"
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|