pepepper/fluxer
1
0
Fork 0
mirror of https://github.com/fluxerapp/fluxer synced 2026-10-07 19:22:14 +09:00
Code Issues Packages Projects Releases Wiki Activity

docs: simplify the reference and tighten the verifier (#2736)

Browse Source
This commit is contained in:
Hampus
2026-09-13 17:38:50 +02:00
committed by GitHub
parent 6af33c7188
commit 7375ac9d80
111 changed files with 1616 additions and 2430 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fluxer_docs/scripts/DocsRouteHeaders.ts
+120
View File
@@ -0,0 +1,120 @@
import assert from 'node:assert/strict';
import {parseFrontmatter} from '@astrojs/markdown-remark';
import {createProcessor} from '@mdx-js/mdx';
import type {Nodes, Root} from 'mdast';
import type {MdxJsxAttribute, MdxJsxFlowElement, MdxJsxTextElement} from 'mdast-util-mdx-jsx';
import remarkGfm from 'remark-gfm';
import {HTTP_METHODS, type MarkdownPage} from './DocsSource.ts';
export interface DocsRouteHeader {
readonly method: string;
readonly path: string;
readonly file: string;
readonly bot: boolean;
readonly unauthenticated: boolean;
readonly line: number;
readonly endLine: number;
}
const STRING_ATTRIBUTES = new Set(['method', 'path', 'oauth2']);
const BOOLEAN_ATTRIBUTES = new Set(['bot', 'unauthenticated', 'auditReason', 'mfa']);
const markdownProcessor = createProcessor({format: 'md', remarkPlugins: [remarkGfm]});
const mdxProcessor = createProcessor({format: 'mdx', remarkPlugins: [remarkGfm]});
function readAttributeValue(attribute: MdxJsxAttribute, location: string): string | boolean {
const value = attribute.value;
if (value === null || value === undefined) return true;
if (typeof value === 'string') return value;
const body = value.data?.estree?.body;
if (body?.length === 1 && body[0].type === 'ExpressionStatement') {
const expression = body[0].expression;
if (
expression.type === 'Literal' &&
(typeof expression.value === 'string' || typeof expression.value === 'boolean')
) {
return expression.value;
}
if (
expression.type === 'TemplateLiteral' &&
expression.expressions.length === 0 &&
expression.quasis.length === 1 &&
typeof expression.quasis[0].value.cooked === 'string'
) {
return expression.quasis[0].value.cooked;
}
}
throw new Error(`${location}: RouteHeader ${attribute.name} requires a literal string or boolean`);
}
function readHeader(node: MdxJsxFlowElement | MdxJsxTextElement, file: string): DocsRouteHeader {
assert(node.position !== undefined, 'Parsed RouteHeader has no source position');
const line = node.position.start.line;
const endLine = node.position.end.line;
assert(Number.isInteger(line) && line > 0 && Number.isInteger(endLine) && endLine >= line);
const location = `${file}:${line}`;
const attributes = new Map<string, string | boolean>();
for (const attribute of node.attributes) {
if (attribute.type === 'mdxJsxExpressionAttribute') {
throw new Error(`${location}: RouteHeader spread attributes are unsupported`);
}
const name = attribute.name;
if (attributes.has(name)) {
throw new Error(`${location}: duplicate RouteHeader attribute ${name}`);
}
const isString = STRING_ATTRIBUTES.has(name);
if (!isString && !BOOLEAN_ATTRIBUTES.has(name)) {
throw new Error(`${location}: unknown RouteHeader attribute ${name}`);
}
const value = readAttributeValue(attribute, location);
if (isString ? typeof value !== 'string' : typeof value !== 'boolean') {
throw new Error(`${location}: RouteHeader ${name} must be ${isString ? 'a string' : 'a boolean'}`);
}
attributes.set(name, value);
}
const method = attributes.get('method');
const routePath = attributes.get('path');
if (typeof method !== 'string' || typeof routePath !== 'string') {
throw new Error(`${location}: RouteHeader requires method and path attributes`);
}
if (!HTTP_METHODS.has(method)) {
throw new Error(`${location}: unsupported RouteHeader method ${method}`);
}
if (!routePath.startsWith('/')) {
throw new Error(`${location}: RouteHeader path must be absolute`);
}
return {
method,
path: routePath,
file,
bot: attributes.get('bot') === true,
unauthenticated: attributes.get('unauthenticated') === true,
line,
endLine,
};
}
export function readRouteHeaders(page: MarkdownPage): Array<DocsRouteHeader> {
let tree: Root;
try {
const {content} = parseFrontmatter(page.source, {frontmatter: 'empty-with-spaces'});
const processor = page.file.endsWith('.mdx') ? mdxProcessor : markdownProcessor;
tree = processor.parse({value: content, path: page.file});
} catch (error) {
throw new Error(`${page.relativePath}: ${String(error)}`, {cause: error});
}
const headers: Array<DocsRouteHeader> = [];
const pending: Array<Nodes> = [tree];
while (pending.length > 0) {
const node = pending.pop();
assert(node !== undefined);
if ((node.type === 'mdxJsxFlowElement' || node.type === 'mdxJsxTextElement') && node.name === 'RouteHeader') {
headers.push(readHeader(node, page.relativePath));
}
if ('children' in node) {
for (let index = node.children.length - 1; index >= 0; index -= 1) {
pending.push(node.children[index]);
}
}
}
return headers;
}
fluxer_docs/scripts/DocsSource.ts
+49 -3
View File
@@ -1,8 +1,16 @@
import {readdir} from 'node:fs/promises';
import {readdir, readFile} from 'node:fs/promises';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
export const DOCS_ROOT = fileURLToPath(new URL('../src/content/docs/', import.meta.url));
export const HTTP_METHODS: ReadonlySet<string> = new Set(['GET', 'HEAD', 'POST', 'PATCH', 'PUT', 'DELETE', 'OPTIONS']);
export interface MarkdownPage {
readonly file: string;
readonly relativePath: string;
readonly source: string;
readonly lines: ReadonlyArray<string>;
}
export async function listMarkdownFiles(directory: string): Promise<Array<string>> {
const files: Array<string> = [];
@@ -13,11 +21,49 @@ export async function listMarkdownFiles(directory: string): Promise<Array<string
const resolved = path.join(directory, entry.name);
if (entry.isDirectory()) {
files.push(...(await listMarkdownFiles(resolved)));
} else if (/\.mdx?$/u.test(entry.name)) {
} else if (entry.isFile() && /\.mdx?$/u.test(entry.name)) {
files.push(resolved);
}
}
return files;
return files.sort();
}
export async function readMarkdownPages(directory: string): Promise<Array<MarkdownPage>> {
const pages: Array<MarkdownPage> = [];
for (const file of await listMarkdownFiles(directory)) {
const source = await readFile(file, 'utf8');
pages.push({
file,
relativePath: path.relative(directory, file).split(path.sep).join('/'),
source,
lines: source.split('\n'),
});
}
return pages;
}
export function splitTableRow(line: string): Array<string> {
const row = line.trim();
const cells: Array<string> = [];
let start = row.startsWith('|') ? 1 : 0;
for (let index = start; index < row.length; index += 1) {
if (row[index] === '\\' && (row[index + 1] === '\\' || row[index + 1] === '|')) {
index += 1;
continue;
}
if (row[index] === '|') {
cells.push(row.slice(start, index).trim());
start = index + 1;
}
}
if (start < row.length) {
cells.push(row.slice(start).trim());
}
return cells;
}
export function routeShape(method: string, routePath: string): string {
return `${method} ${routePath.split('?')[0].replace(/\{[^}]*\}/gu, '{}')}`;
}
export function slugifyHeading(heading: string): string {
fluxer_docs/scripts/DocsTableWidth.ts
+18 -34
View File
@@ -13,6 +13,7 @@ import {
TABLE_MAX_IDENT_CHARS,
TABLE_WIDE_TIER_PX,
} from '../src/table/DocsTableMetrics.ts';
import {splitTableRow} from './DocsSource.ts';
export {
columnWidthPercents,
@@ -53,34 +54,9 @@ export interface DocsTable {
readonly nonParallelReason: string;
}
function splitRow(line: string): Array<string> {
const cells: Array<string> = [];
let current = '';
const body = line.trim().replace(/^\|/u, '').replace(/\|$/u, '');
for (let index = 0; index < body.length; index += 1) {
const character = body[index];
if (character === '\\' && index + 1 < body.length) {
current += body[index + 1];
index += 1;
continue;
}
if (character === '|') {
cells.push(current);
current = '';
continue;
}
current += character;
}
cells.push(current);
return cells.map((cell) => cell.trim());
}
function isDelimiter(line: string): boolean {
const trimmed = line.trim();
if (!trimmed.startsWith('|')) {
return false;
}
return /^\|[\s:|-]+\|?$/u.test(trimmed) && trimmed.includes('-');
function isDelimiter(line: string, columns: number): boolean {
const cells = splitTableRow(line);
return cells.length === columns && cells.every((cell) => /^:?-+:?$/u.test(cell));
}
function stripInline(text: string): string {
@@ -95,7 +71,8 @@ function stripInline(text: string): string {
.replace(/&nbsp;/gu, ' ')
.replace(/&lt;/gu, '<')
.replace(/&gt;/gu, '>')
.replace(/&amp;/gu, '&');
.replace(/&amp;/gu, '&')
.replace(/\\([!-/:-@[-`{-~])/gu, '$1');
}
function pieces(cell: string): Array<Piece> {
@@ -107,7 +84,10 @@ function pieces(cell: string): Array<Piece> {
if (match.index > last) {
out.push({text: stripInline(cell.slice(last, match.index)), code: false});
}
out.push({text: match[1], code: true});
out.push({
text: match[1].replace(/\\([\\|])/gu, (escaped, character: string) => (character === '|' ? character : escaped)),
code: true,
});
last = pattern.lastIndex;
match = pattern.exec(cell);
}
@@ -127,7 +107,7 @@ function cellKind(cell: string): string {
if (parts.length === 0) {
return 'empty';
}
if (parts.every((piece) => piece.code)) {
if (parts.every((piece) => piece.code || /^[\s,]+$/u.test(piece.text))) {
return 'code';
}
if (parts.every((piece) => !piece.code)) {
@@ -151,16 +131,20 @@ export function extractTables(source: string): Array<DocsTable> {
index += 1;
continue;
}
if (!lines[index].trim().startsWith('|') || index + 1 >= lines.length || !isDelimiter(lines[index + 1])) {
if (!lines[index].trim().startsWith('|') || index + 1 >= lines.length) {
index += 1;
continue;
}
const header = splitTableRow(lines[index]);
if (header.length === 0 || !isDelimiter(lines[index + 1], header.length)) {
index += 1;
continue;
}
const header = splitRow(lines[index]);
const start = index;
index += 2;
const body: Array<Array<string>> = [];
while (index < lines.length && lines[index].trim().startsWith('|')) {
body.push(splitRow(lines[index]));
body.push(splitTableRow(lines[index]));
index += 1;
}
const columns = header.length;
fluxer_docs/scripts/VerifyDocsCoverage.ts
+262 -277
View File
@@ -7,20 +7,21 @@ import {mkdir, mkdtemp, readdir, readFile, rm, writeFile} from 'node:fs/promises
import {tmpdir} from 'node:os';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {parseArgs} from 'node:util';
import {extractRoutesFromControllers} from '@fluxer/openapi/src/extractors/RouteExtractor';
import type {OpenAPIDocument} from '@fluxer/openapi/src/OpenAPITypes';
import {installerChecksumLine} from '../src/installer/InstallerDigest.ts';
import {DOCS_ROOT, listMarkdownFiles} from './DocsSource.ts';
import {type DocsRouteHeader, readRouteHeaders} from './DocsRouteHeaders.ts';
import {DOCS_ROOT, HTTP_METHODS, type MarkdownPage, readMarkdownPages, routeShape} from './DocsSource.ts';
const {values: options} = parseArgs({options: {'source-only': {type: 'boolean', default: false}}});
const REPO_ROOT = fileURLToPath(new URL('../../', import.meta.url));
const MAIN_SPEC = path.join(REPO_ROOT, 'fluxer_api/src/api/openapi/openapi.json');
const ADMIN_SPEC = path.join(REPO_ROOT, 'fluxer_admin/openapi-admin.json');
const MEDIA_PROXY_SERVER_DIR = path.join(REPO_ROOT, 'fluxer_media_proxy/src/server');
const ROUTE_HEADER_PATTERN = /<RouteHeader\s+([^>]*?)\/>/gu;
const ATTRIBUTE_PATTERN = /(\w+)\s*=\s*"([^"]*)"/gu;
const HTTP_METHODS = new Set(['GET', 'HEAD', 'POST', 'PATCH', 'PUT', 'DELETE', 'OPTIONS']);
const BLUESKY_OAUTH_CONTROLLER = 'fluxer_api/src/api/bluesky/BlueskyOAuthController.ts';
const DOWNLOAD_CONTROLLER = 'fluxer_api/src/api/download/DownloadController.ts';
@@ -222,15 +223,6 @@ const MEDIA_PROXY_ASSET_PREFIXES = new Map([
['guilds', 'fn parse_guild_member_asset_path'],
]);
interface DocumentedRoute {
readonly method: string;
readonly path: string;
readonly file: string;
readonly bot: boolean;
readonly unauthenticated: boolean;
readonly oauth2: string | null;
}
interface SpecOperation {
readonly method: string;
readonly path: string;
@@ -238,6 +230,11 @@ interface SpecOperation {
readonly security: ReadonlyArray<Record<string, Array<string>>> | null;
}
interface EffectiveSpecSecurity {
readonly schemes: ReadonlySet<string>;
readonly allowsAnonymous: boolean;
}
function stripVersionPrefix(routePath: string): string {
if (routePath === '/v1') {
return '/';
@@ -248,62 +245,35 @@ function stripVersionPrefix(routePath: string): string {
return routePath;
}
function shapeOf(method: string, routePath: string): string {
const withoutQuery = routePath.split('?')[0];
return `${method} ${withoutQuery.replace(/\{[^}]*\}/gu, '{}')}`;
}
async function documentedRoutes(): Promise<Array<DocumentedRoute>> {
const files = await listMarkdownFiles(DOCS_ROOT);
const routes: Array<DocumentedRoute> = [];
for (const file of files) {
const source = await readFile(file, 'utf8');
for (const match of source.matchAll(ROUTE_HEADER_PATTERN)) {
const attributes = new Map<string, string>();
for (const attribute of match[1].matchAll(ATTRIBUTE_PATTERN)) {
attributes.set(attribute[1], attribute[2]);
}
const method = attributes.get('method');
const routePath = attributes.get('path');
if (method == null || routePath == null) {
continue;
}
const bareFlags = new Set(
match[1]
.replace(/\w+\s*=\s*"[^"]*"/gu, ' ')
.split(/\s+/u)
.filter((token) => token.length > 0),
function indexDocumentedRoutes(routes: ReadonlyArray<DocsRouteHeader>): Map<string, DocsRouteHeader> {
const index = new Map<string, DocsRouteHeader>();
for (const route of routes) {
const key = routeShape(route.method, stripVersionPrefix(route.path));
const existing = index.get(key);
if (existing !== undefined) {
throw new Error(
`Duplicate documented route ${key}: ${existing.file}:${existing.line} and ${route.file}:${route.line}`,
);
routes.push({
method,
path: routePath,
file: path.relative(DOCS_ROOT, file),
bot: bareFlags.has('bot'),
unauthenticated: bareFlags.has('unauthenticated'),
oauth2: attributes.get('oauth2') ?? null,
});
}
index.set(key, route);
}
return routes;
return index;
}
interface AliasRoute {
readonly shape: string;
readonly file: string;
readonly successor: string;
}
const ALIAS_TABLE_HEADER = '| Method | Deprecated path | Successor |';
const ALIAS_ROW_PATTERN =
/^\|\s*(GET|HEAD|POST|PATCH|PUT|DELETE|OPTIONS)\s*\|\s*`(\/[^`]+)`\s*\|\s*\[([^\]]+)\]\([^)]+\)\s*\|\s*$/u;
/^\|\s*(GET|HEAD|POST|PATCH|PUT|DELETE|OPTIONS)\s*\|\s*`(\/[^`]+)`\s*\|\s*\[[^\]]+\]\([^)]+\)\s*\|\s*$/u;
async function aliasDocumentedRoutes(): Promise<Array<AliasRoute>> {
const files = await listMarkdownFiles(DOCS_ROOT);
function aliasDocumentedRoutes(pages: ReadonlyArray<MarkdownPage>): Array<AliasRoute> {
const aliases: Array<AliasRoute> = [];
for (const file of files) {
const source = await readFile(file, 'utf8');
for (const {relativePath: file, lines} of pages) {
let inTable = false;
for (const line of source.split('\n')) {
for (const line of lines) {
if (line.trim() === ALIAS_TABLE_HEADER) {
inTable = true;
continue;
@@ -317,9 +287,8 @@ async function aliasDocumentedRoutes(): Promise<Array<AliasRoute>> {
continue;
}
aliases.push({
shape: shapeOf(row[1], stripVersionPrefix(row[2])),
file: path.relative(DOCS_ROOT, file),
successor: row[3],
shape: routeShape(row[1], stripVersionPrefix(row[2])),
file,
});
}
}
@@ -331,13 +300,9 @@ async function specOperations(specPath: string): Promise<Array<SpecOperation>> {
if (typeof spec !== 'object' || spec == null || !('paths' in spec)) {
throw new Error(`Spec has no paths: ${specPath}`);
}
const paths = (
spec as {
paths: Record<string, Record<string, {tags?: Array<string>; security?: Array<Record<string, Array<string>>>}>>;
}
).paths;
const document = spec as OpenAPIDocument;
const operations: Array<SpecOperation> = [];
for (const [routePath, item] of Object.entries(paths)) {
for (const [routePath, item] of Object.entries(document.paths)) {
for (const [method, operation] of Object.entries(item)) {
const upper = method.toUpperCase();
if (!HTTP_METHODS.has(upper)) {
@@ -347,7 +312,7 @@ async function specOperations(specPath: string): Promise<Array<SpecOperation>> {
method: upper,
path: stripVersionPrefix(routePath),
tags: operation.tags ?? [],
security: operation.security ?? null,
security: operation.security === undefined ? (document.security ?? null) : operation.security,
});
}
}
@@ -404,18 +369,19 @@ const astRoute = (route: {method: string; path: string}): string => {
`{${name}}${constraint == null ? '' : constraintTail(constraint)}`,
)
.replace(/\*/gu, '{wildcard}');
return shapeOf(route.method.toUpperCase(), stripVersionPrefix(templated));
return routeShape(route.method.toUpperCase(), stripVersionPrefix(templated));
};
const documented = await documentedRoutes();
const aliasDocumented = await aliasDocumentedRoutes();
const aliasShapes = new Map(aliasDocumented.map((alias) => [alias.shape, alias]));
const documentedFlags = new Map<string, {bot: boolean; unauthenticated: boolean}>();
for (const route of documented) {
documentedFlags.set(shapeOf(route.method, stripVersionPrefix(route.path)), {
bot: route.bot,
unauthenticated: route.unauthenticated,
});
const pages = await readMarkdownPages(DOCS_ROOT);
const documented = pages.flatMap(readRouteHeaders);
const aliasDocumented = aliasDocumentedRoutes(pages);
const aliasShapes = new Map<string, AliasRoute>();
for (const alias of aliasDocumented) {
const existing = aliasShapes.get(alias.shape);
if (existing !== undefined) {
throw new Error(`Duplicate deprecated alias ${alias.shape}: ${existing.file} and ${alias.file}`);
}
aliasShapes.set(alias.shape, alias);
}
const mediaProxySource = await (async () => {
const sources: Array<string> = [];
@@ -484,17 +450,11 @@ const mainDocumented = documented.filter(
const main = await specOperations(MAIN_SPEC);
const admin = await specOperations(ADMIN_SPEC);
const mainShapes = new Set(main.map((operation) => shapeOf(operation.method, operation.path)));
const adminShapes = new Set(admin.map((operation) => shapeOf(operation.method, operation.path)));
const mainShapes = new Set(main.map((operation) => routeShape(operation.method, operation.path)));
const adminShapes = new Set(admin.map((operation) => routeShape(operation.method, operation.path)));
const documentedMain = new Map<string, DocumentedRoute>();
for (const route of mainDocumented) {
documentedMain.set(shapeOf(route.method, stripVersionPrefix(route.path)), route);
}
const documentedAdmin = new Map<string, DocumentedRoute>();
for (const route of adminDocumented) {
documentedAdmin.set(shapeOf(route.method, stripVersionPrefix(route.path)), route);
}
const documentedMain = indexDocumentedRoutes(mainDocumented);
const documentedAdmin = indexDocumentedRoutes(adminDocumented);
const registered = new Map<string, string>();
for (const route of controllerRoutes) {
@@ -572,7 +532,7 @@ failures += section(
'present in the live spec but undocumented',
main
.filter((operation) => {
const shape = shapeOf(operation.method, operation.path);
const shape = routeShape(operation.method, operation.path);
if (documentedMain.has(shape)) {
return false;
}
@@ -598,7 +558,7 @@ failures += section(
.map((shape) => `${shape} is an alias row and a RouteHeader, which double counts it`)
.sort(),
);
console.log(` documented as a deprecated alias of a documented route: ${aliasShapes.size.toString()}`);
console.log(` documented in deprecated alias tables: ${aliasShapes.size.toString()}`);
const wronglyDocumented = [...DELIBERATELY_UNDOCUMENTED.entries()]
.filter(([shape]) => documentedMain.has(shape))
.map(([shape, reason]) => `${shape} must not be documented: ${reason}`);
@@ -616,7 +576,7 @@ for (const [shape, {reason}] of MAIN_SPEC_EXEMPT) {
console.log('media proxy');
const mediaProxyProblems: Array<string> = [];
for (const route of mediaProxyDocumented) {
const shape = shapeOf(route.method, route.path);
const shape = routeShape(route.method, route.path);
if (MEDIA_PROXY_ROUTES.has(shape)) {
continue;
}
@@ -781,9 +741,7 @@ console.log('enum names and error codes');
const enumRows: Array<{file: string; line: number; name: string}> = [];
const codeRows: Array<{file: string; line: number; code: string}> = [];
for (const file of await listMarkdownFiles(DOCS_ROOT)) {
const relative = path.relative(DOCS_ROOT, file);
const lines = (await readFile(file, 'utf8')).split('\n');
for (const {relativePath: relative, lines} of pages) {
let inEnumTable = false;
for (let i = 0; i < lines.length; i += 1) {
const line = lines[i];
@@ -963,9 +921,7 @@ console.log('rate limit buckets, limits and windows');
/([\d,]+) requests? per ([a-z0-9 ]+?)(?:,| for [^.]*?,) on the (?:shared )?`([a-z0-9_:@{}]+)` bucket/gu;
const problems: Array<string> = [];
let claims = 0;
for (const file of await listMarkdownFiles(DOCS_ROOT)) {
const relative = path.relative(DOCS_ROOT, file);
const lines = (await readFile(file, 'utf8')).split('\n');
for (const {relativePath: relative, lines} of pages) {
for (let i = 0; i < lines.length; i += 1) {
for (const claim of lines[i].matchAll(claimPattern)) {
claims += 1;
@@ -1328,15 +1284,14 @@ console.log('self-hosting guide against deploy/self-hosting');
const PIPE_TO_SHELL =
/(?:curl|wget|iwr|Invoke-WebRequest)[^\n|]*\|\s*(?:sudo\s+)?(?:sh|bash|zsh|iex|Invoke-Expression)\b/iu;
const docsPages = await listMarkdownFiles(DOCS_ROOT);
for (const [name, source] of installers) {
if (PIPE_TO_SHELL.test(source)) {
problems.push(`${name} pipes a download into a shell`);
}
}
for (const page of docsPages) {
if (PIPE_TO_SHELL.test(await readFile(page, 'utf8'))) {
problems.push(`${path.relative(DOCS_ROOT, page)} pipes a download into a shell`);
for (const {relativePath, source} of pages) {
if (PIPE_TO_SHELL.test(source)) {
problems.push(`${relativePath} pipes a download into a shell`);
}
}
@@ -1375,10 +1330,9 @@ console.log('self-hosting guide against deploy/self-hosting');
'src/components/InstallerChecksum.astro',
await readFile(fileURLToPath(new URL('../src/components/InstallerChecksum.astro', import.meta.url)), 'utf8'),
]);
for (const page of docsPages) {
const text = await readFile(page, 'utf8');
if (text.includes('install.sh') || text.includes('install.ps1')) {
digestBearing.push([path.relative(DOCS_ROOT, page), text]);
for (const {relativePath, source} of pages) {
if (source.includes('install.sh') || source.includes('install.ps1')) {
digestBearing.push([relativePath, source]);
}
}
for (const [where, text] of digestBearing) {
@@ -1389,148 +1343,10 @@ console.log('self-hosting guide against deploy/self-hosting');
}
}
const shellParse = spawnSync('sh', ['-n', path.join(INSTALLER_ROOT, 'install.sh')], {encoding: 'utf8'});
if (shellParse.error != null) {
problems.push(`sh -n could not run against install.sh: ${shellParse.error.message}`);
} else if (shellParse.status !== 0) {
problems.push(`sh -n rejects install.sh: ${shellParse.stderr.trim()}`);
}
const DOCKER_STUB = [
'#!/bin/sh',
'case "$1 $2" in',
" '--version ') echo 'Docker version 27.1.1, build stub' ;;",
" 'compose version') if [ \"$3\" = '--short' ]; then echo '2.30.3'; else echo 'v2.30.3'; fi ;;",
" 'compose config') echo 'ghcr.io/fluxerapp/fluxer-api:v1' ;;",
'esac',
'exit 0',
'',
].join('\n');
const CURL_STUB = [
'#!/bin/sh',
"out=''",
"prev=''",
'for arg in "$@"; do',
' if [ "$prev" = \'-o\' ]; then out=$arg; fi',
' prev=$arg',
'done',
'[ -z "$out" ] || printf \'name: fluxer\\nservices:\\n api:\\n image: stub\\n\' > "$out"',
'',
].join('\n');
const sandbox = await mkdtemp(path.join(tmpdir(), 'fluxer-installer-'));
try {
const stubBin = path.join(sandbox, 'bin');
await mkdir(stubBin, {recursive: true});
await writeFile(path.join(stubBin, 'docker'), DOCKER_STUB, {mode: 0o755});
await writeFile(path.join(stubBin, 'curl'), CURL_STUB, {mode: 0o755});
await writeFile(path.join(stubBin, 'openssl'), '#!/bin/sh\nexit 0\n', {mode: 0o755});
const instance = path.join(sandbox, 'instance');
await mkdir(instance, {recursive: true});
await writeFile(path.join(instance, '.env'), 'FLUXER_DOMAIN=x.example\nFLUXER_IMAGE_TAG=2026.813.205040\n');
await writeFile(path.join(instance, 'docker-compose.yml'), 'name: fluxer\nservices:\n api:\n image: stub\n');
const planned = (label: string, args: ReadonlyArray<string>, cwd?: string): string | null => {
const run = spawnSync('sh', [path.join(INSTALLER_ROOT, 'install.sh'), ...args], {
cwd,
encoding: 'utf8',
env: {
...process.env,
PATH: `${stubBin}${path.delimiter}${process.env.PATH ?? ''}`,
...(cwd == null ? {} : {PWD: cwd}),
},
});
if (run.error != null) {
problems.push(`install.sh ${label} could not run: ${run.error.message}`);
return null;
}
if (run.status !== 0) {
problems.push(`install.sh ${label} exited ${String(run.status)}: ${run.stderr.trim()}`);
return null;
}
return run.stdout;
};
const plannedRef = (label: string, args: ReadonlyArray<string>): string | null => {
const stdout = planned(label, args);
if (stdout == null) {
return null;
}
const line = stdout.match(/^ {2}ref\s+(\S+)$/mu);
if (line == null) {
problems.push(`install.sh ${label} printed no ref line`);
return null;
}
return line[1];
};
const INSTALL_ARGS = [
'--dry-run',
'--non-interactive',
'--allow-root',
'--domain',
'x.example',
'--email',
'[email protected]',
'--dir',
path.join(sandbox, 'target'),
];
const REF_CASES: ReadonlyArray<readonly [string, ReadonlyArray<string>, string]> = [
['on the default image tag', INSTALL_ARGS, 'main'],
['under --image-tag latest', [...INSTALL_ARGS, '--image-tag', 'latest'], 'main'],
['under --image-tag 2026.813.205040', [...INSTALL_ARGS, '--image-tag', '2026.813.205040'], '2026.813.205040'],
[
'under --ref feature/x --image-tag 2026.813.205040',
[...INSTALL_ARGS, '--ref', 'feature/x', '--image-tag', '2026.813.205040'],
'feature/x',
],
[
'under --update against a pinned .env',
['--update', '--dry-run', '--allow-root', '--dir', instance],
'2026.813.205040',
],
];
for (const [label, args, expected] of REF_CASES) {
const resolved = plannedRef(label, args);
if (resolved != null && resolved !== expected) {
problems.push(`install.sh ${label} plans ref ${resolved}, and the image tag it pairs with wants ${expected}`);
}
}
const composeYmlInstance = path.join(sandbox, 'compose-yml-instance');
await mkdir(composeYmlInstance, {recursive: true});
await writeFile(
path.join(composeYmlInstance, '.env'),
'FLUXER_DOMAIN=x.example\nFLUXER_IMAGE_TAG=2026.813.205040\n',
);
await writeFile(path.join(composeYmlInstance, 'compose.yml'), 'name: fluxer\nservices:\n api:\n image: stub\n');
const COMPOSE_NAME_CASES: ReadonlyArray<readonly [string, ReadonlyArray<string>, string | undefined]> = [
[
'under --update against a compose.yml instance',
['--update', '--dry-run', '--allow-root', '--dir', composeYmlInstance],
undefined,
],
[
'under --update standing in a compose.yml instance',
['--update', '--dry-run', '--allow-root'],
composeYmlInstance,
],
];
for (const [label, args, cwd] of COMPOSE_NAME_CASES) {
const stdout = planned(label, args, cwd);
if (stdout == null) {
continue;
}
if (!/^ {4}compose\.yml is unchanged$/mu.test(stdout)) {
problems.push(`install.sh ${label} does not plan the refreshed stack file onto compose.yml`);
}
if (/^ {4}docker-compose\.yml is new$/mu.test(stdout)) {
problems.push(`install.sh ${label} plans a docker-compose.yml that Compose would never load there`);
}
}
} finally {
await rm(sandbox, {recursive: true, force: true});
if (options['source-only']) {
console.log(' installer execution skipped (--source-only)');
} else {
problems.push(...(await verifyInstallerExecution(INSTALLER_ROOT)));
}
const covered = new Set<string>(
@@ -1791,10 +1607,10 @@ console.log('unthrottled routes and global bucket claims');
admin: route.path.startsWith('/admin'),
}));
const publicShapes = new Set(main.map((operation) => shapeOf(operation.method, operation.path)));
const publicShapes = new Set(main.map((operation) => routeShape(operation.method, operation.path)));
const publicUnthrottled = unthrottled
.filter((entry) => !entry.admin)
.map((entry) => shapeOf(entry.method, stripVersionPrefix(entry.route.replace(/:([a-zA-Z_]+)/gu, '{$1}'))))
.map((entry) => routeShape(entry.method, stripVersionPrefix(entry.route.replace(/:([a-zA-Z_]+)/gu, '{$1}'))))
.filter((shape) => publicShapes.has(shape));
const adminUnthrottled = unthrottled.filter((entry) => entry.admin);
@@ -1868,6 +1684,14 @@ console.log('error registry and abuse signal weights');
for (const entry of registrySource.matchAll(/^\t([A-Z][A-Z0-9_]*):/gmu)) {
registryCodes.add(entry[1]);
}
const validationSource = await readFile(
path.join(REPO_ROOT, 'packages/constants/src/ValidationErrorCodes.ts'),
'utf8',
);
const validationCodes = new Set<string>();
for (const entry of validationSource.matchAll(/^\t([A-Z][A-Z0-9_]*):/gmu)) {
validationCodes.add(entry[1]);
}
const problems: Array<string> = [];
for (const code of registryCodes) {
if (!documentedCodes.has(code)) {
@@ -1890,11 +1714,18 @@ console.log('error registry and abuse signal weights');
}
const documentedRegistryCodes = [...registryCodes].filter((c) => documentedCodes.has(c)).length;
if (documentedEntries < 491) {
const UNDOCUMENTED_VALIDATION_CODES = new Set(['EMAIL_DOMAIN_CANNOT_RECEIVE_MAIL']);
const expectedEntries = registryCodes.size + validationCodes.size - UNDOCUMENTED_VALIDATION_CODES.size;
if (documentedEntries < expectedEntries) {
problems.push(
`errors.md code entries parsed fell to ${documentedEntries.toString()}, floor is 491, the 255 API codes plus the 236 validation codes. The registry parser reads a \`| CODE |\` table row and a \`### \`CODE\`\` or \`#### \`CODE\`\` heading, and one of those shapes has stopped matching`,
`errors.md code entries parsed fell to ${documentedEntries.toString()}, expected ${expectedEntries.toString()}, the ${registryCodes.size.toString()} API codes plus the ${validationCodes.size.toString()} validation codes less the ${UNDOCUMENTED_VALIDATION_CODES.size.toString()} named as undocumented. Either a code lost its entry, or the registry parser stopped matching one of the \`| CODE |\` table row and \`### \`CODE\`\` or \`#### \`CODE\`\` heading shapes it reads`,
);
}
for (const code of documentedCodes) {
if (!registryCodes.has(code) && !validationCodes.has(code)) {
problems.push(`errors.md documents ${code}, which is in neither code registry`);
}
}
if (documentedRegistryCodes < registryCodes.size) {
problems.push(
`errors.md documents ${documentedRegistryCodes.toString()} of the ${registryCodes.size.toString()} registry codes, floor is every one of them`,
@@ -1985,7 +1816,7 @@ console.log('snowflake layout');
if (!page.includes(String(epoch))) {
problems.push(`snowflakes.md does not state the epoch ${epoch.toString()}`);
}
const epochIso = new Date(epoch).toISOString().replace('.000Z', '.000Z');
const epochIso = new Date(epoch).toISOString();
if (!page.includes(epochIso.slice(0, 10))) {
problems.push(`snowflakes.md does not state the epoch date ${epochIso}`);
}
@@ -1997,7 +1828,7 @@ console.log('snowflake layout');
}
const timestampBits = 63 - shift;
const lastMs = epoch + 2 ** timestampBits - 1;
const lastIso = new Date(lastMs).toISOString().replace('Z', 'Z');
const lastIso = new Date(lastMs).toISOString();
if (!page.includes(lastIso.slice(0, 19))) {
problems.push(`snowflakes.md does not state the last representable instant ${lastIso}`);
}
@@ -2012,7 +1843,6 @@ console.log('snowflake layout');
console.log('attachment upload geometry');
{
const limits = await readFile(path.join(REPO_ROOT, 'packages/constants/src/LimitConstants.ts'), 'utf8');
const page = await readFile(path.join(DOCS_ROOT, 'topics/uploads.md'), 'utf8');
const problems: Array<string> = [];
const readConst = (name: string): number | null => {
const found = limits.match(new RegExp(`${name} = ([^;]+);`, 'u'));
@@ -2029,22 +1859,16 @@ console.log('attachment upload geometry');
.reduce((a, b) => a + b, 0);
};
const constants: Array<[string, string]> = [
['ATTACHMENT_UPLOAD_CHUNK_THRESHOLD', 'singlepart threshold'],
['ATTACHMENT_UPLOAD_MIN_CHUNK_SIZE', 'minimum part size'],
['ATTACHMENT_UPLOAD_MAX_CHUNKS', 'maximum part count'],
['ATTACHMENT_MAX_SIZE_NON_PREMIUM', 'non-premium attachment ceiling'],
['ATTACHMENT_MAX_SIZE_PREMIUM', 'premium attachment ceiling'],
['ATTACHMENT_MAX_SIZE_BOT', 'bot attachment ceiling'],
const constants: Array<[string, string, string]> = [
['ATTACHMENT_UPLOAD_CHUNK_THRESHOLD', 'singlepart threshold', 'topics/uploads.md'],
['ATTACHMENT_MAX_SIZE_NON_PREMIUM', 'non-premium attachment ceiling', 'http-api/messages.mdx'],
['ATTACHMENT_MAX_SIZE_PREMIUM', 'premium attachment ceiling', 'http-api/messages.mdx'],
['ATTACHMENT_MAX_SIZE_BOT', 'bot attachment ceiling', 'http-api/messages.mdx'],
];
const divisor = readConst('ATTACHMENT_UPLOAD_TARGET_PART_COUNT');
if (divisor == null) {
problems.push('could not read ATTACHMENT_UPLOAD_TARGET_PART_COUNT from LimitConstants.ts');
} else if (!page.includes(`divided by ${divisor.toString()}`)) {
problems.push(`uploads.md does not say the part size is the declared size divided by ${divisor.toString()}`);
}
let compared = 0;
for (const [name, label] of constants) {
for (const [name, label, file] of constants) {
const page = pages.find((entry) => entry.relativePath === file);
if (!page) throw new Error(`Missing upload documentation: ${file}`);
const value = readConst(name);
if (value == null) {
problems.push(`could not read ${name} from LimitConstants.ts`);
@@ -2052,9 +1876,9 @@ console.log('attachment upload geometry');
}
compared += 1;
const bare = new RegExp(`(?<![\\d,.])${value.toString()}(?![\\d,.])`, 'u');
const grouped = new RegExp(`(?<![\\d,.])${value.toLocaleString('en-US').replace(/,/gu, ',')}(?![\\d,.])`, 'u');
if (!bare.test(page) && !grouped.test(page)) {
problems.push(`uploads.md does not state the ${label} of ${value.toString()}`);
const grouped = new RegExp(`(?<![\\d,.])${value.toLocaleString('en-US')}(?![\\d,.])`, 'u');
if (!bare.test(page.source) && !grouped.test(page.source)) {
problems.push(`${file} does not state the ${label} of ${value.toString()}`);
}
}
console.log(` upload constants compared: ${compared.toString()}`);
@@ -2067,7 +1891,7 @@ console.log('captcha gated operations');
const documented = new Set<string>();
const documentedLabels = new Map<string, string>();
for (const row of page.matchAll(/^\|\s*(GET|POST|PUT|PATCH|DELETE)\s*\|\s*(\/v1\/\S+?)\s*\|/gmu)) {
const normalised = shapeOf(row[1], stripVersionPrefix(row[2]));
const normalised = routeShape(row[1], stripVersionPrefix(row[2]));
documented.add(normalised);
documentedLabels.set(normalised, `${row[1]} ${row[2]}`);
}
@@ -2111,7 +1935,7 @@ console.log('bot capability flag (from the middleware chain)');
if (key.startsWith('GET /admin') || key.includes(' /admin/')) {
continue;
}
const documented = documentedFlags.get(key);
const documented = documentedMain.get(key);
if (documented == null) {
continue;
}
@@ -2171,7 +1995,7 @@ console.log('unauthenticated capability flag (from the middleware chain)');
if (key.startsWith('GET /admin') || key.includes(' /admin/')) {
continue;
}
const documented = documentedFlags.get(key);
const documented = documentedMain.get(key);
if (documented == null) {
continue;
}
@@ -2207,27 +2031,40 @@ console.log('spec security field against the middleware chain');
'LoginRequired admits a bot token, but GuildOperationsService rejects every bot with 400 BOTS_CANNOT_CREATE_GUILDS, so the spec must not advertise botToken',
],
]);
const specSecurity = new Map<string, Set<string>>();
const specSecurity = new Map<string, EffectiveSpecSecurity>();
for (const operation of main) {
const security = operation.security ?? [];
const schemes = new Set<string>();
for (const entry of operation.security ?? []) {
for (const entry of security) {
for (const scheme of Object.keys(entry)) {
schemes.add(scheme);
}
}
specSecurity.set(shapeOf(operation.method, operation.path), schemes);
specSecurity.set(routeShape(operation.method, operation.path), {
schemes,
allowsAnonymous: security.length === 0 || security.some((entry) => Object.keys(entry).length === 0),
});
}
const specBugs: Array<string> = [];
let compared = 0;
for (const route of controllerRoutes) {
const key = astRoute(route);
const declaredSchemes = specSecurity.get(key);
if (declaredSchemes == null || MANUAL_CREDENTIAL.has(key)) {
const declaredSecurity = specSecurity.get(key);
if (declaredSecurity == null || MANUAL_CREDENTIAL.has(key)) {
continue;
}
const declaredSchemes = declaredSecurity.schemes;
compared += 1;
const anyLogin = route.hasLoginRequired || route.hasLoginRequiredAllowSuspicious;
const acceptsBot = anyLogin && !route.hasDefaultUserOnly;
const requiresAuthentication =
anyLogin ||
route.hasDefaultUserOnly ||
route.oauth2BearerTokenRequired ||
route.middlewares.includes('requireOAuth2Scope');
if (requiresAuthentication && declaredSecurity.allowsAnonymous) {
specBugs.push(`${key} allows unauthenticated requests, but the middleware chain requires authentication`);
}
const exemption = BOT_SCHEME_EXEMPT.get(key);
if (exemption != null) {
if (declaredSchemes.has('botToken')) {
@@ -2261,7 +2098,7 @@ const adminTargetOnly = [...documentedAdmin.entries()]
.map(([, route]) => `${route.method} ${route.path} (${route.file})`)
.sort();
const adminLiveOnly = admin
.filter((operation) => !documentedAdmin.has(shapeOf(operation.method, operation.path)))
.filter((operation) => !documentedAdmin.has(routeShape(operation.method, operation.path)))
.map((operation) => `${operation.method} ${operation.path}`)
.sort();
console.log(` live admin operations: ${admin.length.toString()}`);
@@ -2283,3 +2120,151 @@ if (failures > 0) {
}
console.log('OK: every registered fluxer_api route is documented or covered by an exemption rule, and the');
console.log('documented routes match the live main API, media proxy, and admin target shape');
async function verifyInstallerExecution(installerRoot: string): Promise<Array<string>> {
const problems: Array<string> = [];
const shellParse = spawnSync('sh', ['-n', path.join(installerRoot, 'install.sh')], {encoding: 'utf8'});
if (shellParse.error != null) {
problems.push(`sh -n could not run against install.sh: ${shellParse.error.message}`);
} else if (shellParse.status !== 0) {
problems.push(`sh -n rejects install.sh: ${shellParse.stderr.trim()}`);
}
const DOCKER_STUB = [
'#!/bin/sh',
'case "$1 $2" in',
" '--version ') echo 'Docker version 27.1.1, build stub' ;;",
" 'compose version') if [ \"$3\" = '--short' ]; then echo '2.30.3'; else echo 'v2.30.3'; fi ;;",
" 'compose config') echo 'ghcr.io/fluxerapp/fluxer-api:v1' ;;",
'esac',
'exit 0',
'',
].join('\n');
const CURL_STUB = [
'#!/bin/sh',
"out=''",
"prev=''",
'for arg in "$@"; do',
' if [ "$prev" = \'-o\' ]; then out=$arg; fi',
' prev=$arg',
'done',
'[ -z "$out" ] || printf \'name: fluxer\\nservices:\\n api:\\n image: stub\\n\' > "$out"',
'',
].join('\n');
const sandbox = await mkdtemp(path.join(tmpdir(), 'fluxer-installer-'));
try {
const stubBin = path.join(sandbox, 'bin');
await mkdir(stubBin, {recursive: true});
await writeFile(path.join(stubBin, 'docker'), DOCKER_STUB, {mode: 0o755});
await writeFile(path.join(stubBin, 'curl'), CURL_STUB, {mode: 0o755});
await writeFile(path.join(stubBin, 'openssl'), '#!/bin/sh\nexit 0\n', {mode: 0o755});
const instance = path.join(sandbox, 'instance');
await mkdir(instance, {recursive: true});
await writeFile(path.join(instance, '.env'), 'FLUXER_DOMAIN=x.example\nFLUXER_IMAGE_TAG=2026.813.205040\n');
await writeFile(path.join(instance, 'docker-compose.yml'), 'name: fluxer\nservices:\n api:\n image: stub\n');
const planned = (label: string, args: ReadonlyArray<string>, cwd?: string): string | null => {
const run = spawnSync('sh', [path.join(installerRoot, 'install.sh'), ...args], {
cwd,
encoding: 'utf8',
env: {
...process.env,
PATH: `${stubBin}${path.delimiter}${process.env.PATH ?? ''}`,
...(cwd == null ? {} : {PWD: cwd}),
},
});
if (run.error != null) {
problems.push(`install.sh ${label} could not run: ${run.error.message}`);
return null;
}
if (run.status !== 0) {
problems.push(`install.sh ${label} exited ${String(run.status)}: ${run.stderr.trim()}`);
return null;
}
return run.stdout;
};
const plannedRef = (label: string, args: ReadonlyArray<string>): string | null => {
const stdout = planned(label, args);
if (stdout == null) {
return null;
}
const line = stdout.match(/^ {2}ref\s+(\S+)$/mu);
if (line == null) {
problems.push(`install.sh ${label} printed no ref line`);
return null;
}
return line[1];
};
const INSTALL_ARGS = [
'--dry-run',
'--non-interactive',
'--allow-root',
'--domain',
'x.example',
'--email',
'[email protected]',
'--dir',
path.join(sandbox, 'target'),
];
const REF_CASES: ReadonlyArray<readonly [string, ReadonlyArray<string>, string]> = [
['on the default image tag', INSTALL_ARGS, 'main'],
['under --image-tag latest', [...INSTALL_ARGS, '--image-tag', 'latest'], 'main'],
['under --image-tag 2026.813.205040', [...INSTALL_ARGS, '--image-tag', '2026.813.205040'], '2026.813.205040'],
[
'under --ref feature/x --image-tag 2026.813.205040',
[...INSTALL_ARGS, '--ref', 'feature/x', '--image-tag', '2026.813.205040'],
'feature/x',
],
[
'under --update against a pinned .env',
['--update', '--dry-run', '--allow-root', '--dir', instance],
'2026.813.205040',
],
];
for (const [label, args, expected] of REF_CASES) {
const resolved = plannedRef(label, args);
if (resolved != null && resolved !== expected) {
problems.push(`install.sh ${label} plans ref ${resolved}, and the image tag it pairs with wants ${expected}`);
}
}
const composeYmlInstance = path.join(sandbox, 'compose-yml-instance');
await mkdir(composeYmlInstance, {recursive: true});
await writeFile(
path.join(composeYmlInstance, '.env'),
'FLUXER_DOMAIN=x.example\nFLUXER_IMAGE_TAG=2026.813.205040\n',
);
await writeFile(path.join(composeYmlInstance, 'compose.yml'), 'name: fluxer\nservices:\n api:\n image: stub\n');
const COMPOSE_NAME_CASES: ReadonlyArray<readonly [string, ReadonlyArray<string>, string | undefined]> = [
[
'under --update against a compose.yml instance',
['--update', '--dry-run', '--allow-root', '--dir', composeYmlInstance],
undefined,
],
[
'under --update standing in a compose.yml instance',
['--update', '--dry-run', '--allow-root'],
composeYmlInstance,
],
];
for (const [label, args, cwd] of COMPOSE_NAME_CASES) {
const stdout = planned(label, args, cwd);
if (stdout == null) {
continue;
}
if (!/^ {4}compose\.yml is unchanged$/mu.test(stdout)) {
problems.push(`install.sh ${label} does not plan the refreshed stack file onto compose.yml`);
}
if (/^ {4}docker-compose\.yml is new$/mu.test(stdout)) {
problems.push(`install.sh ${label} plans a docker-compose.yml that Compose would never load there`);
}
}
} finally {
await rm(sandbox, {recursive: true, force: true});
}
return problems;
}
fluxer_docs/scripts/VerifyDocsSchemas.ts
+89 -120
View File
@@ -9,14 +9,13 @@ import type {
OpenAPIDocument as Spec,
} from '@fluxer/openapi/src/OpenAPITypes';
import {DOCS_ROOT, listMarkdownFiles, slugifyHeading} from './DocsSource.ts';
import {readRouteHeaders} from './DocsRouteHeaders.ts';
import {DOCS_ROOT, HTTP_METHODS, readMarkdownPages, routeShape, slugifyHeading, splitTableRow} from './DocsSource.ts';
const REPO_ROOT = fileURLToPath(new URL('../../', import.meta.url));
const MAIN_SPEC = path.join(REPO_ROOT, 'fluxer_api/src/api/openapi/openapi.json');
const ADMIN_SPEC = path.join(REPO_ROOT, 'fluxer_admin/openapi-admin.json');
const ROUTE_HEADER = /<RouteHeader\s+method="([A-Z]+)"\s+path="([^"]+)"/u;
const TABLE_ROW = /^\|\s*([^|]+?)\s*\|/u;
const OBJECT_REFERENCE = /\]\([^)]*#[a-z0-9-]*object\)/u;
interface Mismatch {
@@ -33,32 +32,52 @@ function stripVersion(routePath: string): string {
return routePath;
}
function shape(method: string, routePath: string): string {
return `${method} ${routePath.split('?')[0].replace(/\{[^}]*\}/gu, '{}')}`;
function resolveSchemaPointer(spec: Spec, reference: string): SchemaNode | boolean {
if (!reference.startsWith('#')) throw new Error(`Unsupported schema reference: ${reference}`);
const pointer = decodeURIComponent(reference.slice(1));
const prefix = '/components/schemas/';
if (!pointer.startsWith(prefix)) throw new Error(`Unsupported schema reference: ${reference}`);
let target: unknown = spec.components.schemas;
for (const token of pointer.slice(prefix.length).split('/')) {
if (/~(?:[^01]|$)/u.test(token)) throw new Error(`Invalid schema reference escape: ${reference}`);
const key = token.replace(/~1/gu, '/').replace(/~0/gu, '~');
if (
target === null ||
typeof target !== 'object' ||
(Array.isArray(target) && !/^(0|[1-9][0-9]*)$/u.test(key)) ||
!Object.hasOwn(target, key)
) {
throw new Error(`Missing schema reference: ${reference}`);
}
target = (target as Record<string, unknown>)[key];
}
if (typeof target === 'boolean') return target;
if (target === null || typeof target !== 'object' || Array.isArray(target)) {
throw new Error(`Reference does not identify a schema: ${reference}`);
}
return target as SchemaNode;
}
function resolveRef(spec: Spec, node: SchemaNode | boolean | undefined, depth = 0): SchemaNode | undefined {
if (node == null || typeof node === 'boolean') {
if (node == null || node === false) {
return undefined;
}
if (node === true) {
return {};
}
if (depth > 64) {
throw new Error('OpenAPI reference chain exceeds the supported depth');
}
if (node.$ref != null) {
const prefix = '#/components/schemas/';
if (!node.$ref.startsWith(prefix)) {
throw new Error(`Unsupported schema reference: ${node.$ref}`);
}
const name = decodeURIComponent(node.$ref.slice(prefix.length)).replace(/~1/gu, '/').replace(/~0/gu, '~');
const target = spec.components.schemas[name];
if (target == null) {
throw new Error(`Missing schema reference: ${node.$ref}`);
}
const target = resolveSchemaPointer(spec, node.$ref);
const resolved = resolveRef(spec, target, depth + 1);
const {$ref, ...siblings} = node;
if (resolved == null || Object.keys(siblings).length === 0) {
return resolved;
}
if (Object.keys(resolved).length === 0) {
return siblings;
}
return {...resolved, allOf: [...(resolved.allOf ?? []), siblings]};
}
return node;
@@ -195,20 +214,27 @@ function operationIndex(spec: Spec): Map<string, Operation> {
const index = new Map<string, Operation>();
for (const [routePath, item] of Object.entries(spec.paths)) {
for (const [method, operation] of Object.entries(item)) {
index.set(shape(method.toUpperCase(), stripVersion(routePath)), operation);
const upper = method.toUpperCase();
if (!HTTP_METHODS.has(upper)) continue;
const key = routeShape(upper, stripVersion(routePath));
if (index.has(key)) throw new Error(`Duplicate OpenAPI operation: ${key}`);
index.set(key, operation);
}
}
return index;
}
function cleanFieldName(cell: string): string | null {
const name = cell
function fieldNameText(cell: string): string {
return cell
.replace(/<sup>.*?<\/sup>/gu, '')
.replace(/\*\*/gu, '')
.replace(/`/gu, '')
.replace(/\\/gu, '')
.trim()
.replace(/\?$/u, '');
.trim();
}
function cleanFieldName(cell: string): string | null {
const name = fieldNameText(cell).replace(/\?$/u, '');
if (name.length === 0) {
return null;
}
@@ -239,12 +265,11 @@ function sectionIsByReference(lines: ReadonlyArray<string>, start: number): bool
return sawReference && !sawTable;
}
function tableOptionality(lines: ReadonlyArray<string>, start: number): Map<string, boolean> {
const out = new Map<string, boolean>();
function* firstTableRows(lines: ReadonlyArray<string>, start: number): Generator<string> {
let index = start;
while (index < lines.length && !lines[index].startsWith('|')) {
if (lines[index].startsWith('#')) {
return out;
return;
}
index += 1;
}
@@ -253,79 +278,32 @@ function tableOptionality(lines: ReadonlyArray<string>, start: number): Map<stri
if (!line.startsWith('|')) {
break;
}
const cells = line.split('|').slice(1, -1);
if (cells.length < 2) {
yield line;
}
}
interface DocumentedField {
readonly type: string | null;
readonly optional: boolean;
}
function tableFields(lines: ReadonlyArray<string>, start: number): Map<string, DocumentedField> {
const out = new Map<string, DocumentedField>();
for (const line of firstTableRows(lines, start)) {
const cells = splitTableRow(line);
if (cells.length === 0) {
continue;
}
const raw = cells[0]
.replace(/<sup>.*?<\/sup>/gu, '')
.replace(/`/gu, '')
.trim();
const raw = fieldNameText(cells[0]);
const name = cleanFieldName(cells[0]);
if (name == null) {
continue;
}
out.set(name, raw.endsWith('?'));
out.set(name, {type: cells[1] === undefined ? null : normaliseDocType(cells[1]), optional: raw.endsWith('?')});
}
return out;
}
function tableFieldTypes(lines: ReadonlyArray<string>, start: number): Map<string, string> {
const out = new Map<string, string>();
let index = start;
while (index < lines.length && !lines[index].startsWith('|')) {
if (lines[index].startsWith('#')) {
return out;
}
index += 1;
}
for (; index < lines.length; index += 1) {
const line = lines[index];
if (!line.startsWith('|')) {
break;
}
const cells = line.split('|').slice(1, -1);
if (cells.length < 2) {
continue;
}
const name = cleanFieldName(cells[0]);
if (name == null) {
continue;
}
const type = normaliseDocType(cells[1]);
if (type != null) {
out.set(name, type);
}
}
return out;
}
function tableFields(lines: ReadonlyArray<string>, start: number): Set<string> {
const fields = new Set<string>();
let index = start;
while (index < lines.length && !lines[index].startsWith('|')) {
if (lines[index].startsWith('#')) {
return fields;
}
index += 1;
}
for (; index < lines.length; index += 1) {
const line = lines[index];
if (!line.startsWith('|')) {
break;
}
const match = line.match(TABLE_ROW);
if (match == null) {
continue;
}
const name = cleanFieldName(match[1]);
if (name != null) {
fields.add(name);
}
}
return fields;
}
const mainSpec: Spec = JSON.parse(await readFile(MAIN_SPEC, 'utf8'));
const adminSpec: Spec = JSON.parse(await readFile(ADMIN_SPEC, 'utf8'));
const mainIndex = operationIndex(mainSpec);
@@ -352,18 +330,16 @@ function documentReferences(page: string, line: string): Set<string> {
return references;
}
const allFiles = await listMarkdownFiles(DOCS_ROOT);
const pages = await readMarkdownPages(DOCS_ROOT);
const anchorFields = new Map<string, Set<string>>();
const anchorTypes = new Map<string, Map<string, string>>();
const anchorReferences = new Map<string, Set<string>>();
const objectAnchors = new Set<string>();
for (const file of allFiles) {
const slug = path
.relative(DOCS_ROOT, file)
for (const {relativePath, lines} of pages) {
const slug = relativePath
.replace(/\.(mdx|md)$/u, '')
.replace(/\/index$/u, '')
.replace(/^index$/u, '');
const lines = (await readFile(file, 'utf8')).split('\n');
let currentAnchors: Array<string> = [];
const pendingAnchors: Array<string> = [];
for (let i = 0; i < lines.length; i += 1) {
@@ -398,15 +374,14 @@ for (const file of allFiles) {
if (currentAnchors.length === 0 || !line.startsWith('|')) {
continue;
}
const row = line.match(TABLE_ROW);
if (row == null) {
const cells = splitTableRow(line);
if (cells.length === 0) {
continue;
}
const name = cleanFieldName(row[1]);
const name = cleanFieldName(cells[0]);
if (name == null) {
continue;
}
const cells = line.split('|').slice(1, -1);
const declaredType = cells.length >= 2 ? normaliseDocType(cells[1]) : null;
for (const anchor of currentAnchors) {
const anchorKey = `${slug}#${anchor}`;
@@ -436,22 +411,22 @@ let typesCompared = 0;
let optionalityCompared = 0;
const optionalityAdvisories: Array<string> = [];
for (const file of allFiles) {
const relative = path.relative(DOCS_ROOT, file);
for (const page of pages) {
const {relativePath: relative, lines} = page;
if (relative.startsWith('media-proxy/')) {
continue;
}
const lines = (await readFile(file, 'utf8')).split('\n');
const routeHeaders = readRouteHeaders(page);
const pageFields = new Set<string>();
for (let i = 0; i < lines.length; i += 1) {
if (!lines[i].startsWith('|')) {
continue;
}
const match = lines[i].match(TABLE_ROW);
if (match == null) {
const cells = splitTableRow(lines[i]);
if (cells.length === 0) {
continue;
}
const name = cleanFieldName(match[1]);
const name = cleanFieldName(cells[0]);
if (name != null) {
pageFields.add(name);
}
@@ -475,18 +450,14 @@ for (const file of allFiles) {
}
for (const section of sections) {
let header: RegExpMatchArray | null = null;
for (let i = section.start; i < section.end; i += 1) {
const match = lines[i].match(ROUTE_HEADER);
if (match != null) {
header = match;
break;
}
}
const header = routeHeaders.find((candidate) => candidate.line > section.start && candidate.line <= section.end);
if (header == null) {
continue;
}
const key = shape(header[1], stripVersion(header[2]));
if (header.endLine > section.end) {
throw new Error(`${relative}:${header.line}: RouteHeader crosses a section boundary`);
}
const key = routeShape(header.method, stripVersion(header.path));
const operation = index.get(key);
if (operation == null) {
continue;
@@ -580,9 +551,9 @@ for (const file of allFiles) {
continue;
}
checkedBodies += 1;
const documentedTypes = tableFieldTypes(lines, i + 1);
const actualTypes = collectPropertyTypes(spec, jsonSchema);
for (const [field, docType] of documentedTypes) {
for (const [field, {type: docType}] of documented) {
if (docType === null) continue;
const specType = actualTypes.get(field);
if (specType == null) {
continue;
@@ -601,11 +572,9 @@ for (const file of allFiles) {
detail: `${field}: documented ${docType}, schema ${specType}`,
});
}
const documentedOptional = tableOptionality(lines, i + 1);
const requiredFields = collectRequired(spec, jsonSchema);
const bodyProperties = collectProperties(spec, jsonSchema);
for (const [field, isOptional] of documentedOptional) {
if (!bodyProperties.has(field)) {
for (const [field, {optional: isOptional}] of documented) {
if (!actual.has(field)) {
continue;
}
const specRequired = requiredFields.has(field);
@@ -619,7 +588,7 @@ for (const file of allFiles) {
: `${relative} ${key} ${field}: documented required, schema marks it optional`,
);
}
for (const field of documented) {
for (const field of documented.keys()) {
if (actual.has(field)) {
continue;
}
@@ -650,7 +619,7 @@ for (const file of allFiles) {
continue;
}
checkedQueries += 1;
for (const field of documented) {
for (const field of documented.keys()) {
if (!actual.has(field)) {
mismatches.push({page: relative, operation: key, kind: 'query-extra', detail: field});
}
@@ -700,7 +669,7 @@ console.log(`bodies documented by reference to an object section: ${documentedBy
console.log(`query parameter tables checked: ${checkedQueries.toString()}`);
console.log(`success response schemas checked: ${responsesChecked.toString()}`);
console.log(`response fields found documented on the page: ${responseFieldsFound.toString()}`);
console.log(`request body field types compared: ${typesCompared.toString()}`);
console.log(`request and response field types compared: ${typesCompared.toString()}`);
console.log(`request body optionality compared: ${optionalityCompared.toString()}`);
console.log(`optionality advisories: ${optionalityAdvisories.length.toString()}`);
if (optionalityAdvisories.length > 0) {
@@ -732,4 +701,4 @@ if (mismatches.length > 0) {
console.error(`FAIL: ${mismatches.length.toString()} field mismatches`);
process.exit(1);
}
console.log('OK: every documented request body and query table matches the live schema');
console.log('OK: no field mismatches found in the checked tables and checked-in OpenAPI schemas');
fluxer_docs/scripts/VerifyDocsStyle.ts
+16 -38
View File
@@ -3,7 +3,7 @@
import {readdir, readFile} from 'node:fs/promises';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {DOCS_ROOT, listMarkdownFiles} from './DocsSource.ts';
import {DOCS_ROOT, readMarkdownPages} from './DocsSource.ts';
import {
columnWidthPercents,
extractTables,
@@ -67,23 +67,10 @@ function frontmatterOf(source: string): string | null {
return source.slice(4, end);
}
function insideFence(lines: ReadonlyArray<string>, index: number): boolean {
let fenced = false;
for (let cursor = 0; cursor < index; cursor += 1) {
if (lines[cursor].startsWith('```')) {
fenced = !fenced;
}
}
return fenced;
}
const files = await listMarkdownFiles(DOCS_ROOT);
const pages = await readMarkdownPages(DOCS_ROOT);
const findings: Array<Finding> = [];
for (const file of files) {
const relative = path.relative(DOCS_ROOT, file);
const source = await readFile(file, 'utf8');
const lines = source.split('\n');
for (const {file, relativePath: relative, source, lines} of pages) {
const frontmatter = frontmatterOf(source);
if (frontmatter == null) {
@@ -109,10 +96,15 @@ for (const file of files) {
findings.push({file: relative, line: 1, rule: 'extension', detail: 'uses RouteHeader but is not .mdx'});
}
let fenced = false;
for (let index = 0; index < lines.length; index += 1) {
const line = lines[index];
const number = index + 1;
if (insideFence(lines, index)) {
const insideFence = fenced;
if (line.startsWith('```')) {
fenced = !fenced;
}
if (insideFence) {
continue;
}
if (line.includes('—')) {
@@ -140,9 +132,7 @@ for (const file of files) {
}
}
for (const file of files) {
const relative = path.relative(DOCS_ROOT, file);
const lines = (await readFile(file, 'utf8')).split('\n');
for (const {relativePath: relative, lines} of pages) {
const sectionStarts: Array<number> = [];
for (let index = 0; index < lines.length; index += 1) {
if (lines[index].startsWith('## ') && !lines[index].startsWith('### ')) {
@@ -167,15 +157,13 @@ for (const file of files) {
file: relative,
line: response + 1,
rule: 'order',
detail: '"### Response body" must precede "### Response" (conventions.md)',
detail: '"### Response body" must precede "### Response"',
});
}
}
}
for (const file of files) {
const relative = path.relative(DOCS_ROOT, file);
const lines = (await readFile(file, 'utf8')).split('\n');
for (const {relativePath: relative, lines} of pages) {
let block = new Map<string, number>();
let sinceFootnote = 0;
for (let index = 0; index < lines.length; index += 1) {
@@ -206,10 +194,7 @@ for (const file of files) {
const NOTATION_EXAMPLE = 'A superscript marker such as <sup>1</sup> refers to the numbered footnote';
for (const file of files) {
const relative = path.relative(DOCS_ROOT, file);
const source = await readFile(file, 'utf8');
const lines = source.split('\n');
for (const {relativePath: relative, source, lines} of pages) {
const explainsNotation = source.includes(NOTATION_EXAMPLE);
const bounds: Array<number> = [0];
for (let index = 0; index < lines.length; index += 1) {
@@ -292,8 +277,6 @@ const ACCEPTED_TABLE_FINDINGS = new Map<string, Readonly<Partial<Record<TableRul
['admin-api/reports.mdx', {'table-fit': 1, 'table-identifier': 2}],
['admin-api/users.mdx', {'table-fit': 1, 'table-identifier': 1}],
['admin-api/voice.mdx', {'table-identifier': 3}],
['authentication.md', {'table-cell': 2, 'table-identifier': 1}],
['conventions.md', {'table-cell': 1, 'table-parallel': 1}],
['gateway/event-filtering.md', {'table-cell': 2}],
['gateway/events.md', {'table-identifier': 1}],
['gateway/opcodes-and-close-codes.md', {'table-cell': 1}],
@@ -302,7 +285,7 @@ const ACCEPTED_TABLE_FINDINGS = new Map<string, Readonly<Partial<Record<TableRul
['http-api/billing.mdx', {'table-identifier': 5}],
['http-api/calls.mdx', {'table-fit': 1, 'table-cell': 3}],
['http-api/channels.mdx', {'table-cell': 6}],
['http-api/connections.mdx', {'table-fit': 1, 'table-cell': 4, 'table-identifier': 1}],
['http-api/connections.mdx', {'table-cell': 2}],
['http-api/deployment-availability.md', {'table-fit': 1}],
['http-api/discovery.mdx', {'table-cell': 3}],
['http-api/donations.mdx', {'table-cell': 2}],
@@ -338,11 +321,8 @@ const ACCEPTED_TABLE_FINDINGS = new Map<string, Readonly<Partial<Record<TableRul
['http-api/users/settings-protobuf.md', {'table-fit': 2, 'table-identifier': 7, 'table-parallel': 1}],
['http-api/users/settings.mdx', {'table-fit': 1, 'table-cell': 2, 'table-identifier': 1, 'table-parallel': 1}],
['http-api/webhooks.mdx', {'table-identifier': 1, 'table-parallel': 1}],
['media-proxy/overview.md', {'table-parallel': 1}],
['media-proxy/responses-and-limits.md', {'table-cell': 2}],
['media-proxy/routes.mdx', {'table-cell': 1}],
['media-proxy/transformations.md', {'table-parallel': 1}],
['topics/uploads.md', {'table-fit': 1, 'table-identifier': 1}],
['voice/index.md', {'table-parallel': 1}],
]);
@@ -350,9 +330,7 @@ const tableFindingsByPage = new Map<string, Map<TableRule, Array<Finding>>>();
let tablesMeasured = 0;
const overWideTier: Array<Finding> = [];
for (const file of files) {
const relative = path.relative(DOCS_ROOT, file);
const source = await readFile(file, 'utf8');
for (const {relativePath: relative, source} of pages) {
for (const table of extractTables(source)) {
tablesMeasured += 1;
const raise = (rule: TableRule, detail: string): void => {
@@ -541,7 +519,7 @@ for (const finding of findings) {
byRule.set(finding.rule, (byRule.get(finding.rule) ?? 0) + 1);
}
console.log(`pages checked: ${files.length.toString()}`);
console.log(`pages checked: ${pages.length.toString()}`);
console.log(`tables measured: ${tablesMeasured.toString()}`);
for (const rule of TABLE_RULES) {