diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 69bddb60a..c3c4c029d 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -243,7 +243,6 @@ services: volume: nocopy: true - pnpm-store:/home/vscode/.local/share/pnpm/store - - docs-venv:/workspaces/fluxer/fluxer_docs/.venv - cargo-registry:/home/vscode/.cargo/registry - cargo-git:/home/vscode/.cargo/git - rust-target:/workspaces/fluxer/target @@ -351,7 +350,6 @@ services: volumes: pnpm-store: - docs-venv: root-node-modules: fluxer-api-node-modules: fluxer-app-node-modules: diff --git a/.devcontainer/fix-workspace-permissions.sh b/.devcontainer/fix-workspace-permissions.sh index 0ef0c5d4e..6e02c3704 100644 --- a/.devcontainer/fix-workspace-permissions.sh +++ b/.devcontainer/fix-workspace-permissions.sh @@ -28,8 +28,7 @@ for path in \ /home/vscode/.cargo/registry \ /home/vscode/.cargo/git \ /home/vscode/.local \ - /home/vscode/.local/share/pnpm/store \ - /workspaces/fluxer/fluxer_docs/.venv; do + /home/vscode/.local/share/pnpm/store; do repair_tree "$path" done diff --git a/.dockerignore b/.dockerignore index 11a625775..797069ee7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -29,7 +29,8 @@ **/node_modules/ **/target/ **/test-results.json -/fluxer_docs/site/ +/fluxer_docs/dist/ +/fluxer_docs/.astro/ /fluxer_app/.devserver-cache.json /fluxer_app/pkgs/libfluxcore/ /fluxer_app/src/features/i18n/locales/*/messages.mjs diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml index 19a9255e4..6798949e4 100644 --- a/.github/workflows/build-docs.yaml +++ b/.github/workflows/build-docs.yaml @@ -33,5 +33,4 @@ jobs: with: image: fluxer-docs dockerfile: fluxer_docs/Dockerfile - context: fluxer_docs build-version: ${{ inputs['build-version'] }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 27325f657..c498a8ba3 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -460,6 +460,31 @@ jobs: exit 1 fi + docs: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + + - name: Install pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 + + - name: Install Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: '24' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile --filter fluxer_docs... + + - name: Verify documentation matches the live API + run: pnpm --filter fluxer_docs verify + + - name: Build documentation + run: pnpm --filter fluxer_docs build + fonts: runs-on: ubuntu-24.04 timeout-minutes: 10 diff --git a/biome.json b/biome.json index ce27b5b27..207626358 100644 --- a/biome.json +++ b/biome.json @@ -115,6 +115,17 @@ } }, "assist": {"actions": {"source": {"organizeImports": "on"}}}, + "overrides": [ + { + "includes": ["fluxer_docs/scripts/VerifyDocsCoverage.ts"], + "linter": {"rules": {"suspicious": {"noTemplateCurlyInString": "off"}}} + }, + { + "includes": ["**/*.astro"], + "linter": {"rules": {"correctness": {"noUnusedImports": "off", "noUnusedVariables": "off"}}}, + "assist": {"actions": {"source": {"organizeImports": "off"}}} + } + ], "vcs": { "enabled": true, "clientKind": "git", diff --git a/fluxer_docs/.dockerignore b/fluxer_docs/.dockerignore deleted file mode 100644 index 9da31935e..000000000 --- a/fluxer_docs/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -.cache/ -.venv/ -site/ diff --git a/fluxer_docs/.gitignore b/fluxer_docs/.gitignore index 9da31935e..97e663b17 100644 --- a/fluxer_docs/.gitignore +++ b/fluxer_docs/.gitignore @@ -1,3 +1 @@ -.cache/ -.venv/ -site/ +.astro/ diff --git a/fluxer_docs/Dockerfile b/fluxer_docs/Dockerfile index bbf8e7df3..174152775 100644 --- a/fluxer_docs/Dockerfile +++ b/fluxer_docs/Dockerfile @@ -1,28 +1,47 @@ # SPDX-License-Identifier: AGPL-3.0-or-later +ARG BUILD_VERSION -FROM python:3.13-alpine AS builder +FROM node:24-bookworm-slim AS base -WORKDIR /src/fluxer_docs +WORKDIR /usr/src/app -COPY requirements.txt requirements.txt -RUN python -m pip install --no-cache-dir --upgrade pip \ - && python -m pip install --no-cache-dir -r requirements.txt +RUN corepack enable && corepack prepare pnpm@10.29.3 --activate -COPY zensical.toml zensical.toml -COPY docs docs -COPY scripts scripts -RUN zensical build --clean \ - && python scripts/write_csp_header.py site /tmp/security-headers.conf +FROM base AS deploy -FROM nginxinc/nginx-unprivileged:1.29-alpine AS runtime +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + gcc \ + libvips-dev \ + pkg-config \ + python3 && \ + rm -rf /var/lib/apt/lists/* + +COPY . . + +RUN pnpm install --frozen-lockfile +RUN pnpm --filter @fluxer/config run --if-present generate +RUN pnpm --filter fluxer_docs run build +RUN pnpm deploy --legacy --filter=fluxer_docs --prod /out + +FROM node:24-bookworm-slim ARG BUILD_VERSION="" ARG SOURCE_SHA="" ARG SOURCE_DATE="" +WORKDIR /usr/src/app/fluxer_docs + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + libvips42 && \ + rm -rf /var/lib/apt/lists/* + +RUN corepack enable && corepack prepare pnpm@10.29.3 --activate + LABEL maintainer="Fluxer" LABEL org.opencontainers.image.title="fluxer-docs" -LABEL org.opencontainers.image.description="Fluxer documentation site" +LABEL org.opencontainers.image.description="Fluxer API and protocol reference" LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later" LABEL org.opencontainers.image.vendor="Fluxer" LABEL org.opencontainers.image.url="https://fluxer.app" @@ -33,8 +52,24 @@ LABEL org.opencontainers.image.revision="${SOURCE_SHA}" LABEL org.opencontainers.image.created="${SOURCE_DATE}" LABEL app.fluxer.build-version="${BUILD_VERSION}" -COPY nginx.conf /etc/nginx/conf.d/default.conf -COPY --from=builder /tmp/security-headers.conf /etc/nginx/conf.d/security-headers.conf -COPY --from=builder /src/fluxer_docs/site /usr/share/nginx/html +COPY --from=deploy /out ./ +COPY --from=deploy /usr/src/app/fluxer_docs/dist ./dist + +RUN mkdir -p /usr/src/app/.cache/corepack && \ + chown -R 65532:65532 /usr/src/app + +ENV HOME=/usr/src/app +ENV COREPACK_HOME=/usr/src/app/.cache/corepack +ENV NODE_ENV=production +ENV NODE_OPTIONS="--enable-source-maps" +ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt +ENV BUILD_VERSION=${BUILD_VERSION} + +USER 65532:65532 EXPOSE 8080 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD ["./node_modules/.bin/tsx", "scripts/Healthcheck.ts"] + +CMD ["./node_modules/.bin/tsx", "scripts/StartServer.ts"] diff --git a/fluxer_docs/MdastHastData.d.ts b/fluxer_docs/MdastHastData.d.ts new file mode 100644 index 000000000..5449fed83 --- /dev/null +++ b/fluxer_docs/MdastHastData.d.ts @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +/// diff --git a/fluxer_docs/astro.config.ts b/fluxer_docs/astro.config.ts new file mode 100644 index 000000000..b5d1675d5 --- /dev/null +++ b/fluxer_docs/astro.config.ts @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {type RehypePlugin, unified} from '@astrojs/markdown-remark'; +import mdx from '@astrojs/mdx'; +import node from '@astrojs/node'; +import starlight from '@astrojs/starlight'; +import {defineConfig} from 'astro/config'; +import type {Root, RootContent} from 'hast'; +import {jetBrainsMonoFont, notoSansFont, radioCanadaBigFont} from './src/fonts/Fonts'; +import {docsBaseDomain, docsListenHost, docsListenPort, docsPublicEndpoint} from './src/server/DocsConfig'; +import {resolveViteDevWebSocket} from './src/server/ViteDevWebSocket'; +import {fluxerDarkCodeTheme, fluxerLightCodeTheme} from './src/styles/CodeTheme'; +import {rehypeTableColumnWidths} from './src/table/RehypeTableColumns.ts'; + +const docsEndpoint = new URL(docsPublicEndpoint()); +const staticDirectory = fileURLToPath(new URL('../fluxer_static/', import.meta.url)); + +const resolveDocsBasePath = (endpoint: URL): string => { + if (endpoint.pathname === '/') { + return ''; + } + if (endpoint.pathname.endsWith('/')) { + return endpoint.pathname.slice(0, -1); + } + return endpoint.pathname; +}; + +const resolveDocsWebSocketPath = (basePath: string): string => { + if (basePath.length === 0) { + return '/@vite-hmr'; + } + return `${basePath}/@vite-hmr`; +}; + +const isRootRelativeHref = (href: unknown): href is string => { + if (typeof href !== 'string') { + return false; + } + if (!href.startsWith('/')) { + return false; + } + return !href.startsWith('//'); +}; + +const prefixRootRelativeLinks = (node: Root | RootContent, prefix: string): void => { + if (node.type === 'element' && node.tagName === 'a') { + const href = node.properties.href; + if (isRootRelativeHref(href)) { + node.properties.href = `${prefix}${href}`; + } + } + if (node.type !== 'root' && node.type !== 'element') { + return; + } + for (const child of node.children) { + prefixRootRelativeLinks(child, prefix); + } +}; + +const rehypeRootRelativeLinkPrefix = + (prefix: string): RehypePlugin => + () => + (tree) => { + if (prefix.length === 0) { + return; + } + prefixRootRelativeLinks(tree, prefix); + }; + +const basePath = resolveDocsBasePath(docsEndpoint); + +export default defineConfig({ + adapter: node({mode: 'middleware'}), + base: docsEndpoint.pathname, + devToolbar: { + enabled: false, + }, + fonts: [radioCanadaBigFont, notoSansFont, jetBrainsMonoFont], + integrations: [ + starlight({ + components: { + Head: './src/components/Head.astro', + Header: './src/components/Header.astro', + Sidebar: './src/components/Sidebar.astro', + TableOfContents: './src/components/TableOfContents.astro', + ThemeProvider: './src/components/ThemeProvider.astro', + ThemeSelect: './src/components/ThemeSelect.astro', + }, + customCss: [ + './src/fonts/font-stack.css', + './src/styles/tokens.css', + './src/styles/layout.css', + './src/styles/content.css', + './src/styles/route-header.css', + ], + description: 'Fluxer external API and protocol reference.', + expressiveCode: { + styleOverrides: { + borderColor: 'var(--flx-code-border)', + borderRadius: 'var(--flx-radius-lg)', + borderWidth: '1px', + codeBackground: 'var(--flx-code-bg)', + codeFontFamily: 'var(--__sl-font-mono)', + codeFontSize: 'var(--flx-code-block-font-size)', + codeForeground: 'var(--flx-code-fg)', + codeLineHeight: '1.6', + codePaddingBlock: '0.875rem', + codePaddingInline: '1.125rem', + codeSelectionBackground: 'var(--flx-code-selection-bg)', + focusBorder: 'var(--flx-link)', + frames: { + editorActiveTabBackground: 'var(--flx-code-bg)', + editorActiveTabBorderColor: 'var(--flx-code-border)', + editorActiveTabForeground: 'var(--flx-content-heading)', + editorActiveTabIndicatorBottomColor: 'transparent', + editorActiveTabIndicatorHeight: '2px', + editorActiveTabIndicatorTopColor: 'var(--flx-brand)', + editorBackground: 'var(--flx-code-bg)', + editorTabBarBackground: 'var(--flx-code-chrome-bg)', + editorTabBarBorderBottomColor: 'var(--flx-code-border)', + editorTabBorderRadius: '0', + editorTabsMarginInlineStart: '0', + frameBoxShadowCssValue: 'none', + inlineButtonBackground: 'var(--flx-text-primary)', + inlineButtonBackgroundActiveOpacity: '0.2', + inlineButtonBackgroundHoverOrFocusOpacity: '0.12', + inlineButtonBackgroundIdleOpacity: '0', + inlineButtonBorder: 'var(--flx-code-border)', + inlineButtonForeground: 'var(--flx-text-tertiary)', + shadowColor: 'transparent', + terminalBackground: 'var(--flx-code-bg)', + terminalTitlebarBackground: 'var(--flx-code-chrome-bg)', + terminalTitlebarBorderBottomColor: 'var(--flx-code-border)', + terminalTitlebarDotsForeground: 'var(--flx-code-dots)', + terminalTitlebarDotsOpacity: '1', + terminalTitlebarForeground: 'var(--flx-text-tertiary)', + tooltipSuccessBackground: 'var(--flx-accent-success)', + tooltipSuccessForeground: 'var(--flx-on-brand)', + }, + scrollbarThumbColor: 'var(--flx-scrollbar-thumb)', + scrollbarThumbHoverColor: 'var(--flx-scrollbar-thumb)', + uiFontFamily: 'var(--__sl-font)', + uiFontSize: 'var(--sl-text-xs)', + uiFontWeight: '500', + uiPaddingBlock: '0.3125rem', + uiPaddingInline: '0.75rem', + }, + themes: [fluxerDarkCodeTheme, fluxerLightCodeTheme], + }, + favicon: '/favicon-32x32.png', + logo: { + replacesTitle: true, + src: path.join(staticDirectory, 'marketing/branding/logo-color.svg'), + }, + sidebar: [ + { + label: 'Reference', + items: [{label: 'Introduction', link: '/'}, 'authentication', 'snowflakes', 'conventions'], + }, + { + label: 'Self-hosting', + items: ['operator/get-started', 'operator/configuration', 'operator/reverse-proxy', 'operator/upgrading'], + }, + { + label: 'Topics', + items: [ + 'http-api/errors', + 'topics/rate-limits', + 'http-api/permissions', + 'topics/captcha', + 'topics/uploads', + 'topics/locales', + 'http-api/deployment-availability', + ], + }, + { + label: 'HTTP API', + items: [ + {label: 'Overview', link: '/http-api/'}, + 'http-api/instance', + 'http-api/authentication', + 'http-api/gateway', + 'http-api/oauth2', + 'http-api/applications', + 'http-api/connections', + ], + }, + { + label: 'Users', + items: [ + 'http-api/users', + 'http-api/users/current-user', + 'http-api/users/settings', + 'http-api/users/settings-protobuf', + 'http-api/users/email-and-password', + 'http-api/users/mfa', + 'http-api/users/phone-verification', + 'http-api/users/relationships', + 'http-api/users/notes', + 'http-api/users/private-channels', + 'http-api/users/content', + 'http-api/users/gifts', + 'http-api/users/data-harvest', + ], + }, + { + label: 'Messaging', + items: [ + 'http-api/channels', + 'http-api/messages', + 'http-api/read-states', + 'http-api/memes', + 'http-api/gifs', + 'http-api/webhooks', + ], + }, + { + label: 'Guilds', + items: [ + 'http-api/guilds', + 'http-api/guild-channels', + 'http-api/guild-members', + 'http-api/guild-member-search', + 'http-api/guild-moderation', + 'http-api/guild-emojis', + 'http-api/guild-stickers', + 'http-api/expressions', + 'http-api/guild-audit-logs', + ], + }, + { + label: 'Discovery and content', + items: ['http-api/discovery', 'http-api/invites', 'http-api/search', 'http-api/unfurl'], + }, + { + label: 'Commerce', + items: ['http-api/billing', 'http-api/premium', 'http-api/gifts', 'http-api/donations'], + }, + { + label: 'Client surfaces', + items: ['http-api/themes', 'http-api/downloads'], + }, + { + label: 'Safety', + items: ['http-api/reports'], + }, + { + label: 'Gateway', + items: [ + 'gateway/overview', + 'gateway/commands', + 'gateway/events', + 'gateway/event-filtering', + 'gateway/limits-and-rate-limits', + 'gateway/opcodes-and-close-codes', + ], + }, + { + label: 'Media proxy', + items: [ + 'media-proxy/overview', + 'media-proxy/routes', + 'media-proxy/transformations', + 'media-proxy/upload-relay', + 'media-proxy/responses-and-limits', + ], + }, + { + label: 'Voice', + items: ['voice', 'http-api/calls', 'http-api/streams', 'http-api/entrance-sounds'], + }, + { + label: 'Admin API', + items: [ + {label: 'Overview', link: '/admin-api/'}, + 'admin-api/api-keys', + 'admin-api/users', + 'admin-api/guilds', + 'admin-api/applications', + 'admin-api/reports', + 'admin-api/messages', + 'admin-api/blocklists', + 'admin-api/discovery', + 'admin-api/system-dms', + 'admin-api/instance', + 'admin-api/gift-codes', + 'admin-api/gateway', + 'admin-api/voice', + 'admin-api/jobs', + 'admin-api/bulk-jobs', + 'admin-api/archives', + 'admin-api/search-indexes', + ], + }, + ], + social: [ + { + href: 'https://github.com/fluxerapp/fluxer', + icon: 'github', + label: 'GitHub', + }, + ], + tableOfContents: {maxHeadingLevel: 2, minHeadingLevel: 2}, + title: 'Fluxer API', + }), + mdx(), + ], + markdown: { + processor: unified({ + rehypePlugins: [rehypeRootRelativeLinkPrefix(basePath), rehypeTableColumnWidths()], + }), + }, + output: 'server', + publicDir: path.join(staticDirectory, 'web'), + site: docsEndpoint.origin, + server: { + allowedHosts: [docsBaseDomain()], + host: docsListenHost(), + port: docsListenPort(), + }, + vite: { + resolve: { + alias: { + '@': fileURLToPath(new URL('./src', import.meta.url)), + }, + }, + server: { + fs: { + deny: [ + '.env', + '.env.*', + '*.{crt,pem,key,p12,pfx,cer,der}', + '.npmrc', + '.yarnrc.yml', + '**/.git/**', + '**/.fluxer/**', + '**/.devcontainer/**', + '**/.ssh/**', + '**/.aws/**', + '**/.envrc', + '**/*.{toml,yaml,yml}', + '**/*.{sqlite,sqlite3,db}', + ], + }, + ws: resolveViteDevWebSocket({endpoint: docsEndpoint, path: resolveDocsWebSocketPath(basePath)}), + }, + }, +}); diff --git a/fluxer_docs/docs/api-reference.md b/fluxer_docs/docs/api-reference.md deleted file mode 100644 index b77df682a..000000000 --- a/fluxer_docs/docs/api-reference.md +++ /dev/null @@ -1,7 +0,0 @@ -# API Reference - -The Fluxer API is described by an OpenAPI specification. Use it to explore endpoints, generate client libraries, or import the API into tooling such as Postman, Insomnia, or a Swagger/Redoc viewer. - -- **OpenAPI spec**: [`api.fluxer.app/openapi.json`](https://api.fluxer.app/openapi.json) - -*The full endpoint-by-endpoint reference is landing over the coming days.* diff --git a/fluxer_docs/docs/assets/favicon.svg b/fluxer_docs/docs/assets/favicon.svg deleted file mode 100644 index dfb149804..000000000 --- a/fluxer_docs/docs/assets/favicon.svg +++ /dev/null @@ -1,5 +0,0 @@ - - - - - \ No newline at end of file diff --git a/fluxer_docs/docs/assets/logo.svg b/fluxer_docs/docs/assets/logo.svg deleted file mode 100644 index dfb149804..000000000 --- a/fluxer_docs/docs/assets/logo.svg +++ /dev/null @@ -1,5 +0,0 @@ - - - - - \ No newline at end of file diff --git a/fluxer_docs/docs/authentication.md b/fluxer_docs/docs/authentication.md deleted file mode 100644 index 14d689f3e..000000000 --- a/fluxer_docs/docs/authentication.md +++ /dev/null @@ -1,3 +0,0 @@ -# Authentication - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/gateway/events.md b/fluxer_docs/docs/gateway/events.md deleted file mode 100644 index b2d222946..000000000 --- a/fluxer_docs/docs/gateway/events.md +++ /dev/null @@ -1,3 +0,0 @@ -# Events - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/gateway/opcodes-and-close-codes.md b/fluxer_docs/docs/gateway/opcodes-and-close-codes.md deleted file mode 100644 index 021a7956a..000000000 --- a/fluxer_docs/docs/gateway/opcodes-and-close-codes.md +++ /dev/null @@ -1,3 +0,0 @@ -# Opcodes and Close Codes - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/gateway/overview.md b/fluxer_docs/docs/gateway/overview.md deleted file mode 100644 index 7c1d5638a..000000000 --- a/fluxer_docs/docs/gateway/overview.md +++ /dev/null @@ -1,3 +0,0 @@ -# Overview - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/index.md b/fluxer_docs/docs/index.md deleted file mode 100644 index bb953306d..000000000 --- a/fluxer_docs/docs/index.md +++ /dev/null @@ -1,8 +0,0 @@ -# Introduction - -Welcome to the Fluxer documentation. - -These docs are split into two parts: - -- **API Docs** for building on the Fluxer API. -- **Operator Guide** for self-hosting your own Fluxer instance. diff --git a/fluxer_docs/docs/operator/configuration.md b/fluxer_docs/docs/operator/configuration.md deleted file mode 100644 index 6a45a1a2f..000000000 --- a/fluxer_docs/docs/operator/configuration.md +++ /dev/null @@ -1,5 +0,0 @@ -# Configuration - -Configure your Fluxer instance. - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/operator/contributing.md b/fluxer_docs/docs/operator/contributing.md deleted file mode 100644 index b3ad419a6..000000000 --- a/fluxer_docs/docs/operator/contributing.md +++ /dev/null @@ -1,5 +0,0 @@ -# Contributing - -Contribute bug fixes and discuss ideas for Fluxer on GitHub. - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/operator/get-started.md b/fluxer_docs/docs/operator/get-started.md deleted file mode 100644 index e897f8c64..000000000 --- a/fluxer_docs/docs/operator/get-started.md +++ /dev/null @@ -1,293 +0,0 @@ -# Get Started - -Run your own Fluxer instance with Docker Compose. This guide takes you from a fresh server to a working self-hosted instance with the web app, API, gateway, admin dashboard, media uploads, search, storage, and voice signaling behind one public hostname. - -!!! info "Desktop client support is coming very soon" - The desktop client cannot connect to self-hosted instances yet, so reach your instance through the web app in the meantime. Support is actively being worked on and lands very soon. You can track the progress in [issue #1088](https://github.com/fluxerapp/fluxer/issues/1088#issuecomment-4951728735). - -## Requirements - -- A Linux server or VM that can run Docker Engine. -- Docker Engine plus the Docker Compose plugin. -- A hostname for the instance, for example `chat.example.com`. -- Either public inbound `80/tcp` and `443/tcp`, or a Cloudflare Tunnel that routes the hostname to the Caddy container. -- For production voice and video media, a public path to `7881/tcp` and `7882/udp`. -- At least 2 vCPU, 4 GB RAM, and 20 GB disk. Use 4 vCPU and 8 GB RAM or more for a small active community. - -The stack idles around a few GB of memory, and startup is the heaviest point because all service images initialize at once. - -## Step 1: Install Docker - -Install Docker Engine from Docker's official instructions for your distribution: - -- [Install Docker Engine](https://docs.docker.com/engine/install/) -- [Install the Compose plugin](https://docs.docker.com/compose/install/linux/) -- [Linux post-installation steps](https://docs.docker.com/engine/install/linux-postinstall/) - -Confirm the versions: - -```bash -docker --version -docker compose version -``` - -Use Docker Engine 24 or newer and Compose v2.20.2 or newer. Older Compose plugins reject the API healthcheck's `start_interval` key. Engine 25 or newer acts on that key and brings the stack up faster. - -## Step 2: Download the stack - -Create a working directory and download the stack files: - -```bash -mkdir fluxer -cd fluxer - -base=https://raw.githubusercontent.com/fluxerapp/fluxer/961fa1f007b5d7ecd49739463c6aa3185e2eab7a/deploy/self-hosting -curl -fsSLO "$base/docker-compose.yml" -curl -fsSLO "$base/Caddyfile" -curl -fsSLO "$base/livekit.yaml" -curl -fsSLO "$base/tunnel.compose.yml" -curl -fsSL "$base/.env.example" -o .env -``` - -You should now have: - -```text -Caddyfile -docker-compose.yml -livekit.yaml -tunnel.compose.yml -.env -``` - -## Step 3: Configure `.env` - -Set the public hostname at the top of `.env`. - -For a normal public server where Caddy obtains certificates directly: - -```ini -FLUXER_DOMAIN=chat.example.com -FLUXER_PUBLIC_SCHEME=https -FLUXER_PUBLIC_PORT=443 -FLUXER_CADDY_SITE_ADDRESS=chat.example.com -FLUXER_VAPID_EMAIL=admin@example.com -``` - -For a Cloudflare Tunnel where Cloudflare terminates HTTPS and forwards HTTP to Caddy: - -```ini -FLUXER_DOMAIN=chat.example.com -FLUXER_PUBLIC_SCHEME=https -FLUXER_PUBLIC_PORT=443 -FLUXER_CADDY_SITE_ADDRESS=:80 -FLUXER_VAPID_EMAIL=admin@example.com -``` - -`FLUXER_PUBLIC_SCHEME` and `FLUXER_PUBLIC_PORT` describe what users see in their browser. `FLUXER_CADDY_SITE_ADDRESS` describes what Caddy listens on inside the stack. - -Generate the required secrets: - -```bash -for key in POSTGRES_PASSWORD MEILI_MASTER_KEY FLUXER_S3_SECRET_KEY \ - FLUXER_SUDO_MODE_SECRET FLUXER_CONNECTION_INITIATION_SECRET \ - FLUXER_GATEWAY_RPC_AUTH_TOKEN FLUXER_MEDIA_PROXY_SECRET_KEY \ - FLUXER_ADMIN_SECRET_KEY_BASE FLUXER_ADMIN_OAUTH_CLIENT_SECRET \ - LIVEKIT_API_SECRET; do - sed -i "s|^$key=.*|$key=$(openssl rand -hex 32)|" .env -done - -sed -i "s|^FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=.*|FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=$(openssl rand -base64 32)|" .env - -VAPID=$(docker run --rm node:24-alpine npx --yes web-push generate-vapid-keys --json) -pub=$(printf '%s' "$VAPID" | grep -o '"publicKey":"[^"]*"' | cut -d'"' -f4) -priv=$(printf '%s' "$VAPID" | grep -o '"privateKey":"[^"]*"' | cut -d'"' -f4) -sed -i "s|^FLUXER_VAPID_PUBLIC_KEY=.*|FLUXER_VAPID_PUBLIC_KEY=$pub|" .env -sed -i "s|^FLUXER_VAPID_PRIVATE_KEY=.*|FLUXER_VAPID_PRIVATE_KEY=$priv|" .env -``` - -Keep these defaults unless you know you need to change them: - -- `LIVEKIT_API_KEY=fluxer`; the secret is `LIVEKIT_API_SECRET`. -- `FLUXER_S3_ACCESS_KEY=fluxer`; the secret is `FLUXER_S3_SECRET_KEY`. -- Email starts disabled. Enable SMTP later from `.env` and the admin dashboard. -- Passkeys follow `FLUXER_DOMAIN`. Override `FLUXER_PASSKEY_RP_ID`, `FLUXER_PASSKEY_RP_NAME`, and `FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS` only when browsers reach the instance on a different host. - -!!! warning "Changing the passkey relying party invalidates passkeys" - Browsers bind each passkey to the `FLUXER_PASSKEY_RP_ID` it was registered under. Changing that value later leaves every existing passkey unusable, so members have to enrol again. - -!!! warning "Keep `.env` private" - `.env` contains every secret for the instance. Do not commit it, paste it into support tickets, or put it in screenshots. - -## Step 4: Publish the hostname - -=== "Direct public server" - - Create DNS records for the hostname: - - - `A` record from `chat.example.com` to the server IPv4 address. - - Optional `AAAA` record from `chat.example.com` to the server IPv6 address. - - Leave `FLUXER_CADDY_SITE_ADDRESS=chat.example.com`. Caddy will request and renew certificates automatically when `80/tcp` and `443/tcp` can reach the server. - -=== "Cloudflare Tunnel" - - Use this when the server should not expose public web ports. - - 1. Set `FLUXER_CADDY_SITE_ADDRESS=:80` and `COMPOSE_FILE=docker-compose.yml:tunnel.compose.yml`. Without the second value Caddy still publishes 80, 443 and 443/udp on every interface, which is what `tunnel.compose.yml` narrows to a single loopback publish on 80. It needs Compose 2.24.4 or newer. - 2. In Cloudflare, create a Tunnel public hostname for your Fluxer domain. - 3. If `cloudflared` runs inside the Compose project, point the public hostname service to `http://caddy:80`. - 4. If `cloudflared` runs directly on the host, point the public hostname service to `http://127.0.0.1:80`. - - A temporary Compose override keeps the tunnel next to Caddy without saving the token in your main stack: - - ```bash - cat > cloudflared.compose.yml <<'YAML' - services: - cloudflared: - image: cloudflare/cloudflared:latest - restart: unless-stopped - command: tunnel run --token ${CLOUDFLARED_TOKEN:?set CLOUDFLARED_TOKEN} - depends_on: - - caddy - networks: - - fluxer - YAML - - export CLOUDFLARED_TOKEN='paste-your-tunnel-token-here' - docker compose -f docker-compose.yml -f cloudflared.compose.yml up -d cloudflared - ``` - - !!! warning "Voice media is not carried by a normal public hostname tunnel" - The web app, API, admin dashboard, gateway WebSocket, media proxy HTTP routes, and LiveKit signaling can work through the tunnel. LiveKit WebRTC media still needs reachable `7881/tcp` and `7882/udp`, or a TURN deployment. - -## Step 5: Open the firewall - -If you are using a direct public server, allow inbound: - -- `22/tcp` or your SSH port. -- `80/tcp` and `443/tcp` for Caddy. -- `7881/tcp` and `7882/udp` for LiveKit media. - -If you are using a Cloudflare Tunnel for web traffic, you can block inbound `80/tcp` and `443/tcp` at the provider firewall. Keep LiveKit media closed too unless you are intentionally exposing voice/video media or using a TURN server. - -!!! warning "Provider firewall first" - Docker-published ports can bypass host firewalls such as UFW because Docker installs its own packet-filtering rules. Prefer your cloud provider's firewall or security group for internet-facing policy. - -## Step 6: Start the stack - -Start Fluxer: - -```bash -docker compose up -d -``` - -If you are using the Cloudflare override from above, start both files together: - -```bash -docker compose -f docker-compose.yml -f cloudflared.compose.yml up -d -``` - -Watch the startup: - -```bash -docker compose ps -docker compose logs -f api -``` - -The first start can take several minutes while images download and services initialize. `seaweedfs-init` exits after creating object-storage buckets; that is expected. - -## Step 7: Create the owner account - -Open the web app: - -```text -https://chat.example.com -``` - -Register the first account. On a self-hosted instance, the first accepted registration receives wildcard admin access. Use that account for the initial admin login: - -```text -https://chat.example.com/admin -``` - -Complete the initial setup from the admin dashboard. At minimum, review: - -- Branding and instance name. -- Registration mode: open, approval, or closed. -- Email delivery. -- Captcha policy if you open public registration. -- Single-community mode if you want one default community instead of many user-created communities. -- Voice regions and LiveKit reachability if you are enabling voice. - -## Email - -Email is disabled by default. To enable SMTP, set these in `.env` and restart `api`, `worker`, and `admin`: - -```ini -FLUXER_EMAIL_ENABLED=true -FLUXER_EMAIL_PROVIDER=smtp -FLUXER_EMAIL_FROM_EMAIL=noreply@example.com -FLUXER_EMAIL_FROM_NAME=Fluxer -FLUXER_EMAIL_SMTP_HOST=smtp.example.com -FLUXER_EMAIL_SMTP_PORT=587 -FLUXER_EMAIL_SMTP_USERNAME=example -FLUXER_EMAIL_SMTP_PASSWORD=example-secret -FLUXER_EMAIL_SMTP_SECURE=true -``` - -Then test the SMTP configuration from `/admin/instance-config`. - -## Voice and video - -Fluxer uses LiveKit for voice and video. Caddy routes `/livekit` to LiveKit's HTTP/WebSocket signaling port, but browser media flows over WebRTC: - -- `7882/udp` is the normal media path. -- `7881/tcp` is the TCP fallback path. -- `7880/tcp` stays private behind Caddy for signaling. - -On a VPS with `7881/tcp` and `7882/udp` open, LiveKit can usually auto-detect the public IP. Behind NAT, Cloudflare Tunnel, or restrictive networks, add a TURN server and configure LiveKit for it. - -## Backups - -Back up these items before upgrades and on a regular schedule: - -- `.env` -- `postgres-data` -- `seaweedfs-data` - -For a cold backup: - -```bash -docker compose stop api worker gateway admin app-proxy media-proxy static-proxy livekit -docker run --rm -v fluxer_postgres-data:/data -v "$PWD/backups:/backup" alpine tar czf /backup/postgres-data.tgz -C /data . -docker run --rm -v fluxer_seaweedfs-data:/data -v "$PWD/backups:/backup" alpine tar czf /backup/seaweedfs-data.tgz -C /data . -docker compose up -d -``` - -For production, prefer a Postgres-native dump plus object-storage backup so you do not need to stop the instance. - -## Upgrading - -The default image tag is `v1`, which tracks the latest compatible release: - -```bash -docker compose pull -docker compose up -d -``` - -The `fluxer-static` image is part of the default stack, so static asset updates are picked up by the same pull-and-restart flow. - -To pin a specific release, set `FLUXER_IMAGE_TAG` in `.env` to the release tag you want, then pull and restart. - -Each upgrade leaves the previous images behind, which adds up to a few gigabytes over time. To reclaim that space: - -```bash -docker image prune -f -``` - -This removes only the untagged images the upgrade replaced. `docker system prune -a` reclaims more, but it also deletes every image not currently used by a running container, including ones unrelated to Fluxer. - -## Getting help - -- File issues and follow development on [GitHub](https://github.com/fluxerapp/fluxer). -- For direct access to the team, see [Operator Pass](operator-pass.md). diff --git a/fluxer_docs/docs/operator/operator-pass.md b/fluxer_docs/docs/operator/operator-pass.md deleted file mode 100644 index 59f03dc88..000000000 --- a/fluxer_docs/docs/operator/operator-pass.md +++ /dev/null @@ -1,5 +0,0 @@ -# Operator Pass - -A one-time donation with perks that gets you access to our team. - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/operator/upgrading.md b/fluxer_docs/docs/operator/upgrading.md deleted file mode 100644 index b8d171817..000000000 --- a/fluxer_docs/docs/operator/upgrading.md +++ /dev/null @@ -1,5 +0,0 @@ -# Upgrading - -Keep your Fluxer instance up to date. - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/applications.md b/fluxer_docs/docs/resources/applications.md deleted file mode 100644 index 58da3abed..000000000 --- a/fluxer_docs/docs/resources/applications.md +++ /dev/null @@ -1,3 +0,0 @@ -# Applications - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/audit-log.md b/fluxer_docs/docs/resources/audit-log.md deleted file mode 100644 index 49d70f228..000000000 --- a/fluxer_docs/docs/resources/audit-log.md +++ /dev/null @@ -1,3 +0,0 @@ -# Audit Log - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/channels.md b/fluxer_docs/docs/resources/channels.md deleted file mode 100644 index 421873d5a..000000000 --- a/fluxer_docs/docs/resources/channels.md +++ /dev/null @@ -1,3 +0,0 @@ -# Channels - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/connected-accounts.md b/fluxer_docs/docs/resources/connected-accounts.md deleted file mode 100644 index 3f4288f6c..000000000 --- a/fluxer_docs/docs/resources/connected-accounts.md +++ /dev/null @@ -1,3 +0,0 @@ -# Connected Accounts - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/discovery.md b/fluxer_docs/docs/resources/discovery.md deleted file mode 100644 index ead80ebe8..000000000 --- a/fluxer_docs/docs/resources/discovery.md +++ /dev/null @@ -1,3 +0,0 @@ -# Discovery - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/emoji.md b/fluxer_docs/docs/resources/emoji.md deleted file mode 100644 index 392ccbe4b..000000000 --- a/fluxer_docs/docs/resources/emoji.md +++ /dev/null @@ -1,3 +0,0 @@ -# Emoji - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/guilds.md b/fluxer_docs/docs/resources/guilds.md deleted file mode 100644 index c427d71dc..000000000 --- a/fluxer_docs/docs/resources/guilds.md +++ /dev/null @@ -1,3 +0,0 @@ -# Guilds - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/invites.md b/fluxer_docs/docs/resources/invites.md deleted file mode 100644 index 140f8f547..000000000 --- a/fluxer_docs/docs/resources/invites.md +++ /dev/null @@ -1,3 +0,0 @@ -# Invites - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/messages.md b/fluxer_docs/docs/resources/messages.md deleted file mode 100644 index 2ca6e1bca..000000000 --- a/fluxer_docs/docs/resources/messages.md +++ /dev/null @@ -1,3 +0,0 @@ -# Messages - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/presences.md b/fluxer_docs/docs/resources/presences.md deleted file mode 100644 index 6bca15665..000000000 --- a/fluxer_docs/docs/resources/presences.md +++ /dev/null @@ -1,3 +0,0 @@ -# Presences - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/relationships.md b/fluxer_docs/docs/resources/relationships.md deleted file mode 100644 index 3e73d2b56..000000000 --- a/fluxer_docs/docs/resources/relationships.md +++ /dev/null @@ -1,3 +0,0 @@ -# Relationships - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/stickers.md b/fluxer_docs/docs/resources/stickers.md deleted file mode 100644 index c9f21d393..000000000 --- a/fluxer_docs/docs/resources/stickers.md +++ /dev/null @@ -1,3 +0,0 @@ -# Stickers - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/user-settings-proto.md b/fluxer_docs/docs/resources/user-settings-proto.md deleted file mode 100644 index 791e971ec..000000000 --- a/fluxer_docs/docs/resources/user-settings-proto.md +++ /dev/null @@ -1,3 +0,0 @@ -# User Settings Proto - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/user-settings.md b/fluxer_docs/docs/resources/user-settings.md deleted file mode 100644 index 533656aa9..000000000 --- a/fluxer_docs/docs/resources/user-settings.md +++ /dev/null @@ -1,3 +0,0 @@ -# User Settings - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/users.md b/fluxer_docs/docs/resources/users.md deleted file mode 100644 index 7df05143c..000000000 --- a/fluxer_docs/docs/resources/users.md +++ /dev/null @@ -1,3 +0,0 @@ -# Users - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/voice.md b/fluxer_docs/docs/resources/voice.md deleted file mode 100644 index 1060d4a5d..000000000 --- a/fluxer_docs/docs/resources/voice.md +++ /dev/null @@ -1,3 +0,0 @@ -# Voice - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/resources/webhooks.md b/fluxer_docs/docs/resources/webhooks.md deleted file mode 100644 index dcd087006..000000000 --- a/fluxer_docs/docs/resources/webhooks.md +++ /dev/null @@ -1,3 +0,0 @@ -# Webhooks - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/stylesheets/fluxer.css b/fluxer_docs/docs/stylesheets/fluxer.css deleted file mode 100644 index 1e2de768d..000000000 --- a/fluxer_docs/docs/stylesheets/fluxer.css +++ /dev/null @@ -1,98 +0,0 @@ -body { - --md-text-font-family: - system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, - 'Noto Sans', sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol', 'Noto Color Emoji'; - --md-code-font-family: - ui-monospace, SFMono-Regular, 'SF Mono', Menlo, Consolas, 'Liberation Mono', 'Roboto Mono', 'Courier New', monospace; - - --fluxer-display: var(--md-text-font-family); -} - -:root { - --md-primary-fg-color: #4641d9; - --md-primary-fg-color--light: #5b56e0; - --md-primary-fg-color--dark: #3832b8; - --md-accent-fg-color: #3832b8; -} - -[data-md-color-scheme='slate'] { - --md-primary-fg-color: #6f6af0; - --md-primary-fg-color--light: #8a86f4; - --md-primary-fg-color--dark: #4641d9; - --md-accent-fg-color: #8a86f4; -} - -.md-typeset h1, -.md-typeset h2, -.md-typeset h3, -.md-typeset h4, -.md-typeset h5, -.md-typeset h6, -.md-header__title, -.md-tabs__link, -.md-nav__title { - font-family: var(--fluxer-display); - letter-spacing: 0; -} - -html { - font-size: 112.5%; -} - -@media screen and (min-width: 100em) { - html { - font-size: 125%; - } -} - -@media screen and (min-width: 125em) { - html { - font-size: 137.5%; - } -} - -.md-typeset { - font-size: 0.72rem; - line-height: 1.55; -} - -.md-typeset h1 { - font-size: 1.7em; - line-height: 1.2; - margin: 0 0 0.7em; -} - -.md-typeset h2 { - font-size: 1.4em; - line-height: 1.25; - margin: 1.4em 0 0.5em; -} - -.md-typeset h3 { - font-size: 1.15em; - line-height: 1.3; - margin: 1.2em 0 0.4em; -} - -.md-typeset h4 { - font-size: 1em; - margin: 0.9em 0 0.3em; -} - -.md-typeset p, -.md-typeset ul, -.md-typeset ol { - margin: 0.5em 0; -} - -.md-nav { - font-size: 0.66rem; -} - -.md-nav__link { - padding: 0.25rem 0.6rem; -} - -.md-tabs__link { - font-size: 0.68rem; -} diff --git a/fluxer_docs/docs/topics/captcha.md b/fluxer_docs/docs/topics/captcha.md deleted file mode 100644 index b12a6f6c8..000000000 --- a/fluxer_docs/docs/topics/captcha.md +++ /dev/null @@ -1,3 +0,0 @@ -# CAPTCHA - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/topics/client-distribution.md b/fluxer_docs/docs/topics/client-distribution.md deleted file mode 100644 index 09722a5d9..000000000 --- a/fluxer_docs/docs/topics/client-distribution.md +++ /dev/null @@ -1,3 +0,0 @@ -# Client Distribution - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/topics/errors.md b/fluxer_docs/docs/topics/errors.md deleted file mode 100644 index 785525825..000000000 --- a/fluxer_docs/docs/topics/errors.md +++ /dev/null @@ -1,3 +0,0 @@ -# Errors - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/topics/oauth2.md b/fluxer_docs/docs/topics/oauth2.md deleted file mode 100644 index c1ae14fb5..000000000 --- a/fluxer_docs/docs/topics/oauth2.md +++ /dev/null @@ -1,3 +0,0 @@ -# OAuth2 - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/topics/permissions.md b/fluxer_docs/docs/topics/permissions.md deleted file mode 100644 index 3df849958..000000000 --- a/fluxer_docs/docs/topics/permissions.md +++ /dev/null @@ -1,3 +0,0 @@ -# Permissions - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/topics/push-notifications.md b/fluxer_docs/docs/topics/push-notifications.md deleted file mode 100644 index b9bc45711..000000000 --- a/fluxer_docs/docs/topics/push-notifications.md +++ /dev/null @@ -1,3 +0,0 @@ -# Push Notifications - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/topics/rate-limits.md b/fluxer_docs/docs/topics/rate-limits.md deleted file mode 100644 index 92ad498b1..000000000 --- a/fluxer_docs/docs/topics/rate-limits.md +++ /dev/null @@ -1,3 +0,0 @@ -# Rate Limits - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/docs/topics/read-states.md b/fluxer_docs/docs/topics/read-states.md deleted file mode 100644 index 205fe05fe..000000000 --- a/fluxer_docs/docs/topics/read-states.md +++ /dev/null @@ -1,3 +0,0 @@ -# Read States - -*We're still writing this page. More documentation is landing over the coming days.* diff --git a/fluxer_docs/nginx.conf b/fluxer_docs/nginx.conf deleted file mode 100644 index 2de7fd2fa..000000000 --- a/fluxer_docs/nginx.conf +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-or-later - -server { - listen 8080; - server_name _; - root /usr/share/nginx/html; - index index.html; - - include /etc/nginx/conf.d/security-headers.conf; - - location = /_health { - access_log off; - return 204; - } - - location = /_healthz { - access_log off; - return 204; - } - - location = /404.html { - internal; - } - - location ^~ /assets/ { - try_files $uri =404; - include /etc/nginx/conf.d/security-headers.conf; - add_header Cache-Control "public, max-age=31536000, immutable" always; - } - - location ~* \.(?:css|js|mjs|svg|png|jpg|jpeg|gif|webp|avif|ico)$ { - try_files $uri =404; - include /etc/nginx/conf.d/security-headers.conf; - add_header Cache-Control "public, max-age=3600, must-revalidate" always; - } - - location / { - try_files $uri $uri/ =404; - } - - error_page 404 /404.html; -} diff --git a/fluxer_docs/package.json b/fluxer_docs/package.json new file mode 100644 index 000000000..b792a9a8d --- /dev/null +++ b/fluxer_docs/package.json @@ -0,0 +1,37 @@ +{ + "name": "fluxer_docs", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "astro build", + "dev": "astro dev", + "start": "tsx scripts/StartServer.ts", + "typecheck": "astro check --minimumFailingSeverity warning", + "verify": "pnpm verify:sidebar && pnpm verify:coverage && pnpm verify:schemas && pnpm verify:style", + "verify:coverage": "tsx scripts/VerifyDocsCoverage.ts", + "verify:schemas": "tsx scripts/VerifyDocsSchemas.ts", + "verify:sidebar": "tsx scripts/VerifyDocsSidebar.ts", + "verify:style": "tsx scripts/VerifyDocsStyle.ts" + }, + "dependencies": { + "@astrojs/markdown-remark": "catalog:", + "@astrojs/mdx": "catalog:", + "@astrojs/node": "catalog:", + "@astrojs/starlight": "catalog:", + "@fluxer/config": "workspace:*", + "astro": "catalog:", + "send": "catalog:", + "sharp": "catalog:", + "tsx": "catalog:" + }, + "devDependencies": { + "@astrojs/check": "catalog:", + "@fluxer/openapi": "workspace:*", + "@types/hast": "catalog:", + "@types/node": "catalog:", + "@types/send": "catalog:", + "mdast-util-to-hast": "catalog:", + "typescript": "6.0.2" + } +} diff --git a/fluxer_docs/requirements.txt b/fluxer_docs/requirements.txt deleted file mode 100644 index 75fa97c00..000000000 --- a/fluxer_docs/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -zensical==0.0.45 diff --git a/fluxer_docs/scripts/DocsTableGlyphs.json b/fluxer_docs/scripts/DocsTableGlyphs.json new file mode 100644 index 000000000..51775648c --- /dev/null +++ b/fluxer_docs/scripts/DocsTableGlyphs.json @@ -0,0 +1,426 @@ +{ + "prose": { + "0": 8.554, + "1": 4.9, + "2": 7.98, + "3": 8.008, + "4": 7.868, + "5": 8.148, + "6": 8.218, + "7": 7.353, + "8": 7.966, + "9": 8.218, + " ": 3.08, + "!": 3.78, + "\"": 5.068, + "#": 8.736, + "$": 7.84, + "%": 10.346, + "&": 8.274, + "'": 2.828, + "(": 4.536, + ")": 4.536, + "*": 7.378, + "+": 7.462, + ",": 2.688, + "-": 5.25, + ".": 3.08, + "/": 4.088, + ":": 3.08, + ";": 3.08, + "<": 7.462, + "=": 7.462, + ">": 7.462, + "?": 7.77, + "@": 12.516, + "A": 9.24, + "B": 8.722, + "C": 9.338, + "D": 9.492, + "E": 8.554, + "F": 8.064, + "G": 9.856, + "H": 9.87, + "I": 3.57, + "J": 7.84, + "K": 8.974, + "L": 7.546, + "M": 12.068, + "N": 9.87, + "O": 9.758, + "P": 8.582, + "Q": 9.758, + "R": 8.666, + "S": 8.288, + "T": 7.997, + "U": 9.45, + "V": 9.24, + "W": 12.894, + "X": 8.54, + "Y": 8.4, + "Z": 8.652, + "[": 4.354, + "\\": 4.088, + "]": 4.354, + "^": 6.72, + "_": 6.09, + "`": 7, + "a": 7.518, + "b": 8.008, + "c": 7.28, + "d": 8.008, + "e": 7.714, + "f": 4.641, + "g": 8.008, + "h": 8.036, + "i": 3.22, + "j": 3.495, + "k": 7.21, + "l": 3.22, + "m": 12.012, + "n": 8.036, + "o": 7.854, + "p": 8.008, + "q": 8.008, + "r": 4.872, + "s": 6.888, + "t": 5.138, + "u": 7.98, + "v": 7.112, + "w": 10.752, + "x": 7.084, + "y": 7.112, + "z": 6.818, + "{": 4.732, + "|": 3.5, + "}": 4.732, + "~": 7, + "→": 12.53, + "—": 11.41, + "‘": 2.688, + "’": 2.688, + "“": 4.928, + "”": 4.928, + " ": 3.08, + "≤": 8.668, + "≥": 8.668 + }, + "code": { + "0": 7.872, + "1": 7.872, + "2": 7.872, + "3": 7.872, + "4": 7.872, + "5": 7.872, + "6": 7.872, + "7": 7.872, + "8": 7.872, + "9": 7.872, + " ": 7.872, + "!": 7.872, + "\"": 7.872, + "#": 7.872, + "$": 7.872, + "%": 7.872, + "&": 7.872, + "'": 7.872, + "(": 7.872, + ")": 7.872, + "*": 7.872, + "+": 7.872, + ",": 7.872, + "-": 7.872, + ".": 7.872, + "/": 7.872, + ":": 7.872, + ";": 7.872, + "<": 7.872, + "=": 7.872, + ">": 7.872, + "?": 7.872, + "@": 7.872, + "A": 7.872, + "B": 7.872, + "C": 7.872, + "D": 7.872, + "E": 7.872, + "F": 7.872, + "G": 7.872, + "H": 7.872, + "I": 7.872, + "J": 7.872, + "K": 7.872, + "L": 7.872, + "M": 7.872, + "N": 7.872, + "O": 7.872, + "P": 7.872, + "Q": 7.872, + "R": 7.872, + "S": 7.872, + "T": 7.872, + "U": 7.872, + "V": 7.872, + "W": 7.872, + "X": 7.872, + "Y": 7.872, + "Z": 7.872, + "[": 7.872, + "\\": 7.872, + "]": 7.872, + "^": 7.872, + "_": 7.872, + "`": 7.872, + "a": 7.872, + "b": 7.872, + "c": 7.872, + "d": 7.872, + "e": 7.872, + "f": 7.872, + "g": 7.872, + "h": 7.872, + "i": 7.872, + "j": 7.872, + "k": 7.872, + "l": 7.872, + "m": 7.872, + "n": 7.872, + "o": 7.872, + "p": 7.872, + "q": 7.872, + "r": 7.872, + "s": 7.872, + "t": 7.872, + "u": 7.872, + "v": 7.872, + "w": 7.872, + "x": 7.872, + "y": 7.872, + "z": 7.872, + "{": 7.872, + "|": 7.872, + "}": 7.872, + "~": 7.872, + "→": 7.899, + "—": 7.872, + "‘": 7.872, + "’": 7.872, + "“": 7.872, + "”": 7.872, + " ": 7.872, + "≤": 7.899, + "≥": 7.899 + }, + "proseBold": { + "0": 8.554, + "1": 4.9, + "2": 7.98, + "3": 8.008, + "4": 7.868, + "5": 8.148, + "6": 8.218, + "7": 7.353, + "8": 7.966, + "9": 8.218, + " ": 3.08, + "!": 3.78, + "\"": 5.068, + "#": 8.736, + "$": 7.84, + "%": 10.346, + "&": 8.274, + "'": 2.828, + "(": 4.536, + ")": 4.536, + "*": 7.378, + "+": 7.462, + ",": 2.688, + "-": 5.25, + ".": 3.08, + "/": 4.088, + ":": 3.08, + ";": 3.08, + "<": 7.462, + "=": 7.462, + ">": 7.462, + "?": 7.77, + "@": 12.516, + "A": 9.24, + "B": 8.722, + "C": 9.338, + "D": 9.492, + "E": 8.554, + "F": 8.064, + "G": 9.856, + "H": 9.87, + "I": 3.57, + "J": 7.84, + "K": 8.974, + "L": 7.546, + "M": 12.068, + "N": 9.87, + "O": 9.758, + "P": 8.582, + "Q": 9.758, + "R": 8.666, + "S": 8.288, + "T": 7.997, + "U": 9.45, + "V": 9.24, + "W": 12.894, + "X": 8.54, + "Y": 8.4, + "Z": 8.652, + "[": 4.354, + "\\": 4.088, + "]": 4.354, + "^": 6.72, + "_": 6.09, + "`": 7, + "a": 7.518, + "b": 8.008, + "c": 7.28, + "d": 8.008, + "e": 7.714, + "f": 4.641, + "g": 8.008, + "h": 8.036, + "i": 3.22, + "j": 3.495, + "k": 7.21, + "l": 3.22, + "m": 12.012, + "n": 8.036, + "o": 7.854, + "p": 8.008, + "q": 8.008, + "r": 4.872, + "s": 6.888, + "t": 5.138, + "u": 7.98, + "v": 7.112, + "w": 10.752, + "x": 7.084, + "y": 7.112, + "z": 6.818, + "{": 4.732, + "|": 3.5, + "}": 4.732, + "~": 7, + "→": 12.53, + "—": 11.41, + "‘": 2.688, + "’": 2.688, + "“": 4.928, + "”": 4.928, + " ": 3.08, + "≤": 8.668, + "≥": 8.668 + }, + "codeBold": { + "0": 7.872, + "1": 7.872, + "2": 7.872, + "3": 7.872, + "4": 7.872, + "5": 7.872, + "6": 7.872, + "7": 7.872, + "8": 7.872, + "9": 7.872, + " ": 7.872, + "!": 7.872, + "\"": 7.872, + "#": 7.872, + "$": 7.872, + "%": 7.872, + "&": 7.872, + "'": 7.872, + "(": 7.872, + ")": 7.872, + "*": 7.872, + "+": 7.872, + ",": 7.872, + "-": 7.872, + ".": 7.872, + "/": 7.872, + ":": 7.872, + ";": 7.872, + "<": 7.872, + "=": 7.872, + ">": 7.872, + "?": 7.872, + "@": 7.872, + "A": 7.872, + "B": 7.872, + "C": 7.872, + "D": 7.872, + "E": 7.872, + "F": 7.872, + "G": 7.872, + "H": 7.872, + "I": 7.872, + "J": 7.872, + "K": 7.872, + "L": 7.872, + "M": 7.872, + "N": 7.872, + "O": 7.872, + "P": 7.872, + "Q": 7.872, + "R": 7.872, + "S": 7.872, + "T": 7.872, + "U": 7.872, + "V": 7.872, + "W": 7.872, + "X": 7.872, + "Y": 7.872, + "Z": 7.872, + "[": 7.872, + "\\": 7.872, + "]": 7.872, + "^": 7.872, + "_": 7.872, + "`": 7.872, + "a": 7.872, + "b": 7.872, + "c": 7.872, + "d": 7.872, + "e": 7.872, + "f": 7.872, + "g": 7.872, + "h": 7.872, + "i": 7.872, + "j": 7.872, + "k": 7.872, + "l": 7.872, + "m": 7.872, + "n": 7.872, + "o": 7.872, + "p": 7.872, + "q": 7.872, + "r": 7.872, + "s": 7.872, + "t": 7.872, + "u": 7.872, + "v": 7.872, + "w": 7.872, + "x": 7.872, + "y": 7.872, + "z": 7.872, + "{": 7.872, + "|": 7.872, + "}": 7.872, + "~": 7.872, + "→": 7.899, + "—": 7.872, + "‘": 7.872, + "’": 7.872, + "“": 7.872, + "”": 7.872, + " ": 7.872, + "≤": 7.899, + "≥": 7.899 + } +} diff --git a/fluxer_docs/scripts/DocsTableWidth.ts b/fluxer_docs/scripts/DocsTableWidth.ts new file mode 100644 index 000000000..4ba8beb7e --- /dev/null +++ b/fluxer_docs/scripts/DocsTableWidth.ts @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + columnWidthPercents, + maxContentPx, + minContentPx, + type Piece, + PX, + TABLE_BUDGET_PX, + TABLE_FLOOR_CAP_SHARE, + TABLE_MAX_CELL_CHARS, + TABLE_MAX_COLUMNS, + TABLE_MAX_IDENT_CHARS, + TABLE_WIDE_TIER_PX, +} from '../src/table/DocsTableMetrics.ts'; + +export { + columnWidthPercents, + PX, + TABLE_BUDGET_PX, + TABLE_FLOOR_CAP_SHARE, + TABLE_MAX_CELL_CHARS, + TABLE_MAX_COLUMNS, + TABLE_MAX_IDENT_CHARS, + TABLE_WIDE_TIER_PX, +}; + +const CANONICAL_SHAPES = new Set([ + 'field|type|description', + 'status|body|condition', + 'value|name|description', + 'value|description', +]); + +const PARALLEL_SENTINELS = new Set(['unset', 'empty', 'none', 'required', 'n/a', '-', 'same target', 'no default']); + +export interface DocsTable { + readonly line: number; + readonly endLine: number; + readonly columns: number; + readonly rows: number; + readonly header: ReadonlyArray; + readonly canonical: boolean; + readonly minDemandPx: number; + readonly columnMinPx: ReadonlyArray; + readonly columnMaxPx: ReadonlyArray; + readonly worstCellChars: number; + readonly worstCellColumn: number; + readonly worstCellText: string; + readonly col1IdentChars: number; + readonly col1IdentText: string; + readonly parallel: boolean; + readonly nonParallelReason: string; +} + +function splitRow(line: string): Array { + const cells: Array = []; + let current = ''; + const body = line.trim().replace(/^\|/u, '').replace(/\|$/u, ''); + for (let index = 0; index < body.length; index += 1) { + const character = body[index]; + if (character === '\\' && index + 1 < body.length) { + current += body[index + 1]; + index += 1; + continue; + } + if (character === '|') { + cells.push(current); + current = ''; + continue; + } + current += character; + } + cells.push(current); + return cells.map((cell) => cell.trim()); +} + +function isDelimiter(line: string): boolean { + const trimmed = line.trim(); + if (!trimmed.startsWith('|')) { + return false; + } + return /^\|[\s:|-]+\|?$/u.test(trimmed) && trimmed.includes('-'); +} + +function stripInline(text: string): string { + return text + .replace(/!\[([^\]]*)\]\([^)]*\)/gu, '$1') + .replace(/\[([^\]]*)\]\([^)]*\)/gu, '$1') + .replace(/([^<]*)<\/sup>/gu, '$1') + .replace(//giu, ' ') + .replace(/<[^>]+>/gu, '') + .replace(/\*\*([^*]*)\*\*/gu, '$1') + .replace(/\*([^*]*)\*/gu, '$1') + .replace(/ /gu, ' ') + .replace(/</gu, '<') + .replace(/>/gu, '>') + .replace(/&/gu, '&'); +} + +function pieces(cell: string): Array { + const out: Array = []; + const pattern = /`([^`]*)`/gu; + let last = 0; + let match: RegExpExecArray | null = pattern.exec(cell); + while (match != null) { + if (match.index > last) { + out.push({text: stripInline(cell.slice(last, match.index)), code: false}); + } + out.push({text: match[1], code: true}); + last = pattern.lastIndex; + match = pattern.exec(cell); + } + if (last < cell.length) { + out.push({text: stripInline(cell.slice(last)), code: false}); + } + return out.filter((piece) => piece.text.length > 0); +} + +function renderedText(parts: ReadonlyArray): string { + return parts.map((piece) => piece.text).join(''); +} + +function cellKind(cell: string): string { + const stripped = cell.replace(/[^<]*<\/sup>/gu, ''); + const parts = pieces(stripped); + if (parts.length === 0) { + return 'empty'; + } + if (parts.every((piece) => piece.code)) { + return 'code'; + } + if (parts.every((piece) => !piece.code)) { + return /[.!?]\s|[.!?]$/u.test(stripped) ? 'sentence' : 'phrase'; + } + return 'mixed'; +} + +export function extractTables(source: string): Array { + const lines = source.split('\n'); + const tables: Array = []; + let fenced = false; + let index = 0; + while (index < lines.length) { + if (/^\s*(?:```|~~~)/u.test(lines[index])) { + fenced = !fenced; + index += 1; + continue; + } + if (fenced) { + index += 1; + continue; + } + if (!lines[index].trim().startsWith('|') || index + 1 >= lines.length || !isDelimiter(lines[index + 1])) { + index += 1; + continue; + } + const header = splitRow(lines[index]); + const start = index; + index += 2; + const body: Array> = []; + while (index < lines.length && lines[index].trim().startsWith('|')) { + body.push(splitRow(lines[index])); + index += 1; + } + const columns = header.length; + const columnMinPx = new Array(columns).fill(0); + const columnMaxPx = new Array(columns).fill(0); + let worstCellChars = 0; + let worstCellColumn = 0; + let worstCellText = ''; + let col1IdentChars = 0; + let col1IdentText = ''; + for (const [rowIndex, row] of [header, ...body].entries()) { + const isHeader = rowIndex === 0; + for (let column = 0; column < columns; column += 1) { + const parts = pieces(isHeader ? (row[column] ?? '').toUpperCase() : (row[column] ?? '')); + const minimum = minContentPx(parts); + const maximum = maxContentPx(parts, isHeader); + if (maximum > columnMaxPx[column]) { + columnMaxPx[column] = maximum; + } + if (minimum > columnMinPx[column]) { + columnMinPx[column] = minimum; + } + const text = renderedText(parts); + if (!isHeader && text.length > worstCellChars) { + worstCellChars = text.length; + worstCellColumn = column; + worstCellText = text; + } + if (column === 0 && !isHeader) { + for (const run of text.split(/(?<=\s)|(?<=-)(?!\d)/u)) { + const token = run.trim(); + if (token.length > col1IdentChars) { + col1IdentChars = token.length; + col1IdentText = token; + } + } + } + } + } + const headerShape = header.map((cell) => cell.trim().toLowerCase()).join('|'); + let parallel = true; + let nonParallelReason = ''; + for (let column = 0; column < columns && parallel; column += 1) { + if (headerShape === 'status|body|condition' && column === 1) { + continue; + } + const shape = body + .map((row) => (row[column] ?? '').trim()) + .filter((cell) => cell.length > 0 && !PARALLEL_SENTINELS.has(cell.toLowerCase())) + .map(cellKind); + const kinds = new Set(shape); + kinds.delete('empty'); + const prose = kinds.has('sentence') || kinds.has('phrase') || kinds.has('mixed'); + if (kinds.has('code') && prose && shape.length > 2) { + parallel = false; + nonParallelReason = `column ${(column + 1).toString()} mixes code-only cells with prose cells`; + } + } + tables.push({ + line: start + 1, + endLine: index, + columns, + rows: body.length, + header, + canonical: CANONICAL_SHAPES.has(headerShape), + minDemandPx: Math.round(columnMinPx.reduce((a, b) => a + b, 0) + columns * PX.cellPad), + columnMinPx: columnMinPx.map((value) => Math.round(value)), + columnMaxPx: columnMaxPx.map((value) => Math.round(value)), + worstCellChars, + worstCellColumn, + worstCellText, + col1IdentChars, + col1IdentText, + parallel, + nonParallelReason, + }); + } + return tables; +} diff --git a/fluxer_docs/scripts/Healthcheck.ts b/fluxer_docs/scripts/Healthcheck.ts new file mode 100644 index 000000000..116c0e8da --- /dev/null +++ b/fluxer_docs/scripts/Healthcheck.ts @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {checkAstroServerHealth} from '../src/server/AstroServer'; +import {docsListenHost, docsListenPort} from '../src/server/DocsConfig'; + +await checkAstroServerHealth({ + listenHost: docsListenHost(), + listenPort: docsListenPort(), + timeoutMs: 4_000, +}); diff --git a/fluxer_docs/scripts/StartServer.ts b/fluxer_docs/scripts/StartServer.ts new file mode 100644 index 000000000..311a58fd2 --- /dev/null +++ b/fluxer_docs/scripts/StartServer.ts @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {runAstroServer} from '../src/server/AstroServer'; +import {docsListenHost, docsListenPort, docsPublicEndpoint} from '../src/server/DocsConfig'; + +await runAstroServer({ + entrypoint: new URL('../dist/server/entry.mjs', import.meta.url), + initialize: null, + listenHost: docsListenHost(), + listenPort: docsListenPort(), + publicEndpoint: docsPublicEndpoint(), + readiness: null, + requestLocals: null, + shutdown: null, +}); diff --git a/fluxer_docs/scripts/VerifyDocsCoverage.ts b/fluxer_docs/scripts/VerifyDocsCoverage.ts new file mode 100644 index 000000000..59cf85201 --- /dev/null +++ b/fluxer_docs/scripts/VerifyDocsCoverage.ts @@ -0,0 +1,2205 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {spawnSync} from 'node:child_process'; +import {createHash} from 'node:crypto'; +import type {Dirent} from 'node:fs'; +import {mkdir, mkdtemp, readdir, readFile, rm, writeFile} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {extractRoutesFromControllers} from '@fluxer/openapi/src/extractors/RouteExtractor'; +import {installerChecksumLine} from '../src/installer/InstallerDigest.ts'; + +const DOCS_ROOT = fileURLToPath(new URL('../src/content/docs/', import.meta.url)); +const REPO_ROOT = fileURLToPath(new URL('../../', import.meta.url)); +const MAIN_SPEC = path.join(REPO_ROOT, 'fluxer_api/src/api/openapi/openapi.json'); +const ADMIN_SPEC = path.join(REPO_ROOT, 'fluxer_admin/openapi-admin.json'); +const MEDIA_PROXY_SERVER_DIR = path.join(REPO_ROOT, 'fluxer_media_proxy/src/server'); + +const ROUTE_HEADER_PATTERN = /]*?)\/>/gu; +const ATTRIBUTE_PATTERN = /(\w+)\s*=\s*"([^"]*)"/gu; +const HTTP_METHODS = new Set(['GET', 'HEAD', 'POST', 'PATCH', 'PUT', 'DELETE', 'OPTIONS']); + +const BLUESKY_OAUTH_CONTROLLER = 'fluxer_api/src/api/bluesky/BlueskyOAuthController.ts'; +const DOWNLOAD_CONTROLLER = 'fluxer_api/src/api/download/DownloadController.ts'; + +const MAIN_SPEC_EXEMPT = new Map([ + [ + 'GET /oauth2/authorize', + { + file: 'fluxer_api/src/api/oauth/OAuth2Controller.ts', + anchor: "'/oauth2/authorize',", + reason: 'the generator excludes the path outright, at OpenAPIGeneratorCatalog excluded.paths', + }, + ], + [ + 'GET /openapi.json', + { + file: 'fluxer_api/src/api/openapi/OpenAPIController.ts', + anchor: "app.get('/openapi.json'", + reason: 'the handler serves the spec file itself and carries no OpenAPI({...}) block', + }, + ], + [ + 'GET /connections/bluesky/client-metadata.json', + { + file: BLUESKY_OAUTH_CONTROLLER, + anchor: "'/connections/bluesky/client-metadata.json'", + reason: 'an AT Protocol client document with no OpenAPI({...}) block', + }, + ], + [ + 'GET /connections/bluesky/jwks.json', + { + file: BLUESKY_OAUTH_CONTROLLER, + anchor: "'/connections/bluesky/jwks.json'", + reason: 'an AT Protocol client document with no OpenAPI({...}) block', + }, + ], + + [ + 'GET /dl/desktop/{}/{}/{}/latest/{}.sha256', + { + file: DOWNLOAD_CONTROLLER, + anchor: '`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/latest/:format{[a-z_]+\\\\.sha256}`,', + reason: 'the path constrains :format by regex and has no OpenAPI path template', + }, + ], + [ + 'GET /dl/desktop/{}/{}/{}/{}/{}.sha256', + { + file: DOWNLOAD_CONTROLLER, + anchor: '`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/:version/:format{[a-z_]+\\\\.sha256}`,', + reason: 'the path constrains :format by regex and has no OpenAPI path template', + }, + ], + [ + 'GET /dl/{}', + { + file: DOWNLOAD_CONTROLLER, + anchor: '`${DOWNLOAD_PREFIX}/*`,', + reason: 'a Hono wildcard, which is not an OpenAPI path template', + }, + ], +]); + +const DELIBERATELY_UNDOCUMENTED = new Map([ + ['GET /users/@me/mobile-devices', 'mobile notifications, backported separately'], + ['POST /users/@me/mobile-devices', 'mobile notifications, backported separately'], + ['POST /users/@me/mobile-devices/unregister', 'mobile notifications, backported separately'], + ['DELETE /users/@me/mobile-devices/{}', 'mobile notifications, backported separately'], + ['GET /users/@me/push/subscriptions', 'push API, backported separately'], + ['POST /users/@me/push/subscribe', 'push API, backported separately'], + ['POST /users/@me/push/rotate', 'push API, backported separately'], + ['DELETE /users/@me/push/subscriptions/{}', 'push API, backported separately'], +]); + +interface OutOfBandRoute { + readonly reason: string; + readonly documentedIn: {readonly file: string; readonly anchor: string} | null; +} +const OUT_OF_BAND_CREDENTIAL = new Map([ + [ + 'POST /internal/rpc', + { + reason: + 'x-fluxer-rpc-auth must timing-safe-equal Config.internal.gatewayRpcAuthToken, and an empty token rejects every caller. The caller is fluxer_gateway', + documentedIn: null, + }, + ], + [ + 'POST /webhooks/twilio/sms', + { + reason: + 'installed only when config.sms.enabled, and then only when the inbound webhook token and public URL are both set, so a default or self-hosted instance never registers it', + documentedIn: null, + }, + ], + [ + 'POST /webhooks/livekit', + { + reason: 'a LiveKit-signed callback whose shipped target is the internal address http://api:8080/webhooks/livekit', + documentedIn: {file: 'operator/configuration.mdx', anchor: 'http://api:8080/webhooks/livekit'}, + }, + ], + [ + 'POST /webhooks/sweego', + { + reason: + 'a Standard Webhooks delivery signed with FLUXER_EMAIL_WEBHOOK_SECRET, which answers 404 Email not enabled while email is off', + documentedIn: {file: 'operator/configuration.mdx', anchor: 'POST /webhooks/sweego'}, + }, + ], + [ + 'GET /connections/bluesky/callback', + { + reason: + 'the atproto redirect URI, reached with a provider-issued authorisation code. A client MUST NOT call it directly', + documentedIn: {file: 'http-api/connections.mdx', anchor: '/connections/bluesky/callback'}, + }, + ], +]); + +const HEALTH_AND_METRICS_PATHS = new Set(['/_health', '/_health/ready', '/_health/drain', '/_healthz', '/_metrics']); + +interface ExemptionRule { + readonly name: string; + readonly justification: string; + + readonly anchors: ReadonlyArray<{readonly file: string; readonly anchor: string}>; + readonly covers: (shape: string, routePath: string) => boolean; +} +const EXEMPTION_RULES: ReadonlyArray = [ + { + name: 'test harness', + justification: + 'ControllerRegistry registers TestHarnessController only when config.dev.testModeEnabled or NODE_ENV is development, and ensureHarnessAccess rechecks that per request. No production instance serves a /test/ route', + anchors: [ + { + file: 'fluxer_api/src/api/app/ControllerRegistry.ts', + anchor: "if (config.dev.testModeEnabled || config.nodeEnv === 'development') {", + }, + {file: 'fluxer_api/src/api/test/TestHarnessController.ts', anchor: 'function ensureHarnessAccess'}, + ], + covers: (_shape, routePath) => routePath.startsWith('/test/'), + }, + { + name: 'backported separately', + justification: + 'part of a system this backport does not ship. Each shape is listed with its system, and a page that documents one fails the run below', + anchors: [], + covers: (shape) => DELIBERATELY_UNDOCUMENTED.has(shape), + }, + { + name: 'process health and metrics', + justification: + 'loopback gated or reachable only inside the compose network. operator/reverse-proxy.mdx states which probes answer through a proxy once for every service, rather than per route', + anchors: [{file: 'fluxer_docs/src/content/docs/operator/reverse-proxy.mdx', anchor: '/_metrics'}], + covers: (_shape, routePath) => HEALTH_AND_METRICS_PATHS.has(routePath), + }, + { + name: 'out-of-band credential', + justification: + 'no ordinary client holds the credential. Each entry states its guard, and three are covered in prose', + anchors: [{file: 'fluxer_api/src/api/app/ControllerRegistry.ts', anchor: 'if (config.sms.enabled) {'}], + covers: (shape) => OUT_OF_BAND_CREDENTIAL.has(shape), + }, +]; + +const MEDIA_PROXY_ROUTES = new Map([ + ['GET /_health', '.route("/_health", get(routes::ops::health))'], + ['HEAD /_health', '.route("/_health", get(routes::ops::health))'], + ['GET /_metrics', '.route("/_metrics", get(routes::ops::metrics_handler))'], + ['POST /_metadata', '.route("/_metadata", post(routes::internal::metadata_handler))'], + ['POST /_thumbnail', '.route("/_thumbnail", post(routes::internal::thumbnail_handler))'], + ['POST /_frames', '.route("/_frames", post(routes::internal::frames_handler))'], + [ + 'PUT /v1/relay/{}', + '"/v1/relay/{*key}",\n put(routes::relay::relay_put).options(routes::relay::relay_options),', + ], + [ + 'OPTIONS /v1/relay/{}', + '"/v1/relay/{*key}",\n put(routes::relay::relay_put).options(routes::relay::relay_options),', + ], + ['GET /{}', 'if app.cfg.mode == DeploymentMode::Static {'], + ['HEAD /{}', 'if app.cfg.mode == DeploymentMode::Static {'], +]); + +const MEDIA_PROXY_ASSET_PREFIXES = new Map([ + ['external', 'path.strip_prefix("/external/")'], + ['attachments', 'path.starts_with("/attachments/")'], + ['themes', 'path.starts_with("/themes/")'], + ['entrance-sounds', '!= "entrance-sounds"'], + ['emojis', 'AssetKind::Emoji => "emojis"'], + ['stickers', 'AssetKind::Sticker => "stickers"'], + ['avatars', '"avatars" => AssetKind::Avatar'], + ['icons', '"icons" => AssetKind::GuildIcon'], + ['branding', '"branding" => AssetKind::GuildIcon'], + ['banners', '"banners" => AssetKind::Banner'], + ['splashes', '"splashes" => AssetKind::Splash'], + ['embed-splashes', '"embed-splashes" => AssetKind::EmbedSplash'], + ['guilds', 'fn parse_guild_member_asset_path'], +]); + +interface DocumentedRoute { + readonly method: string; + readonly path: string; + readonly file: string; + readonly bot: boolean; + readonly unauthenticated: boolean; + readonly oauth2: string | null; +} + +interface SpecOperation { + readonly method: string; + readonly path: string; + readonly tags: ReadonlyArray; + readonly security: ReadonlyArray>> | null; +} + +async function walk(directory: string): Promise> { + const entries = await readdir(directory, {withFileTypes: true}); + const files: Array = []; + for (const entry of entries) { + const resolved = path.join(directory, entry.name); + if (entry.isDirectory()) { + files.push(...(await walk(resolved))); + continue; + } + if (entry.name.endsWith('.mdx') || entry.name.endsWith('.md')) { + files.push(resolved); + } + } + return files; +} + +function stripVersionPrefix(routePath: string): string { + if (routePath === '/v1') { + return '/'; + } + if (routePath.startsWith('/v1/')) { + return routePath.slice(3); + } + return routePath; +} + +function shapeOf(method: string, routePath: string): string { + const withoutQuery = routePath.split('?')[0]; + return `${method} ${withoutQuery.replace(/\{[^}]*\}/gu, '{}')}`; +} + +async function documentedRoutes(): Promise> { + const files = await walk(DOCS_ROOT); + const routes: Array = []; + for (const file of files) { + const source = await readFile(file, 'utf8'); + for (const match of source.matchAll(ROUTE_HEADER_PATTERN)) { + const attributes = new Map(); + for (const attribute of match[1].matchAll(ATTRIBUTE_PATTERN)) { + attributes.set(attribute[1], attribute[2]); + } + const method = attributes.get('method'); + const routePath = attributes.get('path'); + if (method == null || routePath == null) { + continue; + } + const bareFlags = new Set( + match[1] + .replace(/\w+\s*=\s*"[^"]*"/gu, ' ') + .split(/\s+/u) + .filter((token) => token.length > 0), + ); + routes.push({ + method, + path: routePath, + file: path.relative(DOCS_ROOT, file), + bot: bareFlags.has('bot'), + unauthenticated: bareFlags.has('unauthenticated'), + oauth2: attributes.get('oauth2') ?? null, + }); + } + } + return routes; +} + +interface AliasRoute { + readonly shape: string; + readonly file: string; + readonly successor: string; +} + +const ALIAS_TABLE_HEADER = '| Method | Deprecated path | Successor |'; +const ALIAS_ROW_PATTERN = + /^\|\s*(GET|HEAD|POST|PATCH|PUT|DELETE|OPTIONS)\s*\|\s*`(\/[^`]+)`\s*\|\s*\[([^\]]+)\]\([^)]+\)\s*\|\s*$/u; + +async function aliasDocumentedRoutes(): Promise> { + const files = await walk(DOCS_ROOT); + const aliases: Array = []; + for (const file of files) { + const source = await readFile(file, 'utf8'); + let inTable = false; + for (const line of source.split('\n')) { + if (line.trim() === ALIAS_TABLE_HEADER) { + inTable = true; + continue; + } + if (inTable && line.trim().startsWith('| ---')) { + continue; + } + const row = inTable ? ALIAS_ROW_PATTERN.exec(line.trim()) : null; + if (row == null) { + inTable = false; + continue; + } + aliases.push({ + shape: shapeOf(row[1], stripVersionPrefix(row[2])), + file: path.relative(DOCS_ROOT, file), + successor: row[3], + }); + } + } + return aliases; +} + +async function specOperations(specPath: string): Promise> { + const spec: unknown = JSON.parse(await readFile(specPath, 'utf8')); + if (typeof spec !== 'object' || spec == null || !('paths' in spec)) { + throw new Error(`Spec has no paths: ${specPath}`); + } + const paths = ( + spec as { + paths: Record; security?: Array>>}>>; + } + ).paths; + const operations: Array = []; + for (const [routePath, item] of Object.entries(paths)) { + for (const [method, operation] of Object.entries(item)) { + const upper = method.toUpperCase(); + if (!HTTP_METHODS.has(upper)) { + continue; + } + operations.push({ + method: upper, + path: stripVersionPrefix(routePath), + tags: operation.tags ?? [], + security: operation.security ?? null, + }); + } + } + return operations; +} + +function section(title: string, entries: ReadonlyArray): number { + if (entries.length === 0) { + console.log(` ${title}: none`); + return 0; + } + console.log(` ${title}: ${entries.length.toString()}`); + for (const entry of entries) { + console.log(` ${entry}`); + } + return entries.length; +} + +async function registrationSources(directory: string): Promise> { + const entries = await readdir(directory, {withFileTypes: true}); + const files: Array = []; + for (const entry of entries) { + const resolved = path.join(directory, entry.name); + if (entry.isDirectory()) { + if (entry.name === 'node_modules' || entry.name === 'tests') { + continue; + } + files.push(...(await registrationSources(resolved))); + continue; + } + if (entry.name.endsWith('.ts') && !entry.name.endsWith('.test.ts')) { + files.push(resolved); + } + } + return files; +} +const controllerRoutes = extractRoutesFromControllers( + await registrationSources(path.join(REPO_ROOT, 'fluxer_api/src')), +); + +function constraintTail(constraint: string): string { + const body = constraint.slice(1, -1); + const tail = body.match(/(?:\\[^A-Za-z0-9]|[A-Za-z0-9_])+$/u); + if (tail?.index == null || !/[+*?})\]]$/u.test(body.slice(0, tail.index))) { + return ``; + } + return tail[0].replace(/\\(.)/gu, '$1'); +} +const astRoute = (route: {method: string; path: string}): string => { + const templated = route.path + .replace( + /:([A-Za-z_][A-Za-z0-9_]*)(\{(?:[^{}]|\{[^{}]*\})*\})?/gu, + (_match: string, name: string, constraint: string | undefined) => + `{${name}}${constraint == null ? '' : constraintTail(constraint)}`, + ) + .replace(/\*/gu, '{wildcard}'); + return shapeOf(route.method.toUpperCase(), stripVersionPrefix(templated)); +}; + +const documented = await documentedRoutes(); +const aliasDocumented = await aliasDocumentedRoutes(); +const aliasShapes = new Map(aliasDocumented.map((alias) => [alias.shape, alias])); +const documentedFlags = new Map(); +for (const route of documented) { + documentedFlags.set(shapeOf(route.method, stripVersionPrefix(route.path)), { + bot: route.bot, + unauthenticated: route.unauthenticated, + }); +} +const mediaProxySource = await (async () => { + const sources: Array = []; + const walk = async (dir: string): Promise => { + for (const entry of await readdir(dir, {withFileTypes: true})) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + await walk(full); + } else if (entry.name.endsWith('.rs')) { + sources.push(await readFile(full, 'utf8')); + } + } + }; + await walk(MEDIA_PROXY_SERVER_DIR); + return sources.join('\n'); +})(); + +let failures = 0; + +console.log('source anchors'); +const staleAnchors: Array = []; +for (const [route, anchor] of [...MEDIA_PROXY_ROUTES, ...MEDIA_PROXY_ASSET_PREFIXES]) { + if (!mediaProxySource.includes(anchor)) { + staleAnchors.push(`${route}: fluxer_media_proxy/src/server no longer contains ${anchor}`); + } +} +const exemptSources = new Map(); +const sourceOf = async (file: string): Promise => { + let source = exemptSources.get(file); + if (source == null) { + source = await readFile(path.join(REPO_ROOT, file), 'utf8'); + exemptSources.set(file, source); + } + return source; +}; +for (const [route, {file, anchor}] of MAIN_SPEC_EXEMPT) { + if (!(await sourceOf(file)).includes(anchor)) { + staleAnchors.push(`${route}: ${file} no longer contains ${anchor}`); + } +} +for (const rule of EXEMPTION_RULES) { + for (const {file, anchor} of rule.anchors) { + if (!(await sourceOf(file)).includes(anchor)) { + staleAnchors.push(`exemption rule "${rule.name}": ${file} no longer contains ${anchor}`); + } + } +} +for (const [route, {documentedIn}] of OUT_OF_BAND_CREDENTIAL) { + if (documentedIn == null) { + continue; + } + const page = await readFile(path.join(DOCS_ROOT, documentedIn.file), 'utf8'); + if (!page.includes(documentedIn.anchor)) { + staleAnchors.push(`${route}: ${documentedIn.file} no longer contains ${documentedIn.anchor}`); + } +} +failures += section('stale anchors (the code moved, update this script)', staleAnchors); + +const mediaProxyDocumented = documented.filter((route) => route.file.startsWith('media-proxy/')); +const adminDocumented = documented.filter( + (route) => !route.file.startsWith('media-proxy/') && stripVersionPrefix(route.path).startsWith('/admin'), +); +const mainDocumented = documented.filter( + (route) => !route.file.startsWith('media-proxy/') && !stripVersionPrefix(route.path).startsWith('/admin'), +); + +const main = await specOperations(MAIN_SPEC); +const admin = await specOperations(ADMIN_SPEC); +const mainShapes = new Set(main.map((operation) => shapeOf(operation.method, operation.path))); +const adminShapes = new Set(admin.map((operation) => shapeOf(operation.method, operation.path))); + +const documentedMain = new Map(); +for (const route of mainDocumented) { + documentedMain.set(shapeOf(route.method, stripVersionPrefix(route.path)), route); +} +const documentedAdmin = new Map(); +for (const route of adminDocumented) { + documentedAdmin.set(shapeOf(route.method, stripVersionPrefix(route.path)), route); +} + +const registered = new Map(); +for (const route of controllerRoutes) { + const shape = astRoute(route); + if (!registered.has(shape)) { + registered.set(shape, `${path.relative(REPO_ROOT, route.controllerFile)}:${route.lineNumber.toString()}`); + } +} + +console.log('main API and admin API (from route registration)'); +{ + const ruleCounts = new Map(); + for (const rule of EXEMPTION_RULES) { + ruleCounts.set(rule.name, 0); + } + const undocumented: Array = []; + for (const [shape, site] of [...registered].sort()) { + if (documentedMain.has(shape) || documentedAdmin.has(shape) || aliasShapes.has(shape)) { + continue; + } + const rule = EXEMPTION_RULES.find((candidate) => candidate.covers(shape, shape.split(' ')[1])); + if (rule == null) { + undocumented.push(`${shape} registered at ${site}`); + continue; + } + ruleCounts.set(rule.name, (ruleCounts.get(rule.name) ?? 0) + 1); + } + + const documentedNotRegistered = [...documentedMain, ...documentedAdmin] + .filter(([shape]) => !registered.has(shape)) + .map(([shape, route]) => `${shape} (${route.file}) is documented but fluxer_api registers no such route`) + .sort(); + const deadRules = EXEMPTION_RULES.filter((rule) => (ruleCounts.get(rule.name) ?? 0) === 0).map( + (rule) => `the "${rule.name}" rule covers no registered route, so it is either stale or too narrow to matter`, + ); + const staleEntries = [ + ...[...DELIBERATELY_UNDOCUMENTED.keys()].map((shape) => [shape, 'DELIBERATELY_UNDOCUMENTED'] as const), + ...[...OUT_OF_BAND_CREDENTIAL.keys()].map((shape) => [shape, 'OUT_OF_BAND_CREDENTIAL'] as const), + ] + .filter(([shape]) => !registered.has(shape)) + .map(([shape, list]) => `${list} names ${shape}, which fluxer_api no longer registers`) + .sort(); + + const exemptTotal = [...ruleCounts.values()].reduce((a, b) => a + b, 0); + console.log(` registered routes: ${registered.size.toString()}`); + console.log( + ` documented with a RouteHeader: ${[...registered.keys()].filter((shape) => documentedMain.has(shape) || documentedAdmin.has(shape)).length.toString()}`, + ); + console.log( + ` documented as a deprecated alias: ${[...registered.keys()].filter((shape) => aliasShapes.has(shape)).length.toString()}`, + ); + console.log(` exempt by rule: ${exemptTotal.toString()}`); + for (const rule of EXEMPTION_RULES) { + console.log(` ${rule.name}: ${(ruleCounts.get(rule.name) ?? 0).toString()}. ${rule.justification}`); + } + console.log(' fluxer_admin is a Rust crate and its 83 HTML panel routes are NOT checked here. It serves'); + console.log(' zero API operations, so the admin API above is the /admin routes fluxer_api registers.'); + console.log(' fluxer_media_proxy and fluxer_app_proxy are Rust too. The media proxy is checked against'); + console.log(' the hand-maintained list below, and the app proxy is not checked at all.'); + failures += section('registered but neither documented nor covered by an exemption rule', undocumented); + failures += section('documented but not registered', documentedNotRegistered); + failures += section('exemption rules that cover nothing', deadRules); + failures += section('exemption entries naming a route that is gone', staleEntries); +} + +console.log('main API (against the generated spec)'); +failures += section( + 'documented but absent from the live spec', + [...documentedMain.entries()] + .filter(([shape]) => !mainShapes.has(shape) && !MAIN_SPEC_EXEMPT.has(shape)) + .map(([, route]) => `${route.method} ${route.path} (${route.file})`) + .sort(), +); +failures += section( + 'present in the live spec but undocumented', + main + .filter((operation) => { + const shape = shapeOf(operation.method, operation.path); + if (documentedMain.has(shape)) { + return false; + } + if (aliasShapes.has(shape)) { + return false; + } + return !DELIBERATELY_UNDOCUMENTED.has(shape); + }) + .map((operation) => `${operation.method} ${operation.path} [${operation.tags.join(', ')}]`) + .sort(), +); +failures += section( + 'documented as a deprecated alias but not registered', + [...aliasShapes.values()] + .filter((alias) => !registered.has(alias.shape)) + .map((alias) => `${alias.shape} (${alias.file}) names a route fluxer_api does not register`) + .sort(), +); +failures += section( + 'documented as a deprecated alias and with its own RouteHeader', + [...aliasShapes.keys()] + .filter((shape) => documentedMain.has(shape)) + .map((shape) => `${shape} is an alias row and a RouteHeader, which double counts it`) + .sort(), +); +console.log(` documented as a deprecated alias of a documented route: ${aliasShapes.size.toString()}`); +const wronglyDocumented = [...DELIBERATELY_UNDOCUMENTED.entries()] + .filter(([shape]) => documentedMain.has(shape)) + .map(([shape, reason]) => `${shape} must not be documented: ${reason}`); +failures += section('documented despite belonging to an unshipped system', wronglyDocumented); +console.log( + ` deliberately undocumented and correctly absent: ${(DELIBERATELY_UNDOCUMENTED.size - wronglyDocumented.length).toString()}/${DELIBERATELY_UNDOCUMENTED.size.toString()}`, +); +console.log( + ` exempt non-spec routes documented: ${[...MAIN_SPEC_EXEMPT.keys()].filter((shape) => documentedMain.has(shape)).length.toString()}/${MAIN_SPEC_EXEMPT.size.toString()}`, +); +for (const [shape, {reason}] of MAIN_SPEC_EXEMPT) { + console.log(` ${shape}: ${reason}`); +} + +console.log('media proxy'); +const mediaProxyProblems: Array = []; +for (const route of mediaProxyDocumented) { + const shape = shapeOf(route.method, route.path); + if (MEDIA_PROXY_ROUTES.has(shape)) { + continue; + } + const firstSegment = route.path.replace(/^\//u, '').split('/')[0].split('.')[0]; + if (MEDIA_PROXY_ASSET_PREFIXES.has(firstSegment)) { + if (route.method !== 'GET' && route.method !== 'HEAD') { + mediaProxyProblems.push(`${route.method} ${route.path} (${route.file}) asset paths serve GET and HEAD only`); + } + continue; + } + mediaProxyProblems.push(`${route.method} ${route.path} (${route.file}) is not a route the media proxy serves`); +} +failures += section('documented but not served by fluxer_media_proxy', mediaProxyProblems); + +console.log('gateway dispatch events'); +{ + const gatewayConstants = await readFile(path.join(REPO_ROOT, 'fluxer_api/src/api/constants/Gateway.ts'), 'utf8'); + const unionMatch = gatewayConstants.match(/export type GatewayDispatchEvent\s*=([\s\S]*?);/u); + const apiEvents = new Set(); + if (unionMatch != null) { + for (const entry of unionMatch[1].matchAll(/'([A-Z][A-Z0-9_]*)'/gu)) { + apiEvents.add(entry[1]); + } + } + const gatewayDirectory = path.join(REPO_ROOT, 'fluxer_gateway/src'); + const erlangSources: Array = []; + const collect = async (directory: string): Promise => { + for (const entry of await readdir(directory, {withFileTypes: true})) { + const resolved = path.join(directory, entry.name); + if (entry.isDirectory()) { + await collect(resolved); + continue; + } + if (entry.name.endsWith('.erl') || entry.name.endsWith('.hrl')) { + erlangSources.push(await readFile(resolved, 'utf8')); + } + } + }; + await collect(gatewayDirectory); + const erlang = erlangSources.join('\n'); + + const eventsPage = await readFile(path.join(DOCS_ROOT, 'gateway/events.md'), 'utf8'); + const documentedEvents = new Set(); + for (const entry of eventsPage.matchAll(/^###\s+(?:]*><\/span>)?([A-Z][A-Z0-9_]{3,})\s*$/gmu)) { + documentedEvents.add(entry[1]); + } + + const isReal = (event: string): boolean => { + if (apiEvents.has(event)) { + return true; + } + if (erlang.includes(event)) { + return true; + } + return new RegExp(`\\b${event.toLowerCase()}\\b`, 'u').test(erlang); + }; + + const fabricated = [...documentedEvents].filter((event) => !isReal(event)).sort(); + const undocumented = [...apiEvents].filter((event) => !documentedEvents.has(event)).sort(); + console.log(` events in the GatewayDispatchEvent union: ${apiEvents.size.toString()}`); + console.log(` events documented in gateway/events.md: ${documentedEvents.size.toString()}`); + console.log(' a documented event counts as real if it is in the union, or appears in fluxer_gateway'); + console.log(' as an uppercase binary or a lowercase atom'); + failures += section('documented but not emitted by any service', fabricated); + failures += section('in the dispatch union but undocumented', undocumented); +} + +console.log('gateway opcodes and close codes'); +{ + const erl = await readFile(path.join(REPO_ROOT, 'fluxer_gateway/src/utils/constants.erl'), 'utf8'); + const realOpcodes = new Set(); + for (const entry of erl.matchAll(/gateway_opcode\((\d+)\) -> [a-z_]+/gu)) { + realOpcodes.add(Number.parseInt(entry[1], 10)); + } + const realCloseCodes = new Set(); + for (const entry of erl.matchAll(/close_code_to_num\([a-z_]+\) -> (\d+)/gu)) { + realCloseCodes.add(Number.parseInt(entry[1], 10)); + } + const page = await readFile(path.join(DOCS_ROOT, 'gateway/opcodes-and-close-codes.md'), 'utf8'); + const docOpcodes = new Set(); + const docCloseCodes = new Set(); + const strippedPage = page.replace(/.*?<\/sup>/gu, ''); + for (const row of strippedPage.matchAll(/^\|\s*(\d{1,4})\s*\|/gmu)) { + const value = Number.parseInt(row[1], 10); + if (value >= 4000) { + docCloseCodes.add(value); + continue; + } + if (value <= 99) { + docOpcodes.add(value); + } + } + console.log(` opcodes: ${realOpcodes.size.toString()} real, ${docOpcodes.size.toString()} documented`); + console.log(` close codes: ${realCloseCodes.size.toString()} real, ${docCloseCodes.size.toString()} documented`); + failures += section( + 'opcodes documented but absent from constants.erl', + [...docOpcodes] + .filter((code) => !realOpcodes.has(code)) + .sort((a, b) => a - b) + .map(String), + ); + failures += section( + 'opcodes in constants.erl but undocumented', + [...realOpcodes] + .filter((code) => !docOpcodes.has(code)) + .sort((a, b) => a - b) + .map(String), + ); + failures += section( + 'close codes documented but absent from constants.erl', + [...docCloseCodes] + .filter((code) => !realCloseCodes.has(code)) + .sort((a, b) => a - b) + .map(String), + ); + failures += section( + 'close codes in constants.erl but undocumented', + [...realCloseCodes] + .filter((code) => !docCloseCodes.has(code)) + .sort((a, b) => a - b) + .map(String), + ); +} + +console.log('enum names and error codes'); +{ + const liveTokens = new Set(); + const scanDirectory = async (directory: string): Promise => { + let entries: Array; + try { + entries = await readdir(directory, {withFileTypes: true}); + } catch { + return; + } + for (const entry of entries) { + if (entry.name === 'node_modules' || entry.name === 'target' || entry.name === 'dist') { + continue; + } + const resolved = path.join(directory, entry.name); + if (entry.isDirectory()) { + await scanDirectory(resolved); + continue; + } + if (!/\.(ts|rs|erl|hrl|json)$/u.test(entry.name)) { + continue; + } + const source = await readFile(resolved, 'utf8'); + for (const token of source.matchAll(/[A-Z][A-Z0-9_]+/gu)) { + liveTokens.add(token[0]); + } + } + }; + for (const sub of [ + 'packages', + 'fluxer_api/src', + 'fluxer_admin/src', + 'fluxer_gateway/src', + 'fluxer_media_proxy/src', + ]) { + await scanDirectory(path.join(REPO_ROOT, sub)); + } + + const enumRows: Array<{file: string; line: number; name: string}> = []; + const codeRows: Array<{file: string; line: number; code: string}> = []; + for (const file of await walk(DOCS_ROOT)) { + const relative = path.relative(DOCS_ROOT, file); + const lines = (await readFile(file, 'utf8')).split('\n'); + let inEnumTable = false; + for (let i = 0; i < lines.length; i += 1) { + const line = lines[i]; + if (/^\|\s*Value\s*\|\s*Name\s*\|/u.test(line)) { + inEnumTable = true; + continue; + } + if (inEnumTable && !line.startsWith('|')) { + inEnumTable = false; + } + if (inEnumTable && !line.startsWith('| ---')) { + const row = line.match(/^\|\s*-?\d+\s*\|\s*([A-Z][A-Z0-9_]*)/u); + if (row != null) { + enumRows.push({file: relative, line: i + 1, name: row[1]}); + } + } + for (const cited of line.matchAll(/\b[1-5]\d\d\s+`([A-Z][A-Z0-9_]{2,})`/gu)) { + codeRows.push({file: relative, line: i + 1, code: cited[1]}); + } + } + } + + const fabricatedEnums = enumRows.filter((row) => !liveTokens.has(row.name)); + const fabricatedCodes = codeRows.filter((row) => !liveTokens.has(row.code)); + console.log(` enum rows checked: ${enumRows.length.toString()}`); + console.log(` error codes cited with a status: ${new Set(codeRows.map((r) => r.code)).size.toString()}`); + console.log(' names only, not value bindings: the same name is reused across enums, so a global'); + console.log(' value comparison misattributes. ReportType.GUILD is 2 while ChannelContext.GUILD is 0'); + failures += section( + 'enum names absent from live source', + fabricatedEnums.map((row) => `${row.file}:${row.line.toString()} ${row.name}`).sort(), + ); + failures += section( + 'error codes absent from live source', + [...new Set(fabricatedCodes.map((row) => `${row.file}:${row.line.toString()} ${row.code}`))].sort(), + ); +} + +console.log('registry codes with a producer'); +{ + const CODE_REGISTRIES = ['packages/constants/src/ApiErrorCodes.ts', 'packages/constants/src/ValidationErrorCodes.ts']; + const NON_PRODUCERS = [...CODE_REGISTRIES, 'packages/errors/src/i18n/ErrorCodeMappings.ts']; + const registryKeys = new Map(); + for (const relative of CODE_REGISTRIES) { + const source = await readFile(path.join(REPO_ROOT, relative), 'utf8'); + for (const entry of source.matchAll(/^\t([A-Z][A-Z0-9_]*):/gmu)) { + registryKeys.set(entry[1], relative); + } + } + + const producerTokens = new Set(); + const scanProducers = async (directory: string): Promise => { + let entries: Array; + try { + entries = await readdir(directory, {withFileTypes: true}); + } catch { + return; + } + for (const entry of entries) { + if (entry.name === 'node_modules' || entry.name === 'target' || entry.name === 'dist' || entry.name === 'tests') { + continue; + } + const resolved = path.join(directory, entry.name); + if (entry.isDirectory()) { + await scanProducers(resolved); + continue; + } + if (!/\.(ts|rs|erl|hrl)$/u.test(entry.name) || entry.name.endsWith('.test.ts')) { + continue; + } + if (NON_PRODUCERS.includes(path.relative(REPO_ROOT, resolved))) { + continue; + } + const source = await readFile(resolved, 'utf8'); + for (const token of source.matchAll(/\b[A-Z][A-Z0-9_]+\b/gu)) { + producerTokens.add(token[0]); + } + } + }; + for (const sub of [ + 'packages', + 'fluxer_api/src', + 'fluxer_admin/src', + 'fluxer_gateway/src', + 'fluxer_media_proxy/src', + 'fluxer_unfurl/src', + ]) { + await scanProducers(path.join(REPO_ROOT, sub)); + } + + const withoutProducer = [...registryKeys] + .filter(([code]) => !producerTokens.has(code)) + .map(([code, relative]) => `${relative} ${code}`) + .sort(); + console.log(` registry codes checked: ${registryKeys.size.toString()}`); + console.log(' a producer is any live server file outside the two registries, ErrorCodeMappings.ts and the'); + console.log(' generated specs. Test files do not count, the api test harness controller does'); + failures += section('registry codes with no producer', withoutProducer); +} + +console.log('permission bits'); +{ + const constants = await readFile(path.join(REPO_ROOT, 'packages/constants/src/ChannelConstants.ts'), 'utf8'); + const block = constants.match(/export const Permissions = \{([\s\S]*?)\} as const;/u); + const livePermissions = new Map(); + if (block != null) { + for (const entry of block[1].matchAll(/([A-Z][A-Z0-9_]*):\s*1n\s*<<\s*(\d+)n/gu)) { + livePermissions.set(entry[1], Number.parseInt(entry[2], 10)); + } + } + const page = (await readFile(path.join(DOCS_ROOT, 'http-api/permissions.mdx'), 'utf8')) + .replace(/</gu, '<') + .replace(/>/gu, '>'); + const documentedPermissions = new Map(); + for (const row of page.matchAll(/^\|\s*1\s*<<\s*(\d+)\s*\|\s*([A-Z][A-Z0-9_]*)/gmu)) { + documentedPermissions.set(row[2], Number.parseInt(row[1], 10)); + } + console.log(` live permissions: ${livePermissions.size.toString()}`); + console.log(` documented permissions: ${documentedPermissions.size.toString()}`); + const bitMismatches: Array = []; + for (const [name, bit] of documentedPermissions) { + const liveBit = livePermissions.get(name); + if (liveBit != null && liveBit !== bit) { + bitMismatches.push(`${name}: documented 1 << ${bit.toString()}, live 1 << ${liveBit.toString()}`); + } + } + failures += section('permission bit mismatches', bitMismatches); + failures += section( + 'documented but not a live permission', + [...documentedPermissions.keys()].filter((name) => !livePermissions.has(name)).sort(), + ); + failures += section( + 'live permission but undocumented', + [...livePermissions.keys()].filter((name) => !documentedPermissions.has(name)).sort(), + ); +} + +console.log('rate limit buckets, limits and windows'); +{ + const UNIT_MS = new Map([ + ['second', 1000], + ['seconds', 1000], + ['minute', 60_000], + ['minutes', 60_000], + ['hour', 3_600_000], + ['hours', 3_600_000], + ['day', 86_400_000], + ['days', 86_400_000], + ]); + const toMs = (text: string): number | null => { + const parsed = text.trim().match(/^(\d+)?\s*([a-z]+)$/u); + if (parsed == null) { + return null; + } + const unit = UNIT_MS.get(parsed[2]); + if (unit == null) { + return null; + } + return Number.parseInt(parsed[1] ?? '1', 10) * unit; + }; + + const configDirectory = path.join(REPO_ROOT, 'fluxer_api/src/api/rate_limit_configs'); + const liveBuckets = new Map(); + for (const entry of await readdir(configDirectory)) { + if (!entry.endsWith('.ts')) { + continue; + } + const source = await readFile(path.join(configDirectory, entry), 'utf8'); + for (const config of source.matchAll( + /bucket:\s*'([^']+)',\s*config:\s*\{limit:\s*(\d+),\s*windowMs:\s*ms\('([^']+)'\)/gu, + )) { + liveBuckets.set(config[1], {limit: Number.parseInt(config[2], 10), windowMs: toMs(config[3])}); + } + } + + const claimPattern = + /([\d,]+) requests? per ([a-z0-9 ]+?)(?:,| for [^.]*?,) on the (?:shared )?`([a-z0-9_:@{}]+)` bucket/gu; + const problems: Array = []; + let claims = 0; + for (const file of await walk(DOCS_ROOT)) { + const relative = path.relative(DOCS_ROOT, file); + const lines = (await readFile(file, 'utf8')).split('\n'); + for (let i = 0; i < lines.length; i += 1) { + for (const claim of lines[i].matchAll(claimPattern)) { + claims += 1; + const bucket = claim[3]; + const limit = Number.parseInt(claim[1].replace(/,/gu, ''), 10); + const windowMs = toMs(claim[2]); + const live = liveBuckets.get(bucket); + if (live == null) { + problems.push(`${relative}:${(i + 1).toString()} bucket ${bucket} is not in any rate limit config`); + continue; + } + if (live.limit !== limit) { + problems.push( + `${relative}:${(i + 1).toString()} ${bucket} documented limit ${limit.toString()}, live ${live.limit.toString()}`, + ); + } + if (windowMs != null && live.windowMs != null && live.windowMs !== windowMs) { + problems.push( + `${relative}:${(i + 1).toString()} ${bucket} documented window ${windowMs.toString()}ms, live ${live.windowMs.toString()}ms`, + ); + } + } + } + } + console.log(` buckets defined with a limit and window: ${liveBuckets.size.toString()}`); + console.log(` documented rate limit claims parsed: ${claims.toString()}`); + failures += section('rate limit disagreements', problems); +} + +console.log('instance limit keys'); +{ + const defaults = await readFile(path.join(REPO_ROOT, 'packages/limits/src/LimitDefaults.ts'), 'utf8'); + const liveKeys = new Set(); + for (const entry of defaults.matchAll(/^\s+([a-z][a-z0-9_]*):/gmu)) { + liveKeys.add(entry[1]); + } + const instancePage = await readFile(path.join(DOCS_ROOT, 'http-api/instance.mdx'), 'utf8'); + const documentedKeys = new Set(); + for (const row of instancePage.matchAll(/^\|\s*([^|]+?)\s*\|/gmu)) { + const name = row[1] + .replace(/.*?<\/sup>/gu, '') + .replace(/`/gu, '') + .trim(); + if (/^[a-z][a-z0-9_]*$/u.test(name)) { + documentedKeys.add(name); + } + } + console.log(` live limit keys: ${liveKeys.size.toString()}`); + console.log( + ` documented in the instance limit-keys table: ${[...liveKeys].filter((k) => documentedKeys.has(k)).length.toString()}`, + ); + failures += section( + 'live limit keys missing from the instance table', + [...liveKeys].filter((key) => !documentedKeys.has(key)).sort(), + ); +} + +console.log('media proxy image constants'); +{ + const constants = await readFile(path.join(REPO_ROOT, 'fluxer_media_proxy/src/constants.rs'), 'utf8'); + const ladderBlock = constants.match(/pub const IMAGE_SIZES: &\[u32\] = &\[([\s\S]*?)\];/u); + const ladder = ladderBlock == null ? [] : [...ladderBlock[1].matchAll(/\d+/gu)].map((m) => Number.parseInt(m[0], 10)); + const defaultSize = constants.match(/pub const DEFAULT_IMAGE_SIZE: u32 = (\d+);/u); + const dimsBlock = constants.match(/pub fn dims_for\(kind: AssetKind\) -> Option \{([\s\S]*?)\n\}/u); + const dims: Array<[string, number, number]> = []; + if (dimsBlock != null) { + for (const entry of dimsBlock[1].matchAll( + /((?:AssetKind::\w+\s*\|?\s*)+)=> Some\(Dims \{\s*min:\s*(\d+),\s*max:\s*(\d+),?\s*\}\)/gu, + )) { + dims.push([entry[1].trim(), Number.parseInt(entry[2], 10), Number.parseInt(entry[3], 10)]); + } + } + + const LADDER_PAGE = 'media-proxy/transformations.md'; + + const ladderPage = (await readFile(path.join(DOCS_ROOT, LADDER_PAGE), 'utf8')).replace(/(?<=\d),(?=\d)/gu, ''); + const problems: Array = []; + + const documentedLadder = ladderPage.match( + /`size` selects from a fixed ladder: ([\d,\s]+(?:and\s+\d+)?)\. A requested value snaps up/u, + ); + if (documentedLadder == null) { + problems.push(`${LADDER_PAGE} no longer states the size ladder in the expected sentence`); + } else { + const listed = [...documentedLadder[1].matchAll(/\d+/gu)].map((m) => Number.parseInt(m[0], 10)); + if (listed.join(',') !== ladder.join(',')) { + problems.push(`size ladder differs. documented [${listed.join(', ')}] vs IMAGE_SIZES [${ladder.join(', ')}]`); + } + } + + if (defaultSize != null && !ladderPage.includes(`resolves to ${defaultSize[1]} before clamping`)) { + problems.push(`${LADDER_PAGE} does not state that an absent size resolves to ${defaultSize[1]} before clamping`); + } + + for (const [kinds, min, max] of dims) { + if (!ladderPage.includes(`${min.toString()} through ${max.toString()}`)) { + problems.push(`${LADDER_PAGE} does not state the ${kinds} clamp of ${min.toString()} to ${max.toString()}`); + } + } + + console.log(` size ladder rungs in constants.rs: ${ladder.length.toString()}`); + console.log(` asset classes with a clamp: ${dims.length.toString()}`); + failures += section('media proxy constant disagreements', problems); +} + +console.log('self-hosting guide against deploy/self-hosting'); +{ + const envExample = await readFile(path.join(REPO_ROOT, 'deploy/self-hosting/.env.example'), 'utf8'); + const guide = await readFile(path.join(DOCS_ROOT, 'operator/get-started.mdx'), 'utf8'); + + const placeholders = new Set(); + const declared = new Set(); + const assigned = new Set(); + for (const line of envExample.split('\n')) { + const commented = line.startsWith('#'); + const assignment = (commented ? line.slice(1) : line).match(/^([A-Z][A-Z0-9_]*)=(.*)$/u); + if (assignment == null) { + continue; + } + declared.add(assignment[1]); + if (commented) { + continue; + } + assigned.add(assignment[1]); + if (assignment[2].includes('CHANGE_ME')) { + placeholders.add(assignment[1]); + } + } + + const INSTALLER_ROOT = fileURLToPath(new URL('../src/installer/', import.meta.url)); + const installers: ReadonlyArray = [ + ['install.sh', await readFile(path.join(INSTALLER_ROOT, 'install.sh'), 'utf8')], + ['install.ps1', await readFile(path.join(INSTALLER_ROOT, 'install.ps1'), 'utf8')], + ]; + const shellSource = installers[0][1]; + const powershellSource = installers[1][1]; + + const problems: Array = []; + + const INSTALLER_SECRET_FLOOR = 14; + const INSTALLER_NON_SECRET_FLOOR = 7; + const COMPOSE_REQUIRED_FLOOR = 17; + + const blockBetween = (source: string, opener: string, closer: string, label: string): string | null => { + const from = source.indexOf(opener); + if (from === -1) { + problems.push(`${label} is gone. Nothing in the script opens with \`${opener.trim()}\``); + return null; + } + const rest = source.slice(from + opener.length); + const to = rest.indexOf(closer); + if (to === -1) { + problems.push(`${label} opens but never closes at \`${closer.trim()}\``); + return null; + } + return rest.slice(0, to); + }; + + const shellRows = (fn: string, delimiter: string, label: string): Array => { + const body = blockBetween(shellSource, `${fn}() {\n\tcat <<'${delimiter}'\n`, `\n${delimiter}\n`, label); + if (body == null) { + return []; + } + return body.split('\n').filter((row) => row.trim().length > 0); + }; + + const powershellRows = (variable: string, label: string): Array => { + const body = blockBetween(powershellSource, `$${variable} = @(\n`, '\n)\n', label); + if (body == null) { + return []; + } + return body.split('\n').filter((row) => row.trim().length > 0); + }; + + const parseShellKeys = (fn: string, label: string): Map => { + const keys = new Map(); + for (const row of shellRows(fn, 'KEYS', label)) { + const parsed = row.match(/^([A-Z][A-Z0-9_]*) ([a-z0-9_]+)(?: .*)?$/u); + if (parsed == null) { + problems.push(`${label} carries the row \`${row}\`, which is not "NAME kind"`); + continue; + } + keys.set(parsed[1], parsed[2]); + } + return keys; + }; + + const parsePowerShellKeys = (variable: string, label: string): Map => { + const keys = new Map(); + for (const row of powershellRows(variable, label)) { + const parsed = row.match(/^\s*@\{Name = '([A-Z][A-Z0-9_]*)'; Kind = '([a-z0-9_]+)'/u); + if (parsed == null) { + problems.push(`${label} carries the row \`${row.trim()}\`, which is not an @{Name; Kind} entry`); + continue; + } + keys.set(parsed[1], parsed[2]); + } + return keys; + }; + + const shellSecrets = parseShellKeys('fluxer_secret_keys', 'the install.sh secret list'); + const shellNonSecrets = parseShellKeys('fluxer_non_secret_keys', 'the install.sh non-secret list'); + const powershellSecrets = parsePowerShellKeys('FluxerSecretKeys', 'the install.ps1 secret list'); + const powershellNonSecrets = parsePowerShellKeys('FluxerNonSecretKeys', 'the install.ps1 non-secret list'); + + const shellWrites = new Set([...shellSecrets.keys(), ...shellNonSecrets.keys()]); + const powershellWrites = new Set([...powershellSecrets.keys(), ...powershellNonSecrets.keys()]); + + for (const [name, secrets, nonSecrets] of [ + ['install.sh', shellSecrets, shellNonSecrets], + ['install.ps1', powershellSecrets, powershellNonSecrets], + ] as const) { + if (secrets.size < INSTALLER_SECRET_FLOOR) { + problems.push( + `${name} declares ${secrets.size.toString()} secret keys, floor is ${INSTALLER_SECRET_FLOOR.toString()}`, + ); + } + if (nonSecrets.size < INSTALLER_NON_SECRET_FLOOR) { + problems.push( + `${name} declares ${nonSecrets.size.toString()} non-secret keys, floor is ${INSTALLER_NON_SECRET_FLOOR.toString()}`, + ); + } + } + + for (const name of placeholders) { + for (const [script, secrets] of [ + ['install.sh', shellSecrets], + ['install.ps1', powershellSecrets], + ] as const) { + if (!secrets.has(name)) { + problems.push(`${name} ships as CHANGE_ME but ${script} never sets it`); + } + } + } + + for (const name of new Set([...shellWrites, ...powershellWrites])) { + if (!declared.has(name)) { + problems.push(`the installer writes ${name}, which is not declared in .env.example`); + } + } + + const compareKeyLists = (label: string, shell: Map, powershell: Map): void => { + for (const [name, kind] of shell) { + const other = powershell.get(name); + if (other == null) { + problems.push(`install.sh writes the ${label} ${name} and install.ps1 does not`); + continue; + } + if (other !== kind) { + problems.push(`${name} is ${kind} in install.sh and ${other} in install.ps1`); + } + } + for (const name of powershell.keys()) { + if (!shell.has(name)) { + problems.push(`install.ps1 writes the ${label} ${name} and install.sh does not`); + } + } + }; + compareKeyLists('secret', shellSecrets, powershellSecrets); + compareKeyLists('non-secret value', shellNonSecrets, powershellNonSecrets); + + const expectedSecretKind = (name: string): string => { + if (name === 'FLUXER_VAPID_PUBLIC_KEY') { + return 'vapid_public'; + } + if (name === 'FLUXER_VAPID_PRIVATE_KEY') { + return 'vapid_private'; + } + if (name.endsWith('_BASE64')) { + return 'base64'; + } + return 'hex'; + }; + for (const [script, secrets] of [ + ['install.sh', shellSecrets], + ['install.ps1', powershellSecrets], + ] as const) { + for (const [name, kind] of secrets) { + const wanted = expectedSecretKind(name); + if (kind !== wanted) { + problems.push(`${script} generates ${name} as ${kind}, and the value has to be ${wanted}`); + } + } + } + + const shippedAssets = (await readdir(path.join(REPO_ROOT, 'deploy/self-hosting'))) + .filter((entry) => entry !== '.gitignore') + .sort(); + const shellStackFiles = shellRows('fluxer_stack_files', 'FILES', 'the install.sh download list'); + const powershellStackFiles: Array = []; + for (const row of powershellRows('FluxerStackFiles', 'the install.ps1 download list')) { + const parsed = row.match(/^\s*'([^']+)'\s*$/u); + if (parsed == null) { + problems.push(`the install.ps1 download list carries \`${row.trim()}\`, which is not a quoted file name`); + continue; + } + powershellStackFiles.push(parsed[1]); + } + for (const [script, list] of [ + ['install.sh', shellStackFiles], + ['install.ps1', powershellStackFiles], + ] as const) { + const sorted = [...list].sort(); + if (sorted.join(', ') !== shippedAssets.join(', ')) { + problems.push( + `${script} downloads [${sorted.join(', ')}] and deploy/self-hosting ships [${shippedAssets.join(', ')}]`, + ); + } + } + + const PIPE_TO_SHELL = + /(?:curl|wget|iwr|Invoke-WebRequest)[^\n|]*\|\s*(?:sudo\s+)?(?:sh|bash|zsh|iex|Invoke-Expression)\b/iu; + const docsPages = await walk(DOCS_ROOT); + for (const [name, source] of installers) { + if (PIPE_TO_SHELL.test(source)) { + problems.push(`${name} pipes a download into a shell`); + } + } + for (const page of docsPages) { + if (PIPE_TO_SHELL.test(await readFile(page, 'utf8'))) { + problems.push(`${path.relative(DOCS_ROOT, page)} pipes a download into a shell`); + } + } + + for (const [name, source] of installers) { + if (source.includes('—')) { + problems.push(`${name} contains an em dash`); + } + if (source.includes('\r')) { + problems.push(`${name} contains a CR byte, so the served digest would disagree with sha256sum on the file`); + } + if (!source.endsWith('\n') || source.endsWith('\n\n')) { + problems.push(`${name} does not end with exactly one newline`); + } + } + + for (const [name, source] of installers) { + const published = installerChecksumLine(name, source); + const expected = `${createHash('sha256').update(Buffer.from(source, 'utf8')).digest('hex')} ${name}\n`; + if (published !== expected) { + problems.push( + `the published checksum for ${name} is \`${published.trim()}\` and sha256 of the file is \`${expected.trim()}\``, + ); + } + } + + const INSTALLER_ENDPOINTS = ['install.sh.ts', 'install.sh.sha256.ts', 'install.ps1.ts', 'install.ps1.sha256.ts']; + const digestBearing: Array<[string, string]> = []; + for (const endpoint of INSTALLER_ENDPOINTS) { + const source = await readFile(fileURLToPath(new URL(`../src/pages/${endpoint}`, import.meta.url)), 'utf8'); + if (!source.includes("from '../installer/Installer'")) { + problems.push(`src/pages/${endpoint} no longer serves the script through src/installer/Installer`); + } + digestBearing.push([`src/pages/${endpoint}`, source]); + } + digestBearing.push([ + 'src/components/InstallerChecksum.astro', + await readFile(fileURLToPath(new URL('../src/components/InstallerChecksum.astro', import.meta.url)), 'utf8'), + ]); + for (const page of docsPages) { + const text = await readFile(page, 'utf8'); + if (text.includes('install.sh') || text.includes('install.ps1')) { + digestBearing.push([path.relative(DOCS_ROOT, page), text]); + } + } + for (const [where, text] of digestBearing) { + if (/\b[0-9a-f]{64}\b/u.test(text)) { + problems.push( + `${where} carries a literal 64-character hex digest, which goes stale the next time a script changes`, + ); + } + } + + const shellParse = spawnSync('sh', ['-n', path.join(INSTALLER_ROOT, 'install.sh')], {encoding: 'utf8'}); + if (shellParse.error != null) { + problems.push(`sh -n could not run against install.sh: ${shellParse.error.message}`); + } else if (shellParse.status !== 0) { + problems.push(`sh -n rejects install.sh: ${shellParse.stderr.trim()}`); + } + + const DOCKER_STUB = [ + '#!/bin/sh', + 'case "$1 $2" in', + " '--version ') echo 'Docker version 27.1.1, build stub' ;;", + " 'compose version') if [ \"$3\" = '--short' ]; then echo '2.30.3'; else echo 'v2.30.3'; fi ;;", + " 'compose config') echo 'ghcr.io/fluxerapp/fluxer-api:v1' ;;", + 'esac', + 'exit 0', + '', + ].join('\n'); + const CURL_STUB = [ + '#!/bin/sh', + "out=''", + "prev=''", + 'for arg in "$@"; do', + ' if [ "$prev" = \'-o\' ]; then out=$arg; fi', + ' prev=$arg', + 'done', + '[ -z "$out" ] || printf \'name: fluxer\\nservices:\\n api:\\n image: stub\\n\' > "$out"', + '', + ].join('\n'); + + const sandbox = await mkdtemp(path.join(tmpdir(), 'fluxer-installer-')); + try { + const stubBin = path.join(sandbox, 'bin'); + await mkdir(stubBin, {recursive: true}); + await writeFile(path.join(stubBin, 'docker'), DOCKER_STUB, {mode: 0o755}); + await writeFile(path.join(stubBin, 'curl'), CURL_STUB, {mode: 0o755}); + await writeFile(path.join(stubBin, 'openssl'), '#!/bin/sh\nexit 0\n', {mode: 0o755}); + + const instance = path.join(sandbox, 'instance'); + await mkdir(instance, {recursive: true}); + await writeFile(path.join(instance, '.env'), 'FLUXER_DOMAIN=x.example\nFLUXER_IMAGE_TAG=2026.813.205040\n'); + await writeFile(path.join(instance, 'docker-compose.yml'), 'name: fluxer\nservices:\n api:\n image: stub\n'); + + const plannedRef = (label: string, args: ReadonlyArray): string | null => { + const run = spawnSync('sh', [path.join(INSTALLER_ROOT, 'install.sh'), ...args], { + encoding: 'utf8', + env: {...process.env, PATH: `${stubBin}${path.delimiter}${process.env.PATH ?? ''}`}, + }); + if (run.error != null) { + problems.push(`install.sh ${label} could not run: ${run.error.message}`); + return null; + } + if (run.status !== 0) { + problems.push(`install.sh ${label} exited ${String(run.status)}: ${run.stderr.trim()}`); + return null; + } + const line = run.stdout.match(/^ {2}ref\s+(\S+)$/mu); + if (line == null) { + problems.push(`install.sh ${label} printed no ref line`); + return null; + } + return line[1]; + }; + + const INSTALL_ARGS = [ + '--dry-run', + '--non-interactive', + '--allow-root', + '--domain', + 'x.example', + '--email', + 'a@x.example', + '--dir', + path.join(sandbox, 'target'), + ]; + const REF_CASES: ReadonlyArray, string]> = [ + ['on the default image tag', INSTALL_ARGS, 'main'], + ['under --image-tag latest', [...INSTALL_ARGS, '--image-tag', 'latest'], 'main'], + ['under --image-tag 2026.813.205040', [...INSTALL_ARGS, '--image-tag', '2026.813.205040'], '2026.813.205040'], + [ + 'under --ref feature/x --image-tag 2026.813.205040', + [...INSTALL_ARGS, '--ref', 'feature/x', '--image-tag', '2026.813.205040'], + 'feature/x', + ], + [ + 'under --update against a pinned .env', + ['--update', '--dry-run', '--allow-root', '--dir', instance], + '2026.813.205040', + ], + ]; + for (const [label, args, expected] of REF_CASES) { + const resolved = plannedRef(label, args); + if (resolved != null && resolved !== expected) { + problems.push(`install.sh ${label} plans ref ${resolved}, and the image tag it pairs with wants ${expected}`); + } + } + } finally { + await rm(sandbox, {recursive: true, force: true}); + } + + const covered = new Set( + [...placeholders].filter((name) => shellSecrets.has(name) && powershellSecrets.has(name)), + ); + + const compose = await readFile(path.join(REPO_ROOT, 'deploy/self-hosting/docker-compose.yml'), 'utf8'); + const composeLines = compose.split('\n'); + + const composeRequired = new Set(); + for (const required of compose.matchAll(/\$\{([A-Z][A-Z0-9_]*):\?/gu)) { + composeRequired.add(required[1]); + } + if (composeRequired.size < COMPOSE_REQUIRED_FLOOR) { + problems.push( + `docker-compose.yml parsed to ${composeRequired.size.toString()} :?-required variables, floor is ${COMPOSE_REQUIRED_FLOOR.toString()}`, + ); + } + for (const required of compose.matchAll(/\$\{([A-Z][A-Z0-9_]*):\?([^}]*)\}/gu)) { + if (required[2].trim().length === 0) { + problems.push( + `docker-compose.yml requires ${required[1]} with an empty :? message, so Compose stops on a bare variable name`, + ); + } + } + const vapidEmailDerivation = 'FLUXER_VAPID_EMAIL: ${FLUXER_VAPID_EMAIL:-admin@${FLUXER_DOMAIN}}'; + if (!compose.includes(vapidEmailDerivation)) { + problems.push('docker-compose.yml no longer derives FLUXER_VAPID_EMAIL from FLUXER_DOMAIN'); + } else if (assigned.has('FLUXER_VAPID_EMAIL')) { + problems.push( + '.env.example assigns FLUXER_VAPID_EMAIL, so a copied .env defeats the admin@FLUXER_DOMAIN derivation', + ); + } + let composeRequiredCovered = 0; + for (const name of composeRequired) { + const missing: Array = []; + if (!shellWrites.has(name)) { + missing.push('install.sh'); + } + if (!powershellWrites.has(name)) { + missing.push('install.ps1'); + } + if (missing.length === 0) { + composeRequiredCovered += 1; + continue; + } + problems.push(`docker-compose.yml requires ${name} with :? and ${missing.join(' and ')} never writes it`); + } + const publishers = new Set(); + const publishedPorts = new Set(); + let currentService: string | null = null; + for (let i = 0; i < composeLines.length; i += 1) { + const service = composeLines[i].match(/^ {2}([a-z][a-z0-9_-]*):\s*$/u); + if (service != null) { + currentService = service[1]; + continue; + } + if (!/^ {4}ports:\s*$/u.test(composeLines[i]) || currentService == null) { + continue; + } + publishers.add(currentService); + for (let j = i + 1; j < composeLines.length && /^\s*-\s/u.test(composeLines[j]); j += 1) { + const mapping = composeLines[j].match(/(\d+)\}?(\/udp)?"\s*$/u); + if (mapping != null) { + publishedPorts.add(`${mapping[1]}${mapping[2] ?? ''}`); + } + } + } + if (!guide.includes('The edge and LiveKit are the only services that publish ports')) { + problems.push('the guide no longer states which services publish ports'); + } else if (publishers.size !== 2 || !publishers.has('edge') || !publishers.has('livekit')) { + problems.push( + `the guide says only the edge and LiveKit publish ports, but the compose file publishes from [${[...publishers].sort().join(', ')}]`, + ); + } + for (const port of publishedPorts) { + const bare = port.replace('/udp', ''); + if (!guide.includes(bare)) { + problems.push(`the compose file publishes ${port} but the guide never mentions ${bare}`); + } + } + + const livekitConfig = blockBetween( + compose, + ' LIVEKIT_CONFIG: |\n', + '\n ports:\n', + "the livekit service's LIVEKIT_CONFIG", + ); + if (livekitConfig != null) { + const coupled: ReadonlyArray = [ + ['tcp_port', '${FLUXER_LIVEKIT_TCP_PORT:-7881}'], + ['udp_port', '${FLUXER_LIVEKIT_UDP_PORT:-7882}'], + ['api_key', '${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env}'], + ]; + for (const [key, expression] of coupled) { + if (!livekitConfig.includes(`${key}: ${expression}`)) { + problems.push(`LIVEKIT_CONFIG sets ${key} to something other than ${expression}, so it no longer follows .env`); + } + } + for (const mapping of [ + '${FLUXER_LIVEKIT_TCP_PORT:-7881}:${FLUXER_LIVEKIT_TCP_PORT:-7881}', + '${FLUXER_LIVEKIT_UDP_PORT:-7882}:${FLUXER_LIVEKIT_UDP_PORT:-7882}/udp', + ]) { + if (!compose.includes(`"${mapping}"`)) { + problems.push( + `the livekit service no longer publishes ${mapping}, so the port it advertises in ICE candidates is not the one the host forwards`, + ); + } + } + } + + const configPage = await readFile(path.join(DOCS_ROOT, 'operator/configuration.mdx'), 'utf8'); + const configLines = configPage.split('\n'); + const envValues = new Map(); + for (const line of envExample.split('\n')) { + const assignment = line.match(/^([A-Z][A-Z0-9_]*)=(.*)$/u); + if (assignment != null) { + envValues.set(assignment[1], assignment[2]); + } + } + + const ENV_VALUE_FLOOR = 19; + const ENV_VALUE_UNSET = 'unset'; + let inEnvValueTable = false; + let envEntry: string | null = null; + let envRowsChecked = 0; + const checkEnvValue = (name: string, documented: string): void => { + envRowsChecked += 1; + const actual = envValues.get(name); + if (documented === ENV_VALUE_UNSET) { + if (actual != null) { + problems.push(`configuration.mdx says ${name} is unset but .env.example assigns \`${actual}\``); + } + return; + } + if (actual == null) { + problems.push(`configuration.mdx documents ${name} under "Value in .env.example" but it is not in that file`); + return; + } + if (actual !== documented) { + problems.push(`configuration.mdx says ${name} is \`${documented}\` but .env.example has \`${actual}\``); + } + }; + for (const line of configLines) { + const entryHeading = line.match(/^#### `([A-Z][A-Z0-9_]*)`(?:[^<]*<\/sup>)?\s*$/u); + if (entryHeading != null) { + envEntry = entryHeading[1]; + inEnvValueTable = false; + continue; + } + if (/^#{1,6}\s/u.test(line)) { + envEntry = null; + } + if (envEntry != null) { + const marker = line.match(/^`\.env\.example`\s+(?:`([^`]*)`|(unset)\b)/u); + if (marker != null) { + checkEnvValue(envEntry, marker[1] ?? ENV_VALUE_UNSET); + envEntry = null; + continue; + } + } + if (/^\|\s*Variable\s*\|/u.test(line)) { + inEnvValueTable = line.includes('Value in `.env.example`'); + continue; + } + if (!line.startsWith('|')) { + inEnvValueTable = false; + continue; + } + if (!inEnvValueTable) { + continue; + } + const row = line.match(/^\|\s*([A-Z][A-Z0-9_]*)(?:[^<]*<\/sup>)?\s*\|\s*(?:`([^`]*)`|(unset))\s*\|/u); + if (row == null) { + continue; + } + checkEnvValue(row[1], row[2] ?? ENV_VALUE_UNSET); + } + console.log(` configuration.mdx .env.example values checked: ${envRowsChecked.toString()}`); + if (envRowsChecked < ENV_VALUE_FLOOR) { + problems.push( + `configuration.mdx .env.example values checked fell to ${envRowsChecked.toString()}, floor is ${ENV_VALUE_FLOOR.toString()}`, + ); + } + + const composeServices = new Set(); + { + let inServices = false; + for (const line of composeLines) { + if (/^services:\s*$/u.test(line)) { + inServices = true; + continue; + } + if (/^[a-z]/u.test(line)) { + inServices = false; + } + const service = line.match(/^ {2}([a-z][a-z0-9_-]*):\s*$/u); + if (inServices && service != null) { + composeServices.add(service[1]); + } + } + } + const operatorPages = ['operator/get-started.mdx', 'operator/configuration.mdx', 'operator/upgrading.mdx']; + const namedServices = new Set(); + for (const page of operatorPages) { + const text = await readFile(path.join(DOCS_ROOT, page), 'utf8'); + for (const backticked of text.matchAll(/`([a-z][a-z0-9-]{2,})`/gu)) { + namedServices.add(backticked[1]); + } + } + const knownNonServices = new Set(['docker-compose.yml', 'chat.example.com', 'admin.example.com', 'localhost']); + for (const candidate of namedServices) { + if (!candidate.endsWith('-shard') && !candidate.startsWith('fluxer-') && !composeServices.has(candidate)) { + continue; + } + if (knownNonServices.has(candidate)) { + continue; + } + const isShard = candidate.endsWith('-shard'); + const isImage = candidate.startsWith('fluxer-'); + if (!isShard && !isImage) { + continue; + } + if (isShard && !composeServices.has(candidate)) { + problems.push(`the operator docs name the service \`${candidate}\`, which is not in docker-compose.yml`); + } + if (isImage && !compose.includes(candidate)) { + problems.push(`the operator docs name the image \`${candidate}\`, which docker-compose.yml never references`); + } + } + console.log(` compose services: ${composeServices.size.toString()}`); + + const files = ['docker-compose.yml', 'Caddyfile', '.env.example']; + for (const file of files) { + if (!guide.includes(file)) { + problems.push(`the guide never mentions ${file}, which the stack ships`); + } + } + + console.log(` .env.example placeholders: ${placeholders.size.toString()}`); + console.log(` set by the guide: ${[...placeholders].filter((n) => covered.has(n)).length.toString()}`); + console.log(` installer secret keys (sh): ${shellSecrets.size.toString()}`); + console.log(` installer secret keys (ps1): ${powershellSecrets.size.toString()}`); + console.log(` installer non-secret keys: ${shellNonSecrets.size.toString()}`); + console.log(` compose required variables covered: ${composeRequiredCovered.toString()}`); + console.log(` services publishing ports: ${[...publishers].sort().join(', ')}`); + console.log(` published ports: ${[...publishedPorts].sort().join(', ')}`); + failures += section('self-hosting guide disagreements', problems); +} + +console.log('unthrottled routes and global bucket claims'); +{ + const unthrottled = controllerRoutes + .filter((route) => route.rateLimitConfig == null) + .map((route) => ({ + method: route.method.toUpperCase(), + route: route.path, + admin: route.path.startsWith('/admin'), + })); + + const publicShapes = new Set(main.map((operation) => shapeOf(operation.method, operation.path))); + const publicUnthrottled = unthrottled + .filter((entry) => !entry.admin) + .map((entry) => shapeOf(entry.method, stripVersionPrefix(entry.route.replace(/:([a-zA-Z_]+)/gu, '{$1}')))) + .filter((shape) => publicShapes.has(shape)); + const adminUnthrottled = unthrottled.filter((entry) => entry.admin); + + const page = await readFile(path.join(DOCS_ROOT, 'topics/rate-limits.md'), 'utf8'); + const problems: Array = []; + const uniquePublic = [...new Set(publicUnthrottled)].sort(); + + const unthrottledByDesign = new Set([ + 'GET /dl/desktop/{}/{}/{}/latest', + 'GET /dl/desktop/{}/{}/{}/latest/{}', + 'GET /dl/desktop/{}/{}/{}/versions', + 'GET /dl/desktop/{}/{}/{}/{}/{}', + ]); + const unexpected = uniquePublic.filter((shape) => !unthrottledByDesign.has(shape)); + const nowThrottled = [...unthrottledByDesign].filter((shape) => !uniquePublic.includes(shape)).sort(); + + if (unexpected.length > 0) { + problems.push( + `rate-limits.md says every HTTP API operation outside the desktop downloads declares a bucket, but ${unexpected.length.toString()} more declare none: ${unexpected.join(', ')}`, + ); + } + if (nowThrottled.length > 0) { + problems.push( + `rate-limits.md names the desktop downloads as the only operations with no bucket, but ${nowThrottled.length.toString()} now declare one: ${nowThrottled.join(', ')}`, + ); + } + if (!page.includes('[desktop download](/http-api/downloads/)')) { + problems.push('rate-limits.md no longer names the desktop downloads as the operations with no bucket'); + } + if (adminUnthrottled.length > 0) { + const named = adminUnthrottled.map((entry) => `${entry.method} ${entry.route}`).sort(); + problems.push( + `rate-limits.md says every Admin API operation declares a bucket, but ${named.length.toString()} declare none: ${named.join(', ')}`, + ); + } + for (const stale of ['declares no bucket', 'Two [Admin API]']) { + if (page.includes(stale)) { + problems.push(`rate-limits.md still says \`${stale}\`, and every operation now declares one`); + } + } + + const middleware = await readFile( + path.join(REPO_ROOT, 'fluxer_api/src/api/middleware/RateLimitMiddleware.ts'), + 'utf8', + ); + const high = middleware.match(/HIGH_GLOBAL_RATE_LIMIT\) !== 0n\) \{\s*return (\d+);/u); + if (high != null && !page.includes(`${Number.parseInt(high[1], 10).toLocaleString('en-US')} requests per second`)) { + problems.push(`rate-limits.md does not state the HIGH_GLOBAL_RATE_LIMIT allowance of ${high[1]}`); + } + + console.log(` public operations with no rate limit middleware: ${uniquePublic.length.toString()}`); + console.log(` admin operations with no rate limit middleware: ${adminUnthrottled.length.toString()}`); + failures += section('rate limit prose disagreements', problems); +} + +console.log('error registry and abuse signal weights'); +{ + const errorsPage = await readFile(path.join(DOCS_ROOT, 'http-api/errors.md'), 'utf8'); + const documentedCodes = new Set(); + let documentedEntries = 0; + for (const row of errorsPage.matchAll(/^\|\s*`?([A-Z][A-Z0-9_]{2,})`?\s*\|/gmu)) { + documentedCodes.add(row[1]); + documentedEntries += 1; + } + for (const entry of errorsPage.matchAll(/^#{3,4} `([A-Z][A-Z0-9_]{2,})`\s*$/gmu)) { + documentedCodes.add(entry[1]); + documentedEntries += 1; + } + const registrySource = await readFile(path.join(REPO_ROOT, 'packages/constants/src/ApiErrorCodes.ts'), 'utf8'); + const registryCodes = new Set(); + for (const entry of registrySource.matchAll(/^\t([A-Z][A-Z0-9_]*):/gmu)) { + registryCodes.add(entry[1]); + } + const problems: Array = []; + for (const code of registryCodes) { + if (!documentedCodes.has(code)) { + problems.push(`APIErrorCodes.${code} is not listed in the errors page registry`); + } + } + + const banner = await readFile(path.join(REPO_ROOT, 'fluxer_api/src/api/middleware/AbusiveIpAutoBanner.ts'), 'utf8'); + const weights = new Map(); + for (const rule of banner.matchAll(/if \(status === (\d{3})\) return ([\d.]+);/gu)) { + weights.set(rule[1], rule[2]); + } + for (const [status, weight] of weights) { + if (status === '404') { + continue; + } + if (!errorsPage.includes(`A ${status} weighs ${weight}`) && !errorsPage.includes(`a ${status} weighs ${weight}`)) { + problems.push(`errors.md does not state that a ${status} weighs ${weight}`); + } + } + + const documentedRegistryCodes = [...registryCodes].filter((c) => documentedCodes.has(c)).length; + if (documentedEntries < 491) { + problems.push( + `errors.md code entries parsed fell to ${documentedEntries.toString()}, floor is 491, the 255 API codes plus the 236 validation codes. The registry parser reads a \`| CODE |\` table row and a \`### \`CODE\`\` or \`#### \`CODE\`\` heading, and one of those shapes has stopped matching`, + ); + } + if (documentedRegistryCodes < registryCodes.size) { + problems.push( + `errors.md documents ${documentedRegistryCodes.toString()} of the ${registryCodes.size.toString()} registry codes, floor is every one of them`, + ); + } + console.log(` registry codes: ${registryCodes.size.toString()}, documented: ${documentedRegistryCodes.toString()}`); + console.log(` abuse signal weights compared: ${weights.size.toString()}`); + failures += section('error registry disagreements', problems); +} + +console.log('gateway numeric limits'); +{ + const erlConstants = await readFile(path.join(REPO_ROOT, 'fluxer_gateway/src/utils/constants.erl'), 'utf8'); + const sharding = await readFile(path.join(REPO_ROOT, 'fluxer_gateway/src/gateway/gateway_sharding.erl'), 'utf8'); + const abuse = await readFile(path.join(REPO_ROOT, 'fluxer_gateway/src/gateway/session_abuse_protection.erl'), 'utf8'); + const heartbeat = await readFile( + path.join(REPO_ROOT, 'fluxer_gateway/src/gateway/gateway_handler_heartbeat.erl'), + 'utf8', + ); + const limitsPage = await readFile(path.join(DOCS_ROOT, 'gateway/limits-and-rate-limits.md'), 'utf8'); + const overviewPage = await readFile(path.join(DOCS_ROOT, 'gateway/overview.md'), 'utf8'); + const corpus = `${limitsPage}\n${overviewPage}`; + + const readFn = (source: string, name: string): number | null => { + const found = source.match(new RegExp(`${name}\\(\\) -> (\\d+)`, 'u')); + return found == null ? null : Number.parseInt(found[1], 10); + }; + const readDefine = (source: string, name: string): number | null => { + const found = source.match(new RegExp(`-define\\(${name}, (\\d+)\\)`, 'u')); + return found == null ? null : Number.parseInt(found[1], 10); + }; + const grouped = (value: number): string => value.toLocaleString('en-US'); + + const problems: Array = []; + const interval = readFn(erlConstants, 'heartbeat_interval'); + + const expectations: Array<[string, number | null, (value: string) => string]> = [ + ['heartbeat interval', interval, (v) => `heartbeat interval of ${v} ms`], + ['resume timeout', readFn(erlConstants, 'resume_timeout'), (v) => `resumable for ${v} ms`], + ['max payload size', readFn(erlConstants, 'max_payload_size'), (v) => v], + ['max shard count', readDefine(sharding, 'MAX_SHARD_COUNT'), (v) => `through ${v}`], + ['max guilds per shard', readDefine(sharding, 'MAX_GUILDS_PER_SHARD'), (v) => `at most ${v} guilds`], + ['max sessions per user', readDefine(abuse, 'MAX_SESSIONS_PER_USER'), (v) => `at most ${v} live sessions`], + ]; + if (interval != null && heartbeat.includes('div 3')) { + expectations.push(['heartbeat check cadence, interval div 3', Math.floor(interval / 3), (v) => `every ${v} ms`]); + } + if (interval != null && heartbeat.includes('* 10) >= (Interval * 9)')) { + expectations.push([ + 'heartbeat ping threshold, interval times 9 over 10', + (interval * 9) / 10, + (v) => `When ${v} ms have elapsed`, + ]); + } + let compared = 0; + for (const [label, value, phrase] of expectations) { + if (value == null) { + problems.push(`could not read ${label} from the Gateway source`); + continue; + } + compared += 1; + if (!corpus.includes(phrase(grouped(value))) && !corpus.includes(phrase(String(value)))) { + problems.push(`the Gateway docs never state the ${label} of ${grouped(value)} in the expected phrasing`); + } + } + console.log(` gateway constants compared: ${compared.toString()}`); + failures += section('gateway limit disagreements', problems); +} + +console.log('snowflake layout'); +{ + const core = await readFile(path.join(REPO_ROOT, 'packages/constants/src/Core.ts'), 'utf8'); + const snowflakeSource = await readFile(path.join(REPO_ROOT, 'packages/snowflake/src/Snowflake.ts'), 'utf8'); + const page = await readFile(path.join(DOCS_ROOT, 'snowflakes.md'), 'utf8'); + const problems: Array = []; + + const epochMatch = core.match(/export const FLUXER_EPOCH = (\d+);/u); + const workerBits = snowflakeSource.match(/WORKER_ID_BITS = (\d+)n/u); + const sequenceBits = snowflakeSource.match(/SEQUENCE_BITS = (\d+)n/u); + + if (epochMatch == null || workerBits == null || sequenceBits == null) { + problems.push('could not read FLUXER_EPOCH, WORKER_ID_BITS or SEQUENCE_BITS from source'); + } else { + const epoch = Number.parseInt(epochMatch[1], 10); + const worker = Number.parseInt(workerBits[1], 10); + const sequence = Number.parseInt(sequenceBits[1], 10); + const shift = worker + sequence; + if (!page.includes(String(epoch))) { + problems.push(`snowflakes.md does not state the epoch ${epoch.toString()}`); + } + const epochIso = new Date(epoch).toISOString().replace('.000Z', '.000Z'); + if (!page.includes(epochIso.slice(0, 10))) { + problems.push(`snowflakes.md does not state the epoch date ${epochIso}`); + } + if (!page.includes(`right by ${shift.toString()} bits`)) { + problems.push(`snowflakes.md does not state the timestamp shift of ${shift.toString()} bits`); + } + if (!page.includes(`through \`${((1 << worker) - 1).toString()}\``)) { + problems.push(`snowflakes.md does not state the maximum worker id of ${((1 << worker) - 1).toString()}`); + } + const timestampBits = 63 - shift; + const lastMs = epoch + 2 ** timestampBits - 1; + const lastIso = new Date(lastMs).toISOString().replace('Z', 'Z'); + if (!page.includes(lastIso.slice(0, 19))) { + problems.push(`snowflakes.md does not state the last representable instant ${lastIso}`); + } + console.log( + ` epoch ${epoch.toString()} (${epochIso.slice(0, 10)}), shift ${shift.toString()}, worker ${worker.toString()} bits, sequence ${sequence.toString()} bits`, + ); + console.log(` last representable instant: ${lastIso}`); + } + failures += section('snowflake layout disagreements', problems); +} + +console.log('attachment upload geometry'); +{ + const limits = await readFile(path.join(REPO_ROOT, 'packages/constants/src/LimitConstants.ts'), 'utf8'); + const page = await readFile(path.join(DOCS_ROOT, 'topics/uploads.md'), 'utf8'); + const problems: Array = []; + const readConst = (name: string): number | null => { + const found = limits.match(new RegExp(`${name} = ([^;]+);`, 'u')); + if (found == null) { + return null; + } + const expression = found[1].replace(/_/gu, '').trim(); + if (!/^[\d*+\s]+$/u.test(expression)) { + return null; + } + return expression + .split('+') + .map((term) => term.split('*').reduce((a, b) => a * Number.parseInt(b.trim(), 10), 1)) + .reduce((a, b) => a + b, 0); + }; + + const constants: Array<[string, string]> = [ + ['ATTACHMENT_UPLOAD_CHUNK_THRESHOLD', 'singlepart threshold'], + ['ATTACHMENT_UPLOAD_MIN_CHUNK_SIZE', 'minimum part size'], + ['ATTACHMENT_UPLOAD_MAX_CHUNKS', 'maximum part count'], + ['ATTACHMENT_MAX_SIZE_NON_PREMIUM', 'non-premium attachment ceiling'], + ['ATTACHMENT_MAX_SIZE_PREMIUM', 'premium attachment ceiling'], + ['ATTACHMENT_MAX_SIZE_BOT', 'bot attachment ceiling'], + ]; + const divisor = readConst('ATTACHMENT_UPLOAD_TARGET_PART_COUNT'); + if (divisor == null) { + problems.push('could not read ATTACHMENT_UPLOAD_TARGET_PART_COUNT from LimitConstants.ts'); + } else if (!page.includes(`divided by ${divisor.toString()}`)) { + problems.push(`uploads.md does not say the part size is the declared size divided by ${divisor.toString()}`); + } + let compared = 0; + for (const [name, label] of constants) { + const value = readConst(name); + if (value == null) { + problems.push(`could not read ${name} from LimitConstants.ts`); + continue; + } + compared += 1; + const bare = new RegExp(`(?(); + const documentedLabels = new Map(); + for (const row of page.matchAll(/^\|\s*(GET|POST|PUT|PATCH|DELETE)\s*\|\s*(\/v1\/\S+?)\s*\|/gmu)) { + const normalised = shapeOf(row[1], stripVersionPrefix(row[2])); + documented.add(normalised); + documentedLabels.set(normalised, `${row[1]} ${row[2]}`); + } + + const gated = new Set(); + for (const route of controllerRoutes) { + if (route.middlewares.some((name) => /Captcha|GroupDm\w*Protection/u.test(name))) { + gated.add(astRoute(route)); + } + } + + const problems: Array = []; + for (const route of gated) { + if (!documented.has(route)) { + problems.push(`${route} is captcha gated in source but not listed on the captcha page`); + } + } + for (const route of documented) { + if (!gated.has(route)) { + problems.push( + `the captcha page lists ${documentedLabels.get(route) ?? route}, which is not captcha gated in source`, + ); + } + } + console.log(` captcha gated routes in source: ${gated.size.toString()}, documented: ${documented.size.toString()}`); + failures += section('captcha gating disagreements', problems); +} + +console.log('bot capability flag (from the middleware chain)'); +{ + const BOT_EXEMPT = new Map([ + ['GET /applications/@me', {docs: true, why: 'reads the Authorization header directly in the handler'}], + ['GET /gateway/bot', {docs: true, why: 'requires a bot-shaped token, parsed in the handler'}], + ['POST /guilds', {docs: false, why: 'the service rejects bots with BOTS_CANNOT_CREATE_GUILDS'}], + ['POST /guilds/{}/delete', {docs: false, why: 'the caller must own the guild and a bot never can'}], + ]); + const problems: Array = []; + let compared = 0; + for (const route of controllerRoutes) { + const key = astRoute(route); + if (key.startsWith('GET /admin') || key.includes(' /admin/')) { + continue; + } + const documented = documentedFlags.get(key); + if (documented == null) { + continue; + } + const anyLogin = route.hasLoginRequired || route.hasLoginRequiredAllowSuspicious; + const sourceAcceptsBot = anyLogin && !route.hasDefaultUserOnly; + const exemption = BOT_EXEMPT.get(key); + if (exemption != null) { + if (documented.bot !== exemption.docs) { + problems.push( + `${key} is an exempted route expected to document bot=${String(exemption.docs)} (${exemption.why}), but the page says ${String(documented.bot)}`, + ); + } + continue; + } + compared += 1; + if (documented.bot !== sourceAcceptsBot) { + problems.push( + `${key} documents bot=${String(documented.bot)} but the middleware chain gives ${String(sourceAcceptsBot)}`, + ); + } + } + console.log(` routes compared against the middleware chain: ${compared.toString()}`); + console.log(` exempted, each verified by hand: ${BOT_EXEMPT.size.toString()}`); + failures += section('bot flag disagreements', problems); +} + +console.log('unauthenticated capability flag (from the middleware chain)'); +{ + const WEBHOOK_TOKEN_CREDENTIAL = 'the webhook ID and token in the path are the complete credential'; + const OAUTH2_CLIENT_CREDENTIAL = + 'the client authenticates with HTTP Basic or the client_id and client_secret form fields'; + const UNAUTHENTICATED_EXEMPT = new Map([ + ['GET /applications/@me', {docs: false, why: 'reads the Authorization header directly in the handler'}], + ['GET /gateway/bot', {docs: false, why: 'requires a bot-shaped token, parsed in the handler'}], + [ + 'POST /auth/handoff/complete', + {docs: false, why: 'the approving session token comes from the Authorization header or the body token field'}, + ], + ['POST /oauth2/token', {docs: false, why: OAUTH2_CLIENT_CREDENTIAL}], + ['POST /oauth2/introspect', {docs: false, why: OAUTH2_CLIENT_CREDENTIAL}], + ['POST /oauth2/token/revoke', {docs: false, why: OAUTH2_CLIENT_CREDENTIAL}], + ['GET /webhooks/{}/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}], + ['PATCH /webhooks/{}/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}], + ['DELETE /webhooks/{}/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}], + ['POST /webhooks/{}/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}], + ['GET /webhooks/{}/{}/messages/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}], + ['PATCH /webhooks/{}/{}/messages/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}], + ['DELETE /webhooks/{}/{}/messages/{}', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}], + ['POST /webhooks/{}/{}/github', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}], + ['POST /webhooks/{}/{}/slack', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}], + ['POST /webhooks/{}/{}/instatus', {docs: false, why: WEBHOOK_TOKEN_CREDENTIAL}], + ]); + const problems: Array = []; + let compared = 0; + for (const route of controllerRoutes) { + const key = astRoute(route); + if (key.startsWith('GET /admin') || key.includes(' /admin/')) { + continue; + } + const documented = documentedFlags.get(key); + if (documented == null) { + continue; + } + const anyLogin = route.hasLoginRequired || route.hasLoginRequiredAllowSuspicious; + const sourceIsOpen = !anyLogin && !route.middlewares.some((name) => /OAuth2Scope/u.test(name)); + const exemption = UNAUTHENTICATED_EXEMPT.get(key); + if (exemption != null) { + if (documented.unauthenticated !== exemption.docs) { + problems.push( + `${key} is an exempted route expected to document unauthenticated=${String(exemption.docs)} (${exemption.why}), but the page says ${String(documented.unauthenticated)}`, + ); + } + continue; + } + compared += 1; + if (documented.unauthenticated !== sourceIsOpen) { + problems.push( + `${key} documents unauthenticated=${String(documented.unauthenticated)} but the middleware chain gives ${String(sourceIsOpen)}`, + ); + } + } + console.log(` routes compared against the middleware chain: ${compared.toString()}`); + console.log(` exempted, each verified by hand: ${UNAUTHENTICATED_EXEMPT.size.toString()}`); + failures += section('unauthenticated flag disagreements', problems); +} + +console.log('spec security field against the middleware chain'); +{ + const MANUAL_CREDENTIAL = new Set(['GET /applications/@me']); + const BOT_SCHEME_EXEMPT = new Map([ + [ + 'POST /guilds', + 'LoginRequired admits a bot token, but GuildOperationsService rejects every bot with 400 BOTS_CANNOT_CREATE_GUILDS, so the spec must not advertise botToken', + ], + ]); + const specSecurity = new Map>(); + for (const operation of main) { + const schemes = new Set(); + for (const entry of operation.security ?? []) { + for (const scheme of Object.keys(entry)) { + schemes.add(scheme); + } + } + specSecurity.set(shapeOf(operation.method, operation.path), schemes); + } + const specBugs: Array = []; + let compared = 0; + for (const route of controllerRoutes) { + const key = astRoute(route); + const declaredSchemes = specSecurity.get(key); + if (declaredSchemes == null || MANUAL_CREDENTIAL.has(key)) { + continue; + } + compared += 1; + const anyLogin = route.hasLoginRequired || route.hasLoginRequiredAllowSuspicious; + const acceptsBot = anyLogin && !route.hasDefaultUserOnly; + const exemption = BOT_SCHEME_EXEMPT.get(key); + if (exemption != null) { + if (declaredSchemes.has('botToken')) { + specBugs.push(`${key} declares botToken, but is exempted because ${exemption}`); + } + continue; + } + if (declaredSchemes.has('botToken') && !acceptsBot) { + specBugs.push(`${key} declares botToken, but DefaultUserOnly rejects bots`); + } + if (!declaredSchemes.has('botToken') && acceptsBot && declaredSchemes.size > 0) { + specBugs.push(`${key} omits botToken, but the middleware chain admits a bot token`); + } + if (declaredSchemes.size > 0 && !anyLogin && !route.middlewares.some((name) => /Admin|OAuth2Scope/iu.test(name))) { + specBugs.push(`${key} declares [${[...declaredSchemes].join(', ')}], but the chain has no login policy`); + } + } + console.log(` routes compared: ${compared.toString()}`); + console.log(' GET /applications/@me is excluded because it reads the Authorization header in the'); + console.log(' handler, so it has no middleware to compare against.'); + console.log(` exempted, each verified by hand: ${BOT_SCHEME_EXEMPT.size.toString()}`); + for (const [key, why] of BOT_SCHEME_EXEMPT) { + console.log(` ${key}: ${why}`); + } + failures += section('spec security disagreements with the middleware chain', specBugs); +} + +console.log('admin API (against the generated spec)'); +const adminTargetOnly = [...documentedAdmin.entries()] + .filter(([shape]) => !adminShapes.has(shape)) + .map(([, route]) => `${route.method} ${route.path} (${route.file})`) + .sort(); +const adminLiveOnly = admin + .filter((operation) => !documentedAdmin.has(shapeOf(operation.method, operation.path))) + .map((operation) => `${operation.method} ${operation.path}`) + .sort(); +console.log(` live admin operations: ${admin.length.toString()}`); +console.log(` documented admin operations: ${adminDocumented.length.toString()}`); +console.log( + ` documented in the RESTful target shape, awaiting the live admin refactor: ${adminTargetOnly.length.toString()}`, +); +console.log(` live admin operations with no documented RESTful equivalent: ${adminLiveOnly.length.toString()}`); +failures += section('target-shape routes awaiting the live admin refactor', adminTargetOnly); +failures += section('live admin routes with no documented equivalent', adminLiveOnly); + +console.log(''); +console.log( + `documented routes: ${documented.length.toString()} (main ${mainDocumented.length.toString()}, admin ${adminDocumented.length.toString()}, media proxy ${mediaProxyDocumented.length.toString()})`, +); +if (failures > 0) { + console.error(`FAIL: ${failures.toString()} coverage problems`); + process.exit(1); +} +console.log('OK: every registered fluxer_api route is documented or covered by an exemption rule, and the'); +console.log('documented routes match the live main API, media proxy, and admin target shape'); diff --git a/fluxer_docs/scripts/VerifyDocsSchemas.ts b/fluxer_docs/scripts/VerifyDocsSchemas.ts new file mode 100644 index 000000000..f19964ed7 --- /dev/null +++ b/fluxer_docs/scripts/VerifyDocsSchemas.ts @@ -0,0 +1,792 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {readdir, readFile} from 'node:fs/promises'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +const DOCS_ROOT = fileURLToPath(new URL('../src/content/docs/', import.meta.url)); +const REPO_ROOT = fileURLToPath(new URL('../../', import.meta.url)); +const MAIN_SPEC = path.join(REPO_ROOT, 'fluxer_api/src/api/openapi/openapi.json'); +const ADMIN_SPEC = path.join(REPO_ROOT, 'fluxer_admin/openapi-admin.json'); + +const ROUTE_HEADER = />; + readonly components?: {schemas?: Record}; +} + +interface Operation { + readonly parameters?: Array<{name: string; in: string; schema?: SchemaNode}>; + readonly requestBody?: {content?: Record}; + readonly responses?: Record}>; +} + +interface SchemaNode { + $ref?: string; + required?: Array; + properties?: Record; + allOf?: Array; + oneOf?: Array; + anyOf?: Array; + items?: SchemaNode; + type?: string; +} + +const SUDO_MERGE_FIELDS = ['mfa_method', 'mfa_code', 'webauthn_response', 'webauthn_challenge']; + +const STREAMED_RESPONSE_ROUTES = new Map([ + ['POST /admin/system/heap-snapshots', "'Content-Type': 'application/octet-stream'"], +]); + +const EMPTY_RESPONSE_ROUTES = new Map([['DELETE /admin/users/{}/sessions', 'terminated_count']]); + +const MERGED_SCHEMA_ROUTES = new Map([ + ['POST /auth/sessions/logout', "Validator('json', LogoutAuthSessionsRequest.merge(SudoVerificationSchema))"], + [ + 'POST /guilds/{}/transfer-ownership', + "Validator('json', GuildTransferOwnershipRequest.merge(SudoVerificationSchema))", + ], +]); + +interface Mismatch { + readonly page: string; + readonly operation: string; + readonly kind: string; + readonly detail: string; +} + +async function walk(directory: string): Promise> { + const entries = await readdir(directory, {withFileTypes: true}); + const files: Array = []; + for (const entry of entries) { + if (entry.name === 'node_modules') { + continue; + } + const resolved = path.join(directory, entry.name); + if (entry.isDirectory()) { + files.push(...(await walk(resolved))); + continue; + } + if (/\.mdx?$/u.test(entry.name)) { + files.push(resolved); + } + } + return files; +} + +function stripVersion(routePath: string): string { + if (routePath.startsWith('/v1/')) { + return routePath.slice(3); + } + return routePath; +} + +function shape(method: string, routePath: string): string { + return `${method} ${routePath.split('?')[0].replace(/\{[^}]*\}/gu, '{}')}`; +} + +function resolveRef(spec: Spec, node: SchemaNode | undefined, depth = 0): SchemaNode | undefined { + if (node == null || depth > 12) { + return node; + } + if (node.$ref != null) { + const name = node.$ref.replace('#/components/schemas/', ''); + return resolveRef(spec, spec.components?.schemas?.[name], depth + 1); + } + return node; +} + +function collectRequired(spec: Spec, node: SchemaNode | undefined, depth = 0): Set { + const out = new Set(); + const resolved = resolveRef(spec, node, depth); + if (resolved == null || depth > 12) { + return out; + } + for (const name of resolved.required ?? []) { + out.add(name); + } + for (const branch of resolved.allOf ?? []) { + for (const name of collectRequired(spec, branch, depth + 1)) { + out.add(name); + } + } + const union = [...(resolved.oneOf ?? []), ...(resolved.anyOf ?? [])]; + if (union.length > 0) { + const declaringBranches = new Map(); + const requiringBranches = new Map(); + for (const branch of union) { + const branchRequired = collectRequired(spec, branch, depth + 1); + for (const name of collectProperties(spec, branch, depth + 1)) { + declaringBranches.set(name, (declaringBranches.get(name) ?? 0) + 1); + if (branchRequired.has(name)) { + requiringBranches.set(name, (requiringBranches.get(name) ?? 0) + 1); + } + } + } + for (const [name, declared] of declaringBranches) { + if ((requiringBranches.get(name) ?? 0) === declared) { + out.add(name); + } + } + } + return out; +} + +function collectPropertyTypes(spec: Spec, node: SchemaNode | undefined, depth = 0): Map { + const out = new Map(); + const resolved = resolveRef(spec, node, depth); + if (resolved == null || depth > 12) { + return out; + } + for (const [key, value] of Object.entries(resolved.properties ?? {})) { + const property = resolveRef(spec, value, depth + 1); + const type = property?.type; + if (typeof type === 'string') { + out.set(key, type); + } + } + for (const branch of [...(resolved.allOf ?? [])]) { + for (const [key, type] of collectPropertyTypes(spec, branch, depth + 1)) { + if (!out.has(key)) { + out.set(key, type); + } + } + } + return out; +} + +const DOC_TYPE_TO_JSON = new Map([ + ['snowflake', 'string'], + ['string', 'string'], + ['integer', 'integer'], + ['boolean', 'boolean'], + ['iso8601 timestamp', 'string'], + ['base64 string', 'string'], + ['float', 'number'], + ['number', 'number'], +]); + +function normaliseDocType(cell: string): string | null { + const text = cell + .replace(/.*?<\/sup>/gu, '') + .replace(/\[([^\]]*)\]\([^)]*\)/gu, '$1') + .replace(/`/gu, '') + .trim() + .replace(/^\?/u, '') + .toLowerCase(); + if (text.startsWith('array')) { + return 'array'; + } + if (text.endsWith(' object') || text.includes('object')) { + return 'object'; + } + return DOC_TYPE_TO_JSON.get(text) ?? null; +} + +function isDeprecatedProperty(property: unknown): boolean { + if (property == null || typeof property !== 'object') { + return false; + } + const node = property as {deprecated?: unknown; description?: unknown}; + if (node.deprecated === true) { + return true; + } + return typeof node.description === 'string' && node.description.trimStart().toLowerCase().startsWith('deprecated'); +} + +function collectProperties(spec: Spec, node: SchemaNode | undefined, depth = 0): Set { + const out = new Set(); + const resolved = resolveRef(spec, node, depth); + if (resolved == null || depth > 12) { + return out; + } + for (const [key, property] of Object.entries(resolved.properties ?? {})) { + if (isDeprecatedProperty(property)) { + continue; + } + out.add(key); + } + for (const branch of [...(resolved.allOf ?? []), ...(resolved.oneOf ?? []), ...(resolved.anyOf ?? [])]) { + for (const key of collectProperties(spec, branch, depth + 1)) { + out.add(key); + } + } + return out; +} + +function operationIndex(spec: Spec): Map { + const index = new Map(); + for (const [routePath, item] of Object.entries(spec.paths)) { + for (const [method, operation] of Object.entries(item)) { + index.set(shape(method.toUpperCase(), stripVersion(routePath)), operation); + } + } + return index; +} + +function cleanFieldName(cell: string): string | null { + const name = cell + .replace(/.*?<\/sup>/gu, '') + .replace(/\*\*/gu, '') + .replace(/`/gu, '') + .replace(/\\/gu, '') + .trim() + .replace(/\?$/u, ''); + if (name.length === 0) { + return null; + } + if (name === 'Field' || name === '---' || name === 'Status' || name === 'Name') { + return null; + } + if (!/^[a-z_][a-z0-9_.]*$/iu.test(name)) { + return null; + } + return name; +} + +function sectionIsByReference(lines: ReadonlyArray, start: number): boolean { + let sawTable = false; + let sawReference = false; + for (let index = start; index < lines.length; index += 1) { + const line = lines[index]; + if (line.startsWith('#')) { + break; + } + if (line.startsWith('|')) { + sawTable = true; + } + if (OBJECT_REFERENCE.test(line)) { + sawReference = true; + } + } + return sawReference && !sawTable; +} + +function tableOptionality(lines: ReadonlyArray, start: number): Map { + const out = new Map(); + let index = start; + while (index < lines.length && !lines[index].startsWith('|')) { + if (lines[index].startsWith('#')) { + return out; + } + index += 1; + } + for (; index < lines.length; index += 1) { + const line = lines[index]; + if (!line.startsWith('|')) { + break; + } + const cells = line.split('|').slice(1, -1); + if (cells.length < 2) { + continue; + } + const raw = cells[0] + .replace(/.*?<\/sup>/gu, '') + .replace(/`/gu, '') + .trim(); + const name = cleanFieldName(cells[0]); + if (name == null) { + continue; + } + out.set(name, raw.endsWith('?')); + } + return out; +} + +function tableFieldTypes(lines: ReadonlyArray, start: number): Map { + const out = new Map(); + let index = start; + while (index < lines.length && !lines[index].startsWith('|')) { + if (lines[index].startsWith('#')) { + return out; + } + index += 1; + } + for (; index < lines.length; index += 1) { + const line = lines[index]; + if (!line.startsWith('|')) { + break; + } + const cells = line.split('|').slice(1, -1); + if (cells.length < 2) { + continue; + } + const name = cleanFieldName(cells[0]); + if (name == null) { + continue; + } + const type = normaliseDocType(cells[1]); + if (type != null) { + out.set(name, type); + } + } + return out; +} + +function tableFields(lines: ReadonlyArray, start: number): Set { + const fields = new Set(); + let index = start; + while (index < lines.length && !lines[index].startsWith('|')) { + if (lines[index].startsWith('#')) { + return fields; + } + index += 1; + } + for (; index < lines.length; index += 1) { + const line = lines[index]; + if (!line.startsWith('|')) { + break; + } + const match = line.match(TABLE_ROW); + if (match == null) { + continue; + } + const name = cleanFieldName(match[1]); + if (name != null) { + fields.add(name); + } + } + return fields; +} + +const mainSpec: Spec = JSON.parse(await readFile(MAIN_SPEC, 'utf8')); +const adminSpec: Spec = JSON.parse(await readFile(ADMIN_SPEC, 'utf8')); +const mainIndex = operationIndex(mainSpec); +const adminIndex = operationIndex(adminSpec); + +function slugifyHeading(heading: string): string { + return heading + .replace(/`/gu, '') + .replace(/\[([^\]]*)\]\([^)]*\)/gu, '$1') + .replace(/<[^>]*>/gu, '') + .toLowerCase() + .replace(/[^a-z0-9\s-]/gu, '') + .trim() + .replace(/\s+/gu, '-'); +} + +const allFiles = await walk(DOCS_ROOT); +const anchorFields = new Map>(); +const anchorTypes = new Map>(); +for (const file of allFiles) { + const slug = path + .relative(DOCS_ROOT, file) + .replace(/\.(mdx|md)$/u, '') + .replace(/\/index$/u, '') + .replace(/^index$/u, ''); + const lines = (await readFile(file, 'utf8')).split('\n'); + let currentAnchor: string | null = null; + for (let i = 0; i < lines.length; i += 1) { + const line = lines[i]; + const heading = line.match(/^##\s+(.+?)\s*$/u); + if (heading != null && !line.startsWith('###')) { + currentAnchor = slugifyHeading(heading[1]); + const explicit = line.match(/id=["']([^"']+)["']/u); + if (explicit != null) { + currentAnchor = explicit[1]; + } + continue; + } + if (currentAnchor == null || !line.startsWith('|')) { + continue; + } + const row = line.match(TABLE_ROW); + if (row == null) { + continue; + } + const name = cleanFieldName(row[1]); + if (name == null) { + continue; + } + const cells = line.split('|').slice(1, -1); + const declaredType = cells.length >= 2 ? normaliseDocType(cells[1]) : null; + for (const anchorKey of [`${slug}#${currentAnchor}`]) { + const set = anchorFields.get(anchorKey) ?? new Set(); + set.add(name); + anchorFields.set(anchorKey, set); + if (declaredType != null) { + const typeMap = anchorTypes.get(anchorKey) ?? new Map(); + if (!typeMap.has(name)) { + typeMap.set(name, declaredType); + } + anchorTypes.set(anchorKey, typeMap); + } + } + } +} + +const mismatches: Array = []; +let checkedBodies = 0; +let checkedQueries = 0; +let unionBodiesSkipped = 0; +let referencedElsewhere = 0; +let documentedByReference = 0; +let responsesChecked = 0; +let responseFieldsFound = 0; +let typesCompared = 0; +let optionalityCompared = 0; +const optionalityAdvisories: Array = []; + +for (const file of await walk(DOCS_ROOT)) { + const relative = path.relative(DOCS_ROOT, file); + if (relative.startsWith('media-proxy/')) { + continue; + } + const lines = (await readFile(file, 'utf8')).split('\n'); + const pageFields = new Set(); + for (let i = 0; i < lines.length; i += 1) { + if (!lines[i].startsWith('|')) { + continue; + } + const match = lines[i].match(TABLE_ROW); + if (match == null) { + continue; + } + const name = cleanFieldName(match[1]); + if (name != null) { + pageFields.add(name); + } + } + const isAdmin = relative.startsWith('admin-api/'); + const spec = isAdmin ? adminSpec : mainSpec; + const index = isAdmin ? adminIndex : mainIndex; + + const sections: Array<{start: number; end: number}> = []; + let current = -1; + for (let i = 0; i < lines.length; i += 1) { + if (lines[i].startsWith('## ') && !lines[i].startsWith('### ')) { + if (current !== -1) { + sections.push({start: current, end: i}); + } + current = i; + } + } + if (current !== -1) { + sections.push({start: current, end: lines.length}); + } + + for (const section of sections) { + let header: RegExpMatchArray | null = null; + for (let i = section.start; i < section.end; i += 1) { + const match = lines[i].match(ROUTE_HEADER); + if (match != null) { + header = match; + break; + } + } + if (header == null) { + continue; + } + const key = shape(header[1], stripVersion(header[2])); + const operation = index.get(key); + if (operation == null) { + continue; + } + + const referenced = new Set(); + for (let i = section.start; i < section.end; i += 1) { + for (const link of lines[i].matchAll(/\]\(([^)\s]*)#([a-z0-9-]+)\)/gu)) { + const targetPage = link[1].replace(/^\//u, '').replace(/\/$/u, ''); + if (targetPage.length === 0) { + referenced.add(`${relative.replace(/\.(mdx|md)$/u, '').replace(/\/index$/u, '')}#${link[2]}`); + continue; + } + referenced.add(`${targetPage}#${link[2]}`); + } + } + const referencedFields = new Set(); + for (const anchor of referenced) { + for (const field of anchorFields.get(anchor) ?? []) { + referencedFields.add(field); + } + } + + const successResponse = Object.entries(operation.responses ?? {}).find(([status]) => status.startsWith('2')); + if (successResponse != null) { + const responseSchema = successResponse[1].content?.['application/json']?.schema; + const resolvedResponse = resolveRef(spec, responseSchema); + const target = resolvedResponse?.type === 'array' ? resolvedResponse.items : resolvedResponse; + const responseProperties = collectProperties(spec, target); + const resolvedTarget = resolveRef(spec, target); + const responseIsUnion = + resolvedTarget != null && ((resolvedTarget.oneOf ?? []).length > 0 || (resolvedTarget.anyOf ?? []).length > 0); + if (!STREAMED_RESPONSE_ROUTES.has(key) && !EMPTY_RESPONSE_ROUTES.has(key)) { + const referencedTypes = new Map(); + for (const anchor of referenced) { + for (const [field, type] of anchorTypes.get(anchor) ?? []) { + if (!referencedTypes.has(field)) { + referencedTypes.set(field, type); + } + } + } + for (const [field, specType] of collectPropertyTypes(spec, target)) { + const docType = referencedTypes.get(field); + if (docType == null) { + continue; + } + typesCompared += 1; + if (specType === docType) { + continue; + } + if (specType === 'number' && docType === 'integer') { + continue; + } + mismatches.push({ + page: relative, + operation: key, + kind: 'type-mismatch', + detail: `${field}: documented ${docType}, response schema ${specType}`, + }); + } + } + if ( + responseProperties.size > 0 && + !responseIsUnion && + !STREAMED_RESPONSE_ROUTES.has(key) && + !EMPTY_RESPONSE_ROUTES.has(key) + ) { + responsesChecked += 1; + for (const field of responseProperties) { + if (pageFields.has(field) || referencedFields.has(field)) { + responseFieldsFound += 1; + continue; + } + mismatches.push({page: relative, operation: key, kind: 'response-missing', detail: field}); + } + } + } + + for (let i = section.start; i < section.end; i += 1) { + const heading = lines[i].trim(); + if (heading === '### JSON body') { + const documented = tableFields(lines, i + 1); + const content = operation.requestBody?.content ?? {}; + const jsonSchema = content['application/json']?.schema; + if (jsonSchema == null) { + continue; + } + const actual = collectProperties(spec, jsonSchema); + if (actual.size === 0) { + continue; + } + if (sectionIsByReference(lines, i + 1)) { + documentedByReference += 1; + continue; + } + checkedBodies += 1; + const documentedTypes = tableFieldTypes(lines, i + 1); + const actualTypes = collectPropertyTypes(spec, jsonSchema); + for (const [field, docType] of documentedTypes) { + const specType = actualTypes.get(field); + if (specType == null) { + continue; + } + typesCompared += 1; + if (specType === docType) { + continue; + } + if (specType === 'number' && docType === 'integer') { + continue; + } + mismatches.push({ + page: relative, + operation: key, + kind: 'type-mismatch', + detail: `${field}: documented ${docType}, schema ${specType}`, + }); + } + const documentedOptional = tableOptionality(lines, i + 1); + const requiredFields = collectRequired(spec, jsonSchema); + const bodyProperties = collectProperties(spec, jsonSchema); + for (const [field, isOptional] of documentedOptional) { + if (!bodyProperties.has(field)) { + continue; + } + const specRequired = requiredFields.has(field); + optionalityCompared += 1; + if (specRequired === !isOptional) { + continue; + } + optionalityAdvisories.push( + specRequired + ? `${relative} ${key} ${field}: documented optional, schema marks it required` + : `${relative} ${key} ${field}: documented required, schema marks it optional`, + ); + } + const merged = MERGED_SCHEMA_ROUTES.has(key); + for (const field of documented) { + if (actual.has(field)) { + continue; + } + if (merged && SUDO_MERGE_FIELDS.includes(field)) { + continue; + } + mismatches.push({page: relative, operation: key, kind: 'body-extra', detail: field}); + } + const resolvedBody = resolveRef(spec, jsonSchema); + const isUnion = + resolvedBody != null && ((resolvedBody.oneOf ?? []).length > 0 || (resolvedBody.anyOf ?? []).length > 0); + if (isUnion) { + unionBodiesSkipped += 1; + continue; + } + for (const field of actual) { + if (documented.has(field)) { + continue; + } + if (pageFields.has(field)) { + referencedElsewhere += 1; + continue; + } + mismatches.push({page: relative, operation: key, kind: 'body-missing', detail: field}); + } + } + if (heading === '### Query parameters') { + const documented = tableFields(lines, i + 1); + const actual = new Set((operation.parameters ?? []).filter((p) => p.in === 'query').map((p) => p.name)); + if (actual.size === 0) { + continue; + } + checkedQueries += 1; + for (const field of documented) { + if (!actual.has(field)) { + mismatches.push({page: relative, operation: key, kind: 'query-extra', detail: field}); + } + } + for (const field of actual) { + if (documented.has(field)) { + continue; + } + if (pageFields.has(field)) { + referencedElsewhere += 1; + continue; + } + mismatches.push({page: relative, operation: key, kind: 'query-missing', detail: field}); + } + } + } + } +} + +if (process.env.FLUXER_DOCS_SCHEMA_JSON != null) { + const grouped = new Map>(); + for (const m of mismatches) { + const list = grouped.get(m.page) ?? []; + list.push(m); + grouped.set(m.page, list); + } + const payload: Record> = {}; + for (const [page, list] of grouped) { + payload[page] = list.map((m) => ({operation: m.operation, kind: m.kind, field: m.detail})); + } + const {writeFile} = await import('node:fs/promises'); + await writeFile(process.env.FLUXER_DOCS_SCHEMA_JSON, JSON.stringify(payload, null, 1)); + console.log(`wrote ${process.env.FLUXER_DOCS_SCHEMA_JSON}`); +} + +const byKind = new Map(); +for (const m of mismatches) { + byKind.set(m.kind, (byKind.get(m.kind) ?? 0) + 1); +} + +const authSource = await readFile(path.join(REPO_ROOT, 'fluxer_api/src/api/auth/AuthController.ts'), 'utf8'); +const guildSource = await readFile( + path.join(REPO_ROOT, 'fluxer_api/src/api/guild/controllers/GuildMemberController.ts'), + 'utf8', +); +const adminSystemSource = await readFile( + path.join(REPO_ROOT, 'fluxer_api/src/api/admin/controllers/SystemAdminController.ts'), + 'utf8', +); +const staleExemptions: Array = []; +for (const [route, anchor] of STREAMED_RESPONSE_ROUTES) { + if (!adminSystemSource.includes(anchor)) { + staleExemptions.push(`${route}: no longer streams a file, drop this exemption`); + } +} +const adminUserSecuritySource = await readFile( + path.join(REPO_ROOT, 'fluxer_api/src/api/admin/services/AdminUserSecurityService.ts'), + 'utf8', +); +for (const [route, marker] of EMPTY_RESPONSE_ROUTES) { + if (adminUserSecuritySource.includes(marker)) { + staleExemptions.push(`${route}: the handler now emits ${marker}, drop this exemption`); + } +} +for (const [route, anchor] of MERGED_SCHEMA_ROUTES) { + const present = authSource.includes(anchor) || guildSource.includes(anchor); + if (!present) { + staleExemptions.push(`${route}: the .merge() call is gone, drop this exemption`); + } +} +if (staleExemptions.length > 0) { + console.log('stale exemptions:'); + for (const entry of staleExemptions) { + console.log(` ${entry}`); + } +} + +console.log(`request body tables checked: ${checkedBodies.toString()}`); +console.log(`union bodies skipped for the missing check: ${unionBodiesSkipped.toString()}`); +console.log( + `fields documented in a shared object section rather than the route table: ${referencedElsewhere.toString()}`, +); +console.log(`bodies documented by reference to an object section: ${documentedByReference.toString()}`); +console.log( + `merged-schema exemptions active: ${MERGED_SCHEMA_ROUTES.size.toString()} (the OpenAPI generator drops .merge() operands)`, +); +console.log(`query parameter tables checked: ${checkedQueries.toString()}`); +console.log(`success response schemas checked: ${responsesChecked.toString()}`); +console.log( + `streamed-response exemptions active: ${STREAMED_RESPONSE_ROUTES.size.toString()} (the spec declares JSON, the implementation streams a file)`, +); +console.log( + `empty-response exemptions active: ${EMPTY_RESPONSE_ROUTES.size.toString()} (the spec declares a body, the handler returns none)`, +); +console.log(`response fields found documented on the page: ${responseFieldsFound.toString()}`); +console.log(`request body field types compared: ${typesCompared.toString()}`); +console.log(`request body optionality compared: ${optionalityCompared.toString()}`); +console.log(`optionality advisories: ${optionalityAdvisories.length.toString()}`); +if (optionalityAdvisories.length > 0) { + console.log(' ADVISORY ONLY, this does not fail the run.'); + console.log(' Both current lines were adjudicated by hand. ZERO documentation bugs: in each'); + console.log(' case the page is MORE precise than a bare ? marker can be.'); + console.log(' POST /stripe/checkout/subscription/preapproval requires country_code in the'); + console.log(' handler (StripeCheckoutService.ts:322 throws when it is absent) while the'); + console.log(' shared CreateCheckoutSessionRequest requires only price_id.'); + console.log(' PATCH /admin/discovery/applications/{guild_id} takes a discriminated union.'); + console.log(' reason is required when status is rejected and optional when it is approved,'); + console.log(' which the page states in a footnote instead of flattening it to ?.'); + console.log(' Re-adjudicate only if this list changes.'); + for (const entry of optionalityAdvisories) { + console.log(` ${entry}`); + } +} +for (const [kind, count] of [...byKind.entries()].sort()) { + console.log(` ${kind}: ${count.toString()}`); +} +if (mismatches.length > 0) { + console.log(''); + for (const m of mismatches.slice(0, 120)) { + let label = 'in the schema but undocumented'; + if (m.kind.endsWith('extra')) { + label = 'documented but not in the schema'; + } + if (m.kind === 'type-mismatch') { + label = 'type disagreement'; + } + if (m.kind === 'optionality') { + label = 'optionality disagreement'; + } + console.log(`${m.page} ${m.operation} ${label}: ${m.detail}`); + } + if (mismatches.length > 120) { + console.log(`... and ${(mismatches.length - 120).toString()} more`); + } + console.error(`FAIL: ${mismatches.length.toString()} field mismatches`); + process.exit(1); +} +console.log('OK: every documented request body and query table matches the live schema'); diff --git a/fluxer_docs/scripts/VerifyDocsSidebar.ts b/fluxer_docs/scripts/VerifyDocsSidebar.ts new file mode 100644 index 000000000..5dadfad11 --- /dev/null +++ b/fluxer_docs/scripts/VerifyDocsSidebar.ts @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {readdir, readFile} from 'node:fs/promises'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +const DOCS_ROOT = fileURLToPath(new URL('../src/content/docs/', import.meta.url)); +const ASTRO_CONFIG = fileURLToPath(new URL('../astro.config.ts', import.meta.url)); + +const SIDEBAR_START = 'sidebar: ['; +const QUOTED_ENTRY = /'([^']+)'/gu; +const LINK_ENTRY = /link:\s*'([^']+)'/gu; + +async function walk(directory: string): Promise> { + const entries = await readdir(directory, {withFileTypes: true}); + const files: Array = []; + for (const entry of entries) { + const resolved = path.join(directory, entry.name); + if (entry.isDirectory()) { + files.push(...(await walk(resolved))); + continue; + } + if (entry.name.endsWith('.mdx') || entry.name.endsWith('.md')) { + files.push(resolved); + } + } + return files; +} + +function slugOf(file: string): string { + const relative = path.relative(DOCS_ROOT, file); + const withoutExtension = relative.replace(/\.(mdx|md)$/u, ''); + if (withoutExtension === 'index') { + return ''; + } + return withoutExtension.replace(/\/index$/u, ''); +} + +function sidebarBlock(source: string): string { + const start = source.indexOf(SIDEBAR_START); + if (start === -1) { + throw new Error('sidebar block not found in astro.config.ts'); + } + let depth = 0; + for (let cursor = start + SIDEBAR_START.length - 1; cursor < source.length; cursor += 1) { + const character = source[cursor]; + if (character === '[') { + depth += 1; + } + if (character === ']') { + depth -= 1; + if (depth === 0) { + return source.slice(start, cursor + 1); + } + } + } + throw new Error('sidebar block is unbalanced'); +} + +const config = await readFile(ASTRO_CONFIG, 'utf8'); +const block = sidebarBlock(config); + +const labels = new Set(); +for (const match of block.matchAll(/label:\s*'([^']+)'/gu)) { + labels.add(match[1]); +} + +const linkSlugs = new Set(); +for (const match of block.matchAll(LINK_ENTRY)) { + linkSlugs.add(match[1].replace(/^\//u, '').replace(/\/$/u, '')); +} + +const referenced = new Set(linkSlugs); +for (const match of block.matchAll(QUOTED_ENTRY)) { + const value = match[1]; + if (labels.has(value)) { + continue; + } + if (value.startsWith('/')) { + continue; + } + if (!value.includes('/') && !value.match(/^[a-z0-9-]+$/u)) { + continue; + } + referenced.add(value); +} + +const files = await walk(DOCS_ROOT); +const actual = new Set(files.map(slugOf)); + +const missingPages = [...referenced].filter((slug) => !actual.has(slug)).sort(); +const orphanPages = [...actual].filter((slug) => !referenced.has(slug)).sort(); + +console.log(`sidebar entries: ${referenced.size.toString()}`); +console.log(`content pages: ${actual.size.toString()}`); + +let failures = 0; +if (missingPages.length > 0) { + console.log(''); + console.log(`sidebar references a page that does not exist: ${missingPages.length.toString()}`); + for (const slug of missingPages) { + console.log(` ${slug}`); + } + failures += missingPages.length; +} +if (orphanPages.length > 0) { + console.log(''); + console.log(`page exists but is not in the sidebar: ${orphanPages.length.toString()}`); + for (const slug of orphanPages) { + console.log(` ${slug === '' ? '(index)' : slug}`); + } + failures += orphanPages.length; +} + +const INTERNAL_LINK = /\]\((\/[^)\s#]*)(#[^)\s]*)?\)/gu; +const SAME_PAGE_LINK = /\]\((#[^)\s]*)\)/gu; +const HEADING = /^#{2,6}\s+(.+?)\s*$/u; +const EXPLICIT_ANCHOR = /<[a-z]+\s+id=["']([^"']+)["']/gu; +const dangling: Array = []; +const anchorable = new Set(actual); + +function slugify(heading: string): string { + return heading + .replace(/`/gu, '') + .replace(/\[([^\]]*)\]\([^)]*\)/gu, '$1') + .replace(/<[^>]*>/gu, '') + .toLowerCase() + .replace(/[^a-z0-9\s-]/gu, '') + .trim() + .replace(/\s+/gu, '-'); +} + +const anchorsByPage = new Map>(); +const ambiguousByPage = new Map>(); +for (const file of files) { + const slug = slugOf(file); + const source = await readFile(file, 'utf8'); + const anchors = new Set(); + const counts = new Map(); + for (const line of source.split('\n')) { + const heading = line.match(HEADING); + if (heading != null) { + const generated = slugify(heading[1]); + anchors.add(generated); + counts.set(generated, (counts.get(generated) ?? 0) + 1); + } + } + for (const explicit of source.matchAll(EXPLICIT_ANCHOR)) { + anchors.add(explicit[1]); + } + anchorsByPage.set(slug, anchors); + const ambiguous = new Set(); + for (const [generated, count] of counts) { + if (count > 1 && !source.includes(`id="${generated}"`)) { + ambiguous.add(generated); + } + } + ambiguousByPage.set(slug, ambiguous); +} + +for (const file of files) { + const relative = path.relative(DOCS_ROOT, file); + const selfSlug = slugOf(file); + const source = await readFile(file, 'utf8'); + for (const match of source.matchAll(INTERNAL_LINK)) { + const target = match[1].replace(/^\//u, '').replace(/\/$/u, ''); + if (target.length === 0) { + if (match[2] != null) { + const fragment = match[2].slice(1); + const rootAnchors = anchorsByPage.get(''); + if (rootAnchors != null && fragment.length > 0 && !rootAnchors.has(fragment)) { + dangling.push(`${relative} -> /#${fragment} (anchor)`); + } + } + continue; + } + if (!anchorable.has(target)) { + dangling.push(`${relative} -> /${target}/`); + continue; + } + if (match[2] != null) { + const fragment = match[2].slice(1); + const anchors = anchorsByPage.get(target); + if (fragment.length > 0 && anchors != null && !anchors.has(fragment)) { + dangling.push(`${relative} -> /${target}/#${fragment} (anchor)`); + } else if (fragment.length > 0 && (ambiguousByPage.get(target)?.has(fragment) ?? false)) { + dangling.push(`${relative} -> /${target}/#${fragment} (that heading repeats on the target page)`); + } + } + } + for (const match of source.matchAll(SAME_PAGE_LINK)) { + const fragment = match[1].slice(1); + const anchors = anchorsByPage.get(selfSlug); + if (fragment.length > 0 && anchors != null && !anchors.has(fragment)) { + dangling.push(`${relative} -> #${fragment} (same-page anchor)`); + } else if (fragment.length > 0 && (ambiguousByPage.get(selfSlug)?.has(fragment) ?? false)) { + dangling.push(`${relative} -> #${fragment} (that heading repeats on this page)`); + } + } +} +if (dangling.length > 0) { + console.log(''); + console.log(`links to a page that does not exist: ${dangling.length.toString()}`); + for (const entry of [...new Set(dangling)].sort()) { + console.log(` ${entry}`); + } + failures += dangling.length; +} + +if (failures > 0) { + console.error(`FAIL: ${failures.toString()} structure problems`); + process.exit(1); +} +console.log('OK: sidebar, content and internal links are in sync'); diff --git a/fluxer_docs/scripts/VerifyDocsStyle.ts b/fluxer_docs/scripts/VerifyDocsStyle.ts new file mode 100644 index 000000000..349663d91 --- /dev/null +++ b/fluxer_docs/scripts/VerifyDocsStyle.ts @@ -0,0 +1,585 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {readdir, readFile} from 'node:fs/promises'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import { + columnWidthPercents, + extractTables, + PX, + TABLE_BUDGET_PX, + TABLE_FLOOR_CAP_SHARE, + TABLE_MAX_CELL_CHARS, + TABLE_MAX_COLUMNS, + TABLE_MAX_IDENT_CHARS, + TABLE_WIDE_TIER_PX, +} from './DocsTableWidth.ts'; + +const DOCS_ROOT = fileURLToPath(new URL('../src/content/docs/', import.meta.url)); +const STYLES_ROOT = fileURLToPath(new URL('../src/styles/', import.meta.url)); +const STARLIGHT_STYLES = fileURLToPath(new URL('../node_modules/@astrojs/starlight/style/', import.meta.url)); + +const FORBIDDEN_MARKETING_WORDS = [ + 'seamless', + 'effortless', + 'blazing', + 'cutting-edge', + 'game-changing', + 'powerful and', + 'just simply', + 'simply use', + 'simply call', + 'simply add', + 'simply set', +]; + +const FORBIDDEN_TOPICS = [ + {pattern: /mobile[- ]device/iu, reason: 'mobile notifications API does not exist in the live era'}, + {pattern: /push subscription/iu, reason: 'push API does not exist in the live era'}, + {pattern: /\/push\/events/u, reason: 'push events API does not exist in the live era'}, + {pattern: /voice[- ]public[- ]key/iu, reason: 'voice connection API does not exist in the live era'}, + {pattern: /match-public-key/u, reason: 'voice connection API does not exist in the live era'}, +]; + +const CANONICAL_HEADINGS = new Map([ + ['### JSON params', '### JSON body'], + ['### Query string parameters', '### Query parameters'], + ['### Response Body', '### Response body'], + ['### JSON Body', '### JSON body'], + ['### Query Parameters', '### Query parameters'], +]); + +interface Finding { + readonly file: string; + readonly line: number; + readonly rule: string; + readonly detail: string; +} + +async function walk(directory: string): Promise> { + const entries = await readdir(directory, {withFileTypes: true}); + const files: Array = []; + for (const entry of entries) { + const resolved = path.join(directory, entry.name); + if (entry.isDirectory()) { + files.push(...(await walk(resolved))); + continue; + } + if (entry.name.endsWith('.mdx') || entry.name.endsWith('.md')) { + files.push(resolved); + } + } + return files; +} + +function frontmatterOf(source: string): string | null { + if (!source.startsWith('---\n')) { + return null; + } + const end = source.indexOf('\n---', 4); + if (end === -1) { + return null; + } + return source.slice(4, end); +} + +function insideFence(lines: ReadonlyArray, index: number): boolean { + let fenced = false; + for (let cursor = 0; cursor < index; cursor += 1) { + if (lines[cursor].startsWith('```')) { + fenced = !fenced; + } + } + return fenced; +} + +const files = await walk(DOCS_ROOT); +const findings: Array = []; + +for (const file of files) { + const relative = path.relative(DOCS_ROOT, file); + const source = await readFile(file, 'utf8'); + const lines = source.split('\n'); + const frontmatter = frontmatterOf(source); + + if (frontmatter == null) { + findings.push({file: relative, line: 1, rule: 'frontmatter', detail: 'missing frontmatter block'}); + } else { + if (!/^# SPDX-License-Identifier: AGPL-3\.0-or-later$/mu.test(frontmatter)) { + findings.push({file: relative, line: 2, rule: 'spdx', detail: 'missing SPDX comment in frontmatter'}); + } + if (!/^title:\s*\S/mu.test(frontmatter)) { + findings.push({file: relative, line: 1, rule: 'frontmatter', detail: 'missing title'}); + } + if (!/^description:\s*\S/mu.test(frontmatter)) { + findings.push({file: relative, line: 1, rule: 'frontmatter', detail: 'missing description'}); + } + } + + const usesRouteHeader = source.includes(' = []; + for (let index = 0; index < lines.length; index += 1) { + if (lines[index].startsWith('## ') && !lines[index].startsWith('### ')) { + sectionStarts.push(index); + } + } + sectionStarts.push(lines.length); + for (let s = 0; s + 1 < sectionStarts.length; s += 1) { + let responseBody = -1; + let response = -1; + for (let index = sectionStarts[s]; index < sectionStarts[s + 1]; index += 1) { + const heading = lines[index].trim(); + if (heading === '### Response body' && responseBody === -1) { + responseBody = index; + } + if (heading === '### Response' && response === -1) { + response = index; + } + } + if (responseBody !== -1 && response !== -1 && responseBody > response) { + findings.push({ + file: relative, + line: response + 1, + rule: 'order', + detail: '"### Response body" must precede "### Response" (conventions.md)', + }); + } + } +} + +for (const file of files) { + const relative = path.relative(DOCS_ROOT, file); + const lines = (await readFile(file, 'utf8')).split('\n'); + let block = new Map(); + let sinceFootnote = 0; + for (let index = 0; index < lines.length; index += 1) { + const definition = lines[index].match(/^(\d+)<\/sup>\s/u); + if (definition == null) { + if (lines[index].trim().length > 0) { + sinceFootnote += 1; + } + if (sinceFootnote > 1) { + block = new Map(); + } + continue; + } + sinceFootnote = 0; + const previous = block.get(definition[1]); + if (previous != null) { + findings.push({ + file: relative, + line: index + 1, + rule: 'footnote', + detail: `footnote ${definition[1]} is defined twice in the same block, first at line ${previous.toString()}`, + }); + continue; + } + block.set(definition[1], index + 1); + } +} + +const NOTATION_EXAMPLE = 'A superscript marker such as 1 refers to the numbered footnote'; + +for (const file of files) { + const relative = path.relative(DOCS_ROOT, file); + const source = await readFile(file, 'utf8'); + const lines = source.split('\n'); + const explainsNotation = source.includes(NOTATION_EXAMPLE); + const bounds: Array = [0]; + for (let index = 0; index < lines.length; index += 1) { + if (lines[index].startsWith('## ') && !lines[index].startsWith('### ')) { + bounds.push(index); + } + } + bounds.push(lines.length); + for (let b = 0; b + 1 < bounds.length; b += 1) { + const definitions = new Map(); + const references = new Set(); + for (let index = bounds[b]; index < bounds[b + 1]; index += 1) { + const definition = lines[index].match(/^(\d+)<\/sup>\s/u); + if (definition != null) { + if (!definitions.has(definition[1])) { + definitions.set(definition[1], index + 1); + } + continue; + } + for (const marker of lines[index].matchAll(/(\d+)<\/sup>/gu)) { + references.add(marker[1]); + } + } + for (const [number, line] of definitions) { + if (!references.has(number)) { + findings.push({ + file: relative, + line, + rule: 'footnote', + detail: `footnote ${number} is defined but never referenced in its section`, + }); + } + } + for (const number of references) { + if (definitions.has(number)) { + continue; + } + if (explainsNotation) { + continue; + } + findings.push({ + file: relative, + line: bounds[b] + 1, + rule: 'footnote', + detail: `footnote ${number} is referenced but never defined in its section`, + }); + } + } +} + +type TableRule = + | 'table-columns' + | 'table-fit' + | 'table-cell' + | 'table-identifier' + | 'table-rows' + | 'table-parallel' + | 'table-wide-tight'; + +const TABLE_RULES: ReadonlyArray = [ + 'table-columns', + 'table-fit', + 'table-cell', + 'table-identifier', + 'table-rows', + 'table-parallel', + 'table-wide-tight', +]; + +const TABLE_COUNT_FLOOR = 1800; + +const ACCEPTED_TABLE_FINDINGS = new Map>>>([ + ['admin-api/applications.mdx', {'table-identifier': 1}], + ['admin-api/bulk-jobs.mdx', {'table-fit': 1}], + ['admin-api/discovery.mdx', {'table-identifier': 1}], + ['admin-api/guilds.mdx', {'table-identifier': 3}], + ['admin-api/index.mdx', {'table-fit': 1, 'table-identifier': 2}], + ['admin-api/instance.mdx', {'table-identifier': 5}], + ['admin-api/messages.mdx', {'table-identifier': 1}], + ['admin-api/reports.mdx', {'table-fit': 1, 'table-identifier': 2}], + ['admin-api/users.mdx', {'table-fit': 1, 'table-identifier': 1}], + ['admin-api/voice.mdx', {'table-identifier': 3}], + ['authentication.md', {'table-cell': 2, 'table-identifier': 1}], + ['conventions.md', {'table-cell': 1, 'table-parallel': 1}], + ['gateway/event-filtering.md', {'table-cell': 2}], + ['gateway/events.md', {'table-identifier': 1}], + ['gateway/opcodes-and-close-codes.md', {'table-cell': 1}], + ['gateway/overview.md', {'table-cell': 1}], + ['http-api/authentication.mdx', {'table-identifier': 1}], + ['http-api/billing.mdx', {'table-identifier': 5}], + ['http-api/calls.mdx', {'table-fit': 1, 'table-cell': 3}], + ['http-api/channels.mdx', {'table-cell': 6}], + ['http-api/connections.mdx', {'table-fit': 1, 'table-cell': 4, 'table-identifier': 1}], + ['http-api/deployment-availability.md', {'table-fit': 1}], + ['http-api/discovery.mdx', {'table-cell': 3}], + ['http-api/donations.mdx', {'table-cell': 2}], + ['http-api/entrance-sounds.mdx', {'table-cell': 3, 'table-parallel': 1}], + ['http-api/gifs.mdx', {'table-cell': 5}], + ['http-api/gifts.mdx', {'table-cell': 1}], + ['http-api/guild-audit-logs.mdx', {'table-identifier': 3}], + ['http-api/guild-channels.mdx', {'table-cell': 2}], + ['http-api/guild-emojis.mdx', {'table-cell': 4}], + ['http-api/guild-members.mdx', {'table-identifier': 2}], + ['http-api/guild-moderation.mdx', {'table-cell': 1}], + ['http-api/guild-stickers.mdx', {'table-cell': 3}], + ['http-api/guilds.mdx', {'table-fit': 1, 'table-identifier': 4}], + ['http-api/instance.mdx', {'table-identifier': 4}], + ['http-api/invites.mdx', {'table-cell': 6}], + ['http-api/messages.mdx', {'table-fit': 1, 'table-cell': 20}], + ['http-api/permissions.mdx', {'table-cell': 8}], + ['http-api/premium.mdx', {'table-identifier': 5}], + ['http-api/read-states.mdx', {'table-cell': 2}], + ['http-api/reports.mdx', {'table-cell': 5, 'table-identifier': 1}], + ['http-api/search.mdx', {'table-fit': 1, 'table-cell': 2, 'table-identifier': 1}], + ['http-api/streams.mdx', {'table-cell': 5}], + ['http-api/unfurl.mdx', {'table-cell': 2, 'table-parallel': 1}], + ['http-api/users.mdx', {'table-fit': 1, 'table-identifier': 4}], + ['http-api/users/content.mdx', {'table-cell': 4}], + ['http-api/users/current-user.mdx', {'table-cell': 1, 'table-identifier': 1}], + ['http-api/users/data-harvest.mdx', {'table-cell': 2}], + ['http-api/users/email-and-password.mdx', {'table-identifier': 1}], + ['http-api/users/mfa.mdx', {'table-cell': 3, 'table-parallel': 1}], + ['http-api/users/phone-verification.mdx', {'table-cell': 2}], + ['http-api/users/private-channels.mdx', {'table-cell': 3}], + ['http-api/users/relationships.mdx', {'table-fit': 2, 'table-cell': 3, 'table-identifier': 2}], + ['http-api/users/settings-protobuf.md', {'table-fit': 2, 'table-identifier': 7, 'table-parallel': 1}], + ['http-api/users/settings.mdx', {'table-fit': 1, 'table-cell': 2, 'table-identifier': 1, 'table-parallel': 1}], + ['http-api/webhooks.mdx', {'table-identifier': 1, 'table-parallel': 1}], + ['media-proxy/overview.md', {'table-parallel': 1}], + ['media-proxy/responses-and-limits.md', {'table-cell': 2}], + ['media-proxy/routes.mdx', {'table-cell': 1}], + ['media-proxy/transformations.md', {'table-parallel': 1}], + ['topics/uploads.md', {'table-fit': 1, 'table-identifier': 1}], + ['voice/index.md', {'table-parallel': 1}], +]); + +const tableFindingsByPage = new Map>>(); +let tablesMeasured = 0; +const overWideTier: Array = []; + +for (const file of files) { + const relative = path.relative(DOCS_ROOT, file); + const source = await readFile(file, 'utf8'); + for (const table of extractTables(source)) { + tablesMeasured += 1; + const raise = (rule: TableRule, detail: string): void => { + const perRule = tableFindingsByPage.get(relative) ?? new Map>(); + const list = perRule.get(rule) ?? []; + list.push({file: relative, line: table.line, rule, detail}); + perRule.set(rule, list); + tableFindingsByPage.set(relative, perRule); + }; + const shape = table.header.join(' | '); + if (table.columns > TABLE_MAX_COLUMNS) { + raise( + 'table-columns', + `[${shape}] has ${table.columns.toString()} columns, at most ${TABLE_MAX_COLUMNS.toString()} are allowed`, + ); + } + if (table.minDemandPx > TABLE_BUDGET_PX) { + raise( + 'table-fit', + `[${shape}] demands ${table.minDemandPx.toString()}px against the ${TABLE_BUDGET_PX.toString()}px budget, columns ${table.columnMinPx.join('/')}px`, + ); + } + if (table.minDemandPx > TABLE_WIDE_TIER_PX) { + overWideTier.push({ + file: relative, + line: table.line, + rule: 'table-wide-tier', + detail: `[${shape}] demands ${table.minDemandPx.toString()}px, over the ${TABLE_WIDE_TIER_PX.toString()}px width where the wide CSS tier switches to content sizing, so this table overflows its column instead of wrapping`, + }); + } + if (table.worstCellChars > TABLE_MAX_CELL_CHARS) { + raise( + 'table-cell', + `[${shape}] column ${(table.worstCellColumn + 1).toString()} holds ${table.worstCellChars.toString()} rendered characters, over ${TABLE_MAX_CELL_CHARS.toString()}: "${table.worstCellText.slice(0, 60)}..."`, + ); + } + if (table.col1IdentChars > TABLE_MAX_IDENT_CHARS) { + raise( + 'table-identifier', + `[${shape}] first column holds "${table.col1IdentText}", ${table.col1IdentChars.toString()} characters, over ${TABLE_MAX_IDENT_CHARS.toString()}`, + ); + } + if (table.rows < 2 && !table.canonical) { + raise( + 'table-rows', + `[${shape}] has ${table.rows.toString()} body row(s) and is not a canonical schema shape, so it is a sentence`, + ); + } + if (!table.parallel) { + raise('table-parallel', `[${shape}] ${table.nonParallelReason}`); + } + const shares = columnWidthPercents( + table.columnMinPx.map((minPx, index) => ({minPx, maxPx: table.columnMaxPx[index]})), + TABLE_WIDE_TIER_PX, + ); + const floorCap = TABLE_WIDE_TIER_PX * TABLE_FLOOR_CAP_SHARE; + for (const [index, share] of shares.entries()) { + const allotted = (share / 100) * TABLE_WIDE_TIER_PX; + const unbreakable = table.columnMinPx[index] + PX.cellPad; + if (unbreakable <= floorCap && allotted + PX.proseFallback < unbreakable) { + raise( + 'table-wide-tight', + `[${shape}] column ${(index + 1).toString()} is allotted ${Math.round(allotted).toString()}px but its longest unbreakable word needs ${Math.round(unbreakable).toString()}px, so that word breaks mid-word even at the widest tier`, + ); + } + } + } +} + +if (tablesMeasured < TABLE_COUNT_FLOOR) { + findings.push({ + file: 'scripts/DocsTableWidth.ts', + line: 1, + rule: 'table-extractor', + detail: `only ${tablesMeasured.toString()} tables were measured, floor is ${TABLE_COUNT_FLOOR.toString()}. The table rules have gone blind`, + }); +} + +findings.push(...overWideTier); + +const acceptedByRule = new Map(); +for (const [relative, perRule] of tableFindingsByPage) { + const allowance = ACCEPTED_TABLE_FINDINGS.get(relative) ?? {}; + for (const [rule, list] of perRule) { + const allowed = allowance[rule] ?? 0; + acceptedByRule.set(rule, (acceptedByRule.get(rule) ?? 0) + Math.min(allowed, list.length)); + for (const finding of list.slice(allowed)) { + findings.push(finding); + } + } +} +for (const [relative, allowance] of ACCEPTED_TABLE_FINDINGS) { + const perRule = tableFindingsByPage.get(relative); + for (const rule of TABLE_RULES) { + const allowed = allowance[rule] ?? 0; + const actual = perRule?.get(rule)?.length ?? 0; + if (allowed > actual) { + findings.push({ + file: relative, + line: 1, + rule: 'table-ledger', + detail: `the ledger allows ${allowed.toString()} ${rule} findings but the page now produces ${actual.toString()}. Lower the entry in VerifyDocsStyle.ts`, + }); + } + } +} + +interface CssBlock { + readonly selector: string; + readonly body: string; + readonly line: number; +} + +function cssBlocks(source: string): Array { + const blocks: Array = []; + const pattern = /([^{}]+)\{([^{}]*)\}/gu; + let match: RegExpExecArray | null = pattern.exec(source); + while (match != null) { + blocks.push({ + selector: match[1] + .replace(/\/\*[\s\S]*?\*\//gu, ' ') + .replace(/\s+/gu, ' ') + .trim(), + body: match[2], + line: source.slice(0, match.index).split('\n').length, + }); + match = pattern.exec(source); + } + return blocks; +} + +const SIDEWAYS_SCROLL = /(?:^|;)\s*overflow(?:-x)?\s*:\s*(?:auto|scroll)\s*(?:;|$)/mu; +const SCROLL_FORBIDDEN_SELECTOR = + /\btable\b|\bthead\b|\btbody\b|\btr\b|\.sl-markdown-content\s*(?:,|$)|table-wrapper|overflow-wrapper/u; + +for (const entry of await readdir(STYLES_ROOT, {withFileTypes: true})) { + if (!entry.name.endsWith('.css')) { + continue; + } + const source = await readFile(path.join(STYLES_ROOT, entry.name), 'utf8'); + for (const block of cssBlocks(source)) { + if (!SCROLL_FORBIDDEN_SELECTOR.test(block.selector) || !SIDEWAYS_SCROLL.test(block.body)) { + continue; + } + findings.push({ + file: `src/styles/${entry.name}`, + line: block.line, + rule: 'table-scroll', + detail: `"${block.selector}" makes a table or the markdown container a sideways scroller. No table may ever scroll horizontally`, + }); + } +} + +{ + const starlightTableScrollers: Array = []; + for (const entry of await readdir(STARLIGHT_STYLES, {withFileTypes: true})) { + if (!entry.name.endsWith('.css')) { + continue; + } + const source = await readFile(path.join(STARLIGHT_STYLES, entry.name), 'utf8'); + for (const block of cssBlocks(source)) { + if (block.selector.includes('table') && SIDEWAYS_SCROLL.test(block.body)) { + starlightTableScrollers.push(`${entry.name}:${block.line.toString()} ${block.selector}`); + } + } + } + const content = await readFile(path.join(STYLES_ROOT, 'content.css'), 'utf8'); + const neutralised = cssBlocks(content).some( + (block) => + /\.sl-markdown-content\s+table\b/u.test(block.selector) && + /(?:^|;)\s*display\s*:\s*table\s*(?:;|$)/mu.test(block.body) && + /(?:^|;)\s*overflow\s*:\s*visible\s*(?:;|$)/mu.test(block.body), + ); + if (starlightTableScrollers.length > 0 && !neutralised) { + findings.push({ + file: 'src/styles/content.css', + line: 1, + rule: 'table-scroll', + detail: `Starlight still ships a scrolling table (${starlightTableScrollers.join(', ')}) and content.css no longer overrides it with "display: table" and "overflow: visible"`, + }); + } +} + +const byRule = new Map(); +for (const finding of findings) { + byRule.set(finding.rule, (byRule.get(finding.rule) ?? 0) + 1); +} + +console.log(`pages checked: ${files.length.toString()}`); +console.log(`tables measured: ${tablesMeasured.toString()}`); + +for (const rule of TABLE_RULES) { + console.log( + ` ${rule}: ${(byRule.get(rule) ?? 0).toString()} new, ${(acceptedByRule.get(rule) ?? 0).toString()} accepted`, + ); +} +for (const [rule, count] of [...byRule.entries()].sort()) { + if ((TABLE_RULES as ReadonlyArray).includes(rule)) { + continue; + } + console.log(` ${rule}: ${count.toString()}`); +} +if (findings.length > 0) { + console.log(''); + for (const finding of findings.slice(0, 200)) { + console.log(`${finding.file}:${finding.line.toString()} [${finding.rule}] ${finding.detail}`); + } + if (findings.length > 200) { + console.log(`... and ${(findings.length - 200).toString()} more`); + } + console.error(`FAIL: ${findings.length.toString()} style problems`); + process.exit(1); +} +console.log('OK: house style clean'); diff --git a/fluxer_docs/scripts/write_csp_header.py b/fluxer_docs/scripts/write_csp_header.py deleted file mode 100644 index 2b14ee7ec..000000000 --- a/fluxer_docs/scripts/write_csp_header.py +++ /dev/null @@ -1,94 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-or-later - -from __future__ import annotations - -import base64 -import hashlib -import html.parser -import pathlib -import sys - - -class InlineScriptParser(html.parser.HTMLParser): - def __init__(self) -> None: - super().__init__(convert_charrefs=False) - self._collecting = False - self._parts: list[str] = [] - self.scripts: list[str] = [] - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - if tag.lower() != "script": - return - if any(name.lower() == "src" for name, _ in attrs): - return - self._collecting = True - self._parts = [] - - def handle_data(self, data: str) -> None: - if self._collecting: - self._parts.append(data) - - def handle_entityref(self, name: str) -> None: - if self._collecting: - self._parts.append(f"&{name};") - - def handle_charref(self, name: str) -> None: - if self._collecting: - self._parts.append(f"&#{name};") - - def handle_endtag(self, tag: str) -> None: - if tag.lower() == "script" and self._collecting: - self.scripts.append("".join(self._parts)) - self._collecting = False - - -def script_hashes(site_dir: pathlib.Path) -> list[str]: - hashes: set[str] = set() - for path in sorted(site_dir.rglob("*.html")): - parser = InlineScriptParser() - parser.feed(path.read_text(encoding="utf-8")) - for script in parser.scripts: - digest = hashlib.sha256(script.encode("utf-8")).digest() - hashes.add("'sha256-" + base64.b64encode(digest).decode("ascii") + "'") - return sorted(hashes) - - -def main() -> None: - site_dir = pathlib.Path(sys.argv[1]) - output = pathlib.Path(sys.argv[2]) - script_sources = " ".join(script_hashes(site_dir)) - csp = "; ".join( - [ - "default-src 'self'", - f"script-src 'self' {script_sources}", - "style-src 'self' 'unsafe-inline'", - "img-src 'self' data:", - "font-src 'none'", - "connect-src 'self' https://api.github.com", - "worker-src 'self'", - "frame-src 'none'", - "object-src 'none'", - "base-uri 'self'", - "form-action 'self'", - "frame-ancestors 'none'", - "upgrade-insecure-requests", - ] - ) - output.write_text( - "\n".join( - [ - 'add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always;', - 'add_header X-Content-Type-Options "nosniff" always;', - 'add_header Referrer-Policy "strict-origin-when-cross-origin" always;', - 'add_header X-Frame-Options "DENY" always;', - 'add_header Permissions-Policy "accelerometer=(), camera=(), geolocation=(), gyroscope=(), magnetometer=(), microphone=(), payment=(), usb=()" always;', - f'add_header Content-Security-Policy "{csp}" always;', - "", - ] - ), - encoding="utf-8", - ) - - -if __name__ == "__main__": - main() diff --git a/fluxer_docs/serve.sh b/fluxer_docs/serve.sh deleted file mode 100755 index d744c9067..000000000 --- a/fluxer_docs/serve.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -VENV="$HERE/.venv" -REQUIREMENTS_STAMP="$VENV/.requirements.sha256" -ADDR="${ZENSICAL_DEV_ADDR:-0.0.0.0:8000}" -LOG="${ZENSICAL_LOG:-/tmp/zensical-serve.log}" -PORT="${ADDR##*:}" - -cd "$HERE" - -ensure_env() { - if [ ! -x "$VENV/bin/python" ]; then - python3 -m venv "$VENV" - "$VENV/bin/python" -m pip install --quiet --upgrade pip - fi - python_fingerprint="$(python3 -c 'import platform, sys; print(f"{sys.implementation.cache_tag}:{platform.machine()}")')" - requirements_hash="$(printf '%s\0%s\n' "$python_fingerprint" "$(sha256sum "$HERE/requirements.txt" | cut -d' ' -f1)" | sha256sum | cut -d' ' -f1)" - if [ ! -f "$REQUIREMENTS_STAMP" ] || [ "$(cat "$REQUIREMENTS_STAMP")" != "$requirements_hash" ]; then - "$VENV/bin/python" -m pip install --quiet --require-virtualenv -r "$HERE/requirements.txt" - printf '%s\n' "$requirements_hash" >"$REQUIREMENTS_STAMP" - fi -} - -case "${1:-serve}" in ---bootstrap) - ensure_env - ;; ---daemon) - ensure_env - if curl -sf -o /dev/null "http://127.0.0.1:${PORT}/" 2>/dev/null; then - echo "zensical already serving on ${ADDR}" - exit 0 - fi - setsid "$VENV/bin/zensical" serve -a "$ADDR" >"$LOG" 2>&1 & - disown 2>/dev/null || true - echo "zensical serving on ${ADDR} (logs: ${LOG})" - ;; -serve) - ensure_env - exec "$VENV/bin/zensical" serve -a "$ADDR" - ;; -*) - ensure_env - exec "$VENV/bin/zensical" "$@" - ;; -esac diff --git a/fluxer_docs/src/components/Head.astro b/fluxer_docs/src/components/Head.astro new file mode 100644 index 000000000..9e3e05825 --- /dev/null +++ b/fluxer_docs/src/components/Head.astro @@ -0,0 +1,19 @@ +--- +// SPDX-License-Identifier: AGPL-3.0-or-later +import Default from '@astrojs/starlight/components/Head.astro'; +import LatinFontFaces from '@/fonts/LatinFontFaces.astro'; +import {docsSearchIndexing} from '@/server/DocsConfig'; + +function resolveRobotsDirective(searchIndexingEnabled: boolean): string { + if (searchIndexingEnabled) { + return 'index,follow'; + } + return 'noindex,nofollow,noarchive,nosnippet,noimageindex'; +} + +const robots = resolveRobotsDirective(docsSearchIndexing()); +--- + + + + diff --git a/fluxer_docs/src/components/Header.astro b/fluxer_docs/src/components/Header.astro new file mode 100644 index 000000000..96d3acad8 --- /dev/null +++ b/fluxer_docs/src/components/Header.astro @@ -0,0 +1,10 @@ +--- +// SPDX-License-Identifier: AGPL-3.0-or-later +import SiteTitle from '@astrojs/starlight/components/SiteTitle.astro'; +import ThemeSelect from '@/components/ThemeSelect.astro'; +--- + + + + + diff --git a/fluxer_docs/src/components/InstallerChecksum.astro b/fluxer_docs/src/components/InstallerChecksum.astro new file mode 100644 index 000000000..d17ff486b --- /dev/null +++ b/fluxer_docs/src/components/InstallerChecksum.astro @@ -0,0 +1,19 @@ +--- +// SPDX-License-Identifier: AGPL-3.0-or-later +import {powershellInstaller, shellInstaller} from '@/installer/Installer'; + +interface Props { + script: 'install.sh' | 'install.ps1'; +} + +const {script} = Astro.props; +const installer = script === 'install.ps1' ? powershellInstaller : shellInstaller; +--- + +{installer.digest} + + diff --git a/fluxer_docs/src/components/RouteHeader.astro b/fluxer_docs/src/components/RouteHeader.astro new file mode 100644 index 000000000..5d47fd9a8 --- /dev/null +++ b/fluxer_docs/src/components/RouteHeader.astro @@ -0,0 +1,103 @@ +--- +// SPDX-License-Identifier: AGPL-3.0-or-later +interface Props { + method: 'GET' | 'HEAD' | 'POST' | 'PATCH' | 'PUT' | 'DELETE'; + path: string; + bot?: boolean; + oauth2?: string; + unauthenticated?: boolean; + auditReason?: boolean; + mfa?: boolean; +} + +interface Capability { + label: string; + title: string; + href?: string; +} + +function resolveAuditReasonHref(routePath: string): string { + if (routePath === '/v1/admin') { + return '/admin-api/#audit-reasons-and-effects'; + } + if (routePath.startsWith('/v1/admin/')) { + return '/admin-api/#audit-reasons-and-effects'; + } + return '/http-api/guild-audit-logs/#audit-log-reason'; +} + +const {method, path, bot, oauth2, unauthenticated, auditReason, mfa} = Astro.props; + +const docsBase = import.meta.env.BASE_URL.replace(/\/$/, ''); +const withBase = (href: string): string => `${docsBase}${href}`; + +const segments = path.split(/(\{[^}]+\})/).filter((segment) => segment.length > 0); + +const capabilities: Array = []; +if (unauthenticated === true) { + capabilities.push({ + label: 'Unauthenticated', + title: 'Accepts requests without an authentication credential', + href: withBase('/authentication/'), + }); +} +if (bot === true) { + capabilities.push({ + label: 'Bot', + title: 'Accepts a bot token authorisation scheme', + href: withBase('/authentication/#authorization-schemes'), + }); +} +if (oauth2 != null) { + capabilities.push({ + label: oauth2, + title: `Requires the ${oauth2} OAuth2 scope`, + href: withBase('/http-api/oauth2/#oauth2-scopes'), + }); +} +if (auditReason === true) { + capabilities.push({ + label: 'Audit reason', + title: 'Supports the X-Audit-Log-Reason request header', + href: withBase(resolveAuditReasonHref(path)), + }); +} +if (mfa === true) { + capabilities.push({ + label: 'MFA', + title: 'Requires multi-factor authentication or an elevated sudo session', + href: withBase('/http-api/users/mfa/'), + }); +} +--- + + + + {method}{ + segments.map((segment) => + segment.startsWith('{') && segment.endsWith('}') ? ( + {segment} + ) : ( + segment + ), + ) + } + + { + capabilities.length > 0 && ( + + {capabilities.map((capability) => + capability.href == null ? ( + + {capability.label} + + ) : ( + + {capability.label} + + ), + )} + + ) + } + diff --git a/fluxer_docs/src/components/Sidebar.astro b/fluxer_docs/src/components/Sidebar.astro new file mode 100644 index 000000000..cfd073e14 --- /dev/null +++ b/fluxer_docs/src/components/Sidebar.astro @@ -0,0 +1,24 @@ +--- +// SPDX-License-Identifier: AGPL-3.0-or-later +import Default from '@astrojs/starlight/components/Sidebar.astro'; +import Search from '@astrojs/starlight/components/Search.astro'; +import SiteTitle from '@astrojs/starlight/components/SiteTitle.astro'; +import SocialIcons from '@astrojs/starlight/components/SocialIcons.astro'; +import ThemeSelect from '@/components/ThemeSelect.astro'; +--- + + + + + + + + + + + + + + + + diff --git a/fluxer_docs/src/components/StarlightToc.ts b/fluxer_docs/src/components/StarlightToc.ts new file mode 100644 index 000000000..dd466d3b2 --- /dev/null +++ b/fluxer_docs/src/components/StarlightToc.ts @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +const PAGE_TITLE_ID = '_top'; + +class FluxerStarlightTOC extends HTMLElement { + private _current = this.querySelector('a[aria-current="true"]'); + private minH = Number.parseInt(this.dataset.minH || '2', 10); + private maxH = Number.parseInt(this.dataset.maxH || '3', 10); + + private tocHeadingSelector = + `h1#${PAGE_TITLE_ID},` + + `:where(${[...Array.from({length: 1 + this.maxH - this.minH}).map((_, index) => `h${this.minH + index}`)].join()})[id]`; + + protected set current(link: HTMLAnchorElement) { + if (link === this._current) { + return; + } + if (this._current) { + this._current.removeAttribute('aria-current'); + } + link.setAttribute('aria-current', 'true'); + this._current = link; + } + + private onIdle = (cb: IdleRequestCallback) => (window.requestIdleCallback || ((cb) => setTimeout(cb, 1)))(cb); + + constructor() { + super(); + this.markInitialEntry(); + this.onIdle(() => this.init()); + } + + private markInitialEntry(): void { + const links = [...this.querySelectorAll('a')]; + if (links.length === 0) { + return; + } + const navBarHeight = document.querySelector('header')?.getBoundingClientRect().height || 0; + const mobileTocHeight = this.querySelector('summary')?.getBoundingClientRect().height || 0; + const readingLine = navBarHeight + mobileTocHeight + 32; + let match: HTMLAnchorElement | undefined; + for (const heading of document.querySelectorAll(this.tocHeadingSelector)) { + if (heading.getBoundingClientRect().top > readingLine) { + break; + } + const link = links.find((candidate) => candidate.hash === `#${encodeURIComponent(heading.id)}`); + if (link) { + match = link; + } + } + this.current = match ?? links[0]; + } + + private init = (): void => { + const links = [...this.querySelectorAll('a')]; + + const isHeading = (el: Element): el is HTMLHeadingElement => el.matches(this.tocHeadingSelector); + + const getElementHeading = (el: Element | null): HTMLHeadingElement | null => { + if (!el) { + return null; + } + const origin = el; + while (el) { + if (el.matches('.sl-markdown-content, main > *')) { + return document.getElementById(PAGE_TITLE_ID) as HTMLHeadingElement; + } + if (isHeading(el)) { + return el; + } + const childHeading = el.querySelector(this.tocHeadingSelector); + if (childHeading) { + return childHeading; + } + el = el.previousElementSibling; + while (el?.lastElementChild) { + el = el.lastElementChild; + } + const h = getElementHeading(el); + if (h) { + return h; + } + } + return getElementHeading(origin.parentElement); + }; + + const setCurrent: IntersectionObserverCallback = (entries) => { + for (const {isIntersecting, target} of entries) { + if (!isIntersecting) { + continue; + } + const heading = getElementHeading(target); + if (!heading) { + continue; + } + const link = links.find((candidate) => candidate.hash === `#${encodeURIComponent(heading.id)}`); + if (link) { + this.current = link; + break; + } + } + }; + + const toObserve = document.querySelectorAll( + [ + `main :where(${this.tocHeadingSelector})`, + `main :where(${this.tocHeadingSelector}, .sl-heading-wrapper) ~ *:not(:has(${this.tocHeadingSelector}))`, + `main .sl-markdown-content > *:not(:has(${this.tocHeadingSelector}))`, + `main > *:not(:has(${this.tocHeadingSelector}))`, + ].join(), + ); + + let observer: IntersectionObserver | undefined; + const observe = () => { + if (observer) { + return; + } + observer = new IntersectionObserver(setCurrent, {rootMargin: this.getRootMargin()}); + for (const h of toObserve) { + observer.observe(h); + } + }; + observe(); + + let timeout: ReturnType; + window.addEventListener('resize', () => { + if (observer) { + observer.disconnect(); + observer = undefined; + } + clearTimeout(timeout); + timeout = setTimeout(() => this.onIdle(observe), 200); + }); + }; + + private getRootMargin(): `-${number}px 0% ${number}px` { + const navBarHeight = document.querySelector('header')?.getBoundingClientRect().height || 0; + const mobileTocHeight = this.querySelector('summary')?.getBoundingClientRect().height || 0; + const top = navBarHeight + mobileTocHeight + 32; + const bottom = top + 53; + const height = document.documentElement.clientHeight; + return `-${top}px 0% ${bottom - height}px`; + } +} + +if (!customElements.get('starlight-toc')) { + customElements.define('starlight-toc', FluxerStarlightTOC); +} diff --git a/fluxer_docs/src/components/TableOfContents.astro b/fluxer_docs/src/components/TableOfContents.astro new file mode 100644 index 000000000..dc419c493 --- /dev/null +++ b/fluxer_docs/src/components/TableOfContents.astro @@ -0,0 +1,22 @@ +--- +// SPDX-License-Identifier: AGPL-3.0-or-later + +import TableOfContentsList from './TableOfContentsList.astro'; + +const {toc} = Astro.locals.starlightRoute; +--- + +{ + toc && ( + + + + ) +} + + diff --git a/fluxer_docs/src/components/TableOfContentsList.astro b/fluxer_docs/src/components/TableOfContentsList.astro new file mode 100644 index 000000000..c734fa6ac --- /dev/null +++ b/fluxer_docs/src/components/TableOfContentsList.astro @@ -0,0 +1,51 @@ +--- +// SPDX-License-Identifier: AGPL-3.0-or-later + +interface TocItem { + slug: string; + text: string; + children: Array; +} + +interface Props { + toc: Array; + depth?: number; + + markFirst?: boolean; +} + +const {toc, depth = 0, markFirst = false} = Astro.props; +--- + +
    + { + toc.map((heading, index) => ( +
  • + + {heading.text} + + {heading.children.length > 0 && } +
  • + )) + } +
+ + diff --git a/fluxer_docs/src/components/ThemeProvider.astro b/fluxer_docs/src/components/ThemeProvider.astro new file mode 100644 index 000000000..fe9647157 --- /dev/null +++ b/fluxer_docs/src/components/ThemeProvider.astro @@ -0,0 +1,42 @@ +--- +// SPDX-License-Identifier: AGPL-3.0-or-later +import {Icon} from '@astrojs/starlight/components'; +--- + + + + diff --git a/fluxer_docs/src/components/ThemeSelect.astro b/fluxer_docs/src/components/ThemeSelect.astro new file mode 100644 index 000000000..c4d31b42c --- /dev/null +++ b/fluxer_docs/src/components/ThemeSelect.astro @@ -0,0 +1,66 @@ +--- +// SPDX-License-Identifier: AGPL-3.0-or-later +import Select from '@astrojs/starlight/components/Select.astro'; +--- + + +