diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 545dcd8d5..7ab669fef 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -426,6 +426,28 @@ jobs: 'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml', 'packages/markdown_parser/rust/src/**') }} + lint: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + + - name: Install pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 + + - name: Install Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: '24' + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Check formatting and lint + run: pnpm exec biome ci . + i18n: runs-on: ubuntu-24.04 timeout-minutes: 25 diff --git a/packages/config/src/__tests__/DeployEnvCoverage.test.ts b/packages/config/src/__tests__/DeployEnvCoverage.test.ts index 8f8c1e7c6..23b76de74 100644 --- a/packages/config/src/__tests__/DeployEnvCoverage.test.ts +++ b/packages/config/src/__tests__/DeployEnvCoverage.test.ts @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import {readFileSync} from 'node:fs'; +import {readdirSync, readFileSync} from 'node:fs'; import path from 'node:path'; import {fileURLToPath} from 'node:url'; import {describe, expect, test} from 'vitest'; @@ -11,9 +11,15 @@ const INTERPOLATION_PATTERN = /\$\{([A-Z][A-Z0-9_]*)[:?}-]/g; const CADDY_PLACEHOLDER_PATTERN = /\{\$([A-Z][A-Z0-9_]*)[:}]/g; const DECLARATION_PATTERN = /^#?([A-Z][A-Z0-9_]*)=/gm; const ACTIVE_DECLARATION_PATTERN = /^([A-Z][A-Z0-9_]*)=/gm; +const LINE_DECLARATION_PATTERN = /^#?([A-Z][A-Z0-9_]*)=(.*)$/u; +const LINE_REFERENCE_PATTERN = /\$\{?([A-Z][A-Z0-9_]*)/gu; const read = (name: string) => readFileSync(path.join(SELF_HOSTING, name), 'utf8'); +const composeFiles = readdirSync(SELF_HOSTING) + .filter((name) => name.endsWith('.yml')) + .sort(); + const namesMatching = (source: string, pattern: RegExp) => new Set([...source.matchAll(pattern)].map(([, name]) => name)); @@ -23,7 +29,7 @@ const example = read('.env.example'); const declared = new Set([...example.matchAll(DECLARATION_PATTERN)].map(([, name]) => name)); describe('.env.example covers every name the compose files interpolate', () => { - for (const file of ['docker-compose.yml', 'docker-compose.proxy.yml']) { + for (const file of composeFiles) { test(`every \${NAME} in ${file} has a line in .env.example`, () => { const missing = [...interpolatedNames(read(file))].filter((name) => !declared.has(name)).sort(); expect(missing).toEqual([]); @@ -48,4 +54,29 @@ describe('.env.example covers every name the compose files interpolate', () => { } expect([...repeated].sort()).toEqual([]); }); + + test('every name a .env.example line reads is declared above it', () => { + const lines = example.split('\n'); + const declaredAt = new Map(); + lines.forEach((line, index) => { + const match = LINE_DECLARATION_PATTERN.exec(line); + if (match && !declaredAt.has(match[1])) { + declaredAt.set(match[1], index); + } + }); + const unresolved: Array = []; + lines.forEach((line, index) => { + const match = LINE_DECLARATION_PATTERN.exec(line); + if (!match) { + return; + } + for (const [, name] of match[2].matchAll(LINE_REFERENCE_PATTERN)) { + const at = declaredAt.get(name); + if (at === undefined || at > index) { + unresolved.push(`${match[1]} reads ${name}`); + } + } + }); + expect(unresolved.sort()).toEqual([]); + }); }); diff --git a/packages/config/src/__tests__/NonDefaultPortCompose.test.ts b/packages/config/src/__tests__/NonDefaultPortCompose.test.ts new file mode 100644 index 000000000..a44de0a3b --- /dev/null +++ b/packages/config/src/__tests__/NonDefaultPortCompose.test.ts @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {readFileSync} from 'node:fs'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {loadConfig, resetConfig} from '@fluxer/config/src/ConfigLoader'; +import {normalizePublicEndpoint} from '@fluxer/config/src/EndpointDerivation'; +import type {MasterConfig} from '@fluxer/config/src/MasterConfig'; +import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest'; + +const SELF_HOSTING = path.join(fileURLToPath(new URL('../../../../', import.meta.url)), 'deploy/self-hosting'); + +const compose = readFileSync(path.join(SELF_HOSTING, 'docker-compose.yml'), 'utf8'); +const lines = compose.split('\n'); + +const DOMAIN = 'chat.example.com'; +const PUBLIC_PORT = '19080'; +const ORIGIN_PORT = '29080'; + +const SECRETS: Record = { + FLUXER_DOMAIN: DOMAIN, + POSTGRES_PASSWORD: 'postgres-password', + MEILI_MASTER_KEY: 'meili-master-key', + FLUXER_S3_ACCESS_KEY: 's3-access-key', + FLUXER_S3_SECRET_KEY: 's3-secret-key', + LIVEKIT_API_KEY: 'livekit-api-key', + LIVEKIT_API_SECRET: 'livekit-api-secret', + FLUXER_ERLANG_COOKIE: 'erlang-cookie', + FLUXER_SUDO_MODE_SECRET: 'sudo-mode-secret', + FLUXER_CONNECTION_INITIATION_SECRET: 'connection-initiation-secret', + FLUXER_GATEWAY_RPC_AUTH_TOKEN: 'gateway-rpc-auth-token', + FLUXER_MEDIA_PROXY_SECRET_KEY: 'media-proxy-secret-key', + FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64: 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=', + FLUXER_ADMIN_SECRET_KEY_BASE: 'admin-secret-key-base', + FLUXER_ADMIN_OAUTH_CLIENT_SECRET: 'admin-oauth-client-secret', + FLUXER_VAPID_PUBLIC_KEY: 'BB76bTFIuoqmxJtTfZX0yGTn1f_qu9H03B_nkj8OyExJFkN7Y-HBZZzShnHZoEhXKc5ZRy3jFu7OkBbnaQG-4aw', + FLUXER_VAPID_PRIVATE_KEY: 'Xgi-3P8J-I3Q6U1HlCcXMuc_tKLGAM9nIfznX3Hz68o', +}; + +const PORT_ONLY_ENV: Record = { + ...SECRETS, + FLUXER_PUBLIC_SCHEME: 'http', + FLUXER_PUBLIC_PORT: PUBLIC_PORT, +}; + +const DOCUMENTED_RECIPE_ENV: Record = { + ...PORT_ONLY_ENV, + FLUXER_PUBLIC_ORIGIN: `http://${DOMAIN}:${PUBLIC_PORT}`, + FLUXER_HTTP_PORT: PUBLIC_PORT, +}; + +const ORIGIN_ONLY_ENV: Record = { + ...SECRETS, + FLUXER_PUBLIC_ORIGIN: `https://${DOMAIN}:${ORIGIN_PORT}`, + FLUXER_HTTPS_PORT: ORIGIN_PORT, +}; + +const indentOf = (line: string): number => line.length - line.trimStart().length; + +function unquote(value: string): string { + const trimmed = value.trim(); + const quote = trimmed.slice(0, 1); + const quoted = (quote === '"' || quote === "'") && trimmed.length > 1 && trimmed.endsWith(quote); + return quoted ? trimmed.slice(1, -1) : trimmed; +} + +function anchorStart(anchor: string): number { + const start = lines.findIndex((line) => line.startsWith('x-') && line.endsWith(`&${anchor}`)); + if (start === -1) { + throw new Error(`docker-compose.yml has no &${anchor} anchor`); + } + return start; +} + +function mapping(start: number, indent: number): Map { + const entries = new Map(); + for (let index = start; index < lines.length; index += 1) { + const line = lines[index]; + if (line.trim().length === 0 || line.trimStart().startsWith('#')) { + continue; + } + if (indentOf(line) < indent) { + break; + } + if (indentOf(line) > indent) { + continue; + } + const merge = /^<<: \*([a-z][a-z0-9-]*)$/u.exec(line.trim()); + if (merge) { + for (const [key, value] of mapping(anchorStart(merge[1]) + 1, 2)) { + entries.set(key, value); + } + continue; + } + const entry = /^([A-Za-z_][A-Za-z0-9_]*): (.*)$/u.exec(line.trim()); + if (entry) { + entries.set(entry[1], unquote(entry[2])); + } + } + return entries; +} + +function serviceSection(service: string, key: string): number { + const start = lines.indexOf(` ${service}:`); + if (start === -1) { + throw new Error(`docker-compose.yml has no ${service} service`); + } + for (let index = start + 1; index < lines.length; index += 1) { + const line = lines[index]; + if (line.trim().length > 0 && indentOf(line) <= 2) { + break; + } + if (line === ` ${key}:`) { + return index; + } + } + return -1; +} + +function serviceList(service: string, key: string): Array { + const start = serviceSection(service, key); + if (start === -1) { + return []; + } + const items: Array = []; + for (let index = start + 1; index < lines.length; index += 1) { + const line = lines[index]; + if (line.trim().length === 0) { + continue; + } + if (indentOf(line) <= 4) { + break; + } + const item = /^- (.*)$/u.exec(line.trim()); + if (item) { + items.push(unquote(item[1])); + } + } + return items; +} + +const SERVICES_WITHOUT_ENDPOINT_REPAIR = new Set(['edge']); + +function serviceNames(): Array { + const section = compose.slice(compose.indexOf('\nservices:\n'), compose.indexOf('\nnetworks:\n')); + return [...section.matchAll(/\n {2}([a-z][a-z0-9_-]*):\n/gu)].map(([, name]) => name); +} + +function closingBrace(value: string, open: number): number { + let depth = 0; + for (let index = open; index < value.length; index += 1) { + if (value[index] === '{') { + depth += 1; + } else if (value[index] === '}') { + depth -= 1; + if (depth === 0) { + return index; + } + } + } + throw new Error(`unbalanced interpolation in ${value}`); +} + +function expand(value: string, env: Record): string { + let out = ''; + let index = 0; + while (index < value.length) { + const dollar = value.indexOf('$', index); + if (dollar === -1) { + return out + value.slice(index); + } + out += value.slice(index, dollar); + if (value[dollar + 1] !== '{') { + const bare = /^[A-Za-z_][A-Za-z0-9_]*/u.exec(value.slice(dollar + 1)); + out += bare ? (env[bare[0]] ?? '') : '$'; + index = dollar + 1 + (bare ? bare[0].length : 0); + continue; + } + const close = closingBrace(value, dollar + 1); + const inner = value.slice(dollar + 2, close); + const operator = inner.search(/:[-?+]/u); + const name = operator === -1 ? inner : inner.slice(0, operator); + const fallback = operator === -1 ? '' : inner.slice(operator + 2); + const current = env[name] ?? ''; + if (current.length > 0) { + out += inner[operator + 1] === '+' && operator !== -1 ? expand(fallback, env) : current; + } else if (inner[operator + 1] === '-' && operator !== -1) { + out += expand(fallback, env); + } else if (inner[operator + 1] === '?' && operator !== -1) { + throw new Error(`${name} is required: ${fallback}`); + } + index = close + 1; + } + return out; +} + +function expandedEnvironment(service: string, env: Record): Record { + const expanded: Record = {}; + for (const [key, value] of mapping(serviceSection(service, 'environment') + 1, 6)) { + expanded[key] = expand(value, env); + } + return expanded; +} + +function publicUrl(value: string): URL | null { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + return null; + } + return parsed.hostname === DOMAIN ? parsed : null; +} + +function publicUrlNames(environment: Record): Array { + return Object.keys(environment) + .filter((name) => publicUrl(environment[name]) !== null) + .sort(); +} + +function repairedPublicUrls(service: string, env: Record): Array<[string, string]> { + const environment = expandedEnvironment(service, env); + const publicPort = Number.parseInt(environment.FLUXER_PUBLIC_PORT ?? '', 10); + return publicUrlNames(environment).map((name) => [ + `${service}.${name}`, + normalizePublicEndpoint( + environment[name], + environment.FLUXER_BASE_DOMAIN ?? '', + Number.isNaN(publicPort) ? undefined : publicPort, + ), + ]); +} + +function withoutPort(entries: Array<[string, string]>, port: string): Array { + return entries + .filter(([, value]) => publicUrl(value)?.port !== port) + .map(([name, value]) => `${name}=${value}`) + .sort(); +} + +function publishedPorts(env: Record): Array { + return serviceList('edge', 'ports') + .map((mapped) => expand(mapped, env).split(':')) + .map((parts) => parts[parts.length - 2] ?? ''); +} + +function browserFacingUrls(config: MasterConfig): Array<[string, string]> { + const entries: Array<[string, string]> = [ + ...Object.entries(config.endpoints), + ['integrations.voice.url', config.integrations.voice.url], + ['services.media_proxy.upload_relay.endpoint', config.services.media_proxy.upload_relay.endpoint], + ...config.auth.passkeys.additional_allowed_origins.map((origin, index): [string, string] => [ + `auth.passkeys.additional_allowed_origins.${index}`, + origin, + ]), + ]; + return entries.filter(([, value]) => publicUrl(value) !== null); +} + +async function loadApiConfig(env: Record): Promise { + for (const key of Object.keys(process.env)) { + if (key.startsWith('FLUXER_')) { + vi.stubEnv(key, undefined); + } + } + for (const [key, value] of Object.entries(expandedEnvironment('api', env))) { + if (key.startsWith('FLUXER_')) { + vi.stubEnv(key, value); + } + } + try { + return await loadConfig(); + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } +} + +describe('the shipped compose stack expanded on a non-default port', () => { + test('every service handed a public URL is handed the base domain and port that repair it', () => { + const starved = serviceNames() + .filter((service) => !SERVICES_WITHOUT_ENDPOINT_REPAIR.has(service)) + .filter((service) => { + const environment = expandedEnvironment(service, PORT_ONLY_ENV); + return ( + publicUrlNames(environment).length > 0 && (!environment.FLUXER_BASE_DOMAIN || !environment.FLUXER_PUBLIC_PORT) + ); + }); + expect(starved).toEqual([]); + }); + + test('every public URL the stack hands a browser carries the port', () => { + const entries = serviceNames() + .filter((service) => !SERVICES_WITHOUT_ENDPOINT_REPAIR.has(service)) + .flatMap((service) => repairedPublicUrls(service, PORT_ONLY_ENV)); + expect(withoutPort(entries, PUBLIC_PORT)).toEqual([]); + }); + + test('the edge listens on its scheme default whatever the public port is', () => { + const environment = expandedEnvironment('edge', PORT_ONLY_ENV); + expect(environment.FLUXER_EDGE_SITE_ADDRESS).toBe(`http://${DOMAIN}`); + expect(publishedPorts(PORT_ONLY_ENV)).not.toContain(PUBLIC_PORT); + }); + + test('the documented recipe publishes the port every public URL advertises', () => { + expect(publishedPorts(DOCUMENTED_RECIPE_ENV)).toContain(PUBLIC_PORT); + }); +}); + +describe('a public origin carrying a port while FLUXER_PUBLIC_PORT stays standard', () => { + beforeEach(() => { + resetConfig(); + }); + + afterEach(() => { + resetConfig(); + vi.unstubAllEnvs(); + }); + + test('the compose overrides all carry the origin port', () => { + const environment = expandedEnvironment('api', ORIGIN_ONLY_ENV); + const entries = publicUrlNames(environment).map((name): [string, string] => [name, environment[name]]); + expect(withoutPort(entries, ORIGIN_PORT)).toEqual([]); + }); + + test('the loaded config either ports every public URL or refuses the split', async () => { + const loaded = await loadApiConfig(ORIGIN_ONLY_ENV); + if (loaded instanceof Error) { + expect(loaded.message).not.toBe(''); + return; + } + expect(withoutPort(browserFacingUrls(loaded), ORIGIN_PORT)).toEqual([]); + }); + + test('the port-only recipe ports every public URL the loaded config exposes', async () => { + const loaded = await loadApiConfig(PORT_ONLY_ENV); + if (loaded instanceof Error) { + throw loaded; + } + expect(withoutPort(browserFacingUrls(loaded), PUBLIC_PORT)).toEqual([]); + }); +});