refactor(schema): simplify validation and OpenAPI generation (#2709)

This commit is contained in:
Hampus authored and GitHub committed 2026-09-11 23:24:26 +02:00
1 parent 4bc5593f9f
commit de1fd95a99
510 files changed
+29494 -36131

No files matched your search

+3 -8
View File
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createECDH} from 'node:crypto';
import {type ConfigObject, isConfigObject} from '@fluxer/config/src/config_loader/ConfigObject';
import {buildNamedFluxerEnvOverrides} from '@fluxer/config/src/config_loader/EnvironmentOverrides';
import {
buildUrl,
@@ -11,8 +12,6 @@ import {
} from '@fluxer/config/src/EndpointDerivation';
import type {MasterConfig} from '@fluxer/config/src/MasterConfig';
type ConfigObject = Record<string, unknown>;
let cachedConfig: MasterConfig | null = null;
const DEFAULT_PASSKEY_ORIGINS = [
@@ -312,18 +311,14 @@ function defaultConfig(): MasterConfig {
};
}
function isPlainObject(value: unknown): value is ConfigObject {
return value !== null && typeof value === 'object' && !Array.isArray(value);
}
function mergeConfig<T>(base: T, overrides: unknown): T {
if (!isPlainObject(base) || !isPlainObject(overrides)) {
if (!isConfigObject(base) || !isConfigObject(overrides)) {
return overrides === undefined ? base : (overrides as T);
}
const out: ConfigObject = {...base};
for (const [key, value] of Object.entries(overrides)) {
const current = out[key];
out[key] = isPlainObject(current) && isPlainObject(value) ? mergeConfig(current, value) : value;
out[key] = isConfigObject(current) && isConfigObject(value) ? mergeConfig(current, value) : value;
}
return out as T;
}
+1 -7
View File
@@ -17,13 +17,7 @@ export function extractKVClientConfig(master: MasterConfig) {
kvUrl: master.internal.kv,
kvMode: master.internal.kv_mode,
kvClusterNodes: master.internal.kv_cluster_nodes,
kvClusterNatMap: master.internal.kv_cluster_nat_map as Record<
string,
{
host: string;
port: number;
}
>,
kvClusterNatMap: master.internal.kv_cluster_nat_map,
};
}
@@ -1,8 +1,10 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
import {
buildUrl,
canonicalizeDomain,
type DerivedEndpoints,
type DomainConfig,
deriveDomain,
deriveEndpointsFromDomain,
@@ -12,35 +14,19 @@ import {
import {describe, expect, test} from 'vitest';
describe('buildUrl', () => {
test('omits standard HTTP port (80)', () => {
expect(buildUrl('http', 'example.com', 80, '/path')).toBe('http://example.com/path');
});
test('omits standard HTTPS port (443)', () => {
expect(buildUrl('https', 'example.com', 443, '/path')).toBe('https://example.com/path');
});
test('omits standard WebSocket port (80)', () => {
expect(buildUrl('ws', 'example.com', 80, '/gateway')).toBe('ws://example.com/gateway');
});
test('omits standard secure WebSocket port (443)', () => {
expect(buildUrl('wss', 'example.com', 443, '/gateway')).toBe('wss://example.com/gateway');
});
test('includes non-standard port', () => {
expect(buildUrl('http', 'localhost', 8088, '/api')).toBe('http://localhost:8088/api');
});
test('includes non-standard HTTPS port', () => {
expect(buildUrl('https', 'example.com', 8443, '/api')).toBe('https://example.com:8443/api');
});
test('handles missing port', () => {
expect(buildUrl('https', 'example.com', undefined, '/api')).toBe('https://example.com/api');
});
test('handles missing path', () => {
expect(buildUrl('https', 'example.com', 443)).toBe('https://example.com');
});
test('handles empty path', () => {
expect(buildUrl('https', 'example.com', 443, '')).toBe('https://example.com');
});
test('handles root path', () => {
expect(buildUrl('https', 'example.com', 443, '/')).toBe('https://example.com/');
test.each([
['http', 'example.com', 80, '/path', 'http://example.com/path'],
['https', 'example.com', 443, '/path', 'https://example.com/path'],
['ws', 'example.com', 80, '/gateway', 'ws://example.com/gateway'],
['wss', 'example.com', 443, '/gateway', 'wss://example.com/gateway'],
['http', 'localhost', 8088, '/api', 'http://localhost:8088/api'],
['https', 'example.com', 8443, '/api', 'https://example.com:8443/api'],
['https', 'example.com', undefined, '/api', 'https://example.com/api'],
['https', 'example.com', 443, undefined, 'https://example.com'],
['https', 'example.com', 443, '', 'https://example.com'],
['https', 'example.com', 443, '/', 'https://example.com/'],
] as const)('%s %s:%s%s becomes %s', (scheme, domain, port, path, expected) => {
expect(buildUrl(scheme, domain, port, path)).toBe(expected);
});
});
@@ -50,258 +36,269 @@ describe('deriveDomain', () => {
public_scheme: 'https',
internal_scheme: 'http',
};
test('uses base domain for api endpoint', () => {
expect(deriveDomain('api', baseConfig)).toBe('fluxer.dev');
test.each([
'api',
'api_client',
'app',
'gateway',
'media',
'static_cdn',
'admin',
'docs',
'marketing',
'invite',
'gift',
] as const)('uses the base domain for %s without an override', (endpoint) => {
expect(deriveDomain(endpoint, baseConfig)).toBe('fluxer.dev');
});
test('uses base domain for app endpoint', () => {
expect(deriveDomain('app', baseConfig)).toBe('fluxer.dev');
});
test('uses base domain for gateway endpoint', () => {
expect(deriveDomain('gateway', baseConfig)).toBe('fluxer.dev');
});
test('uses base domain for media endpoint', () => {
expect(deriveDomain('media', baseConfig)).toBe('fluxer.dev');
});
test('uses custom static CDN domain when specified', () => {
const config = {...baseConfig, static_cdn_domain: 'cdn.fluxer.dev'};
expect(deriveDomain('static_cdn', config)).toBe('cdn.fluxer.dev');
});
test('uses base domain for static CDN when custom domain not specified', () => {
expect(deriveDomain('static_cdn', baseConfig)).toBe('fluxer.dev');
});
test('uses custom invite domain when specified', () => {
const config = {...baseConfig, invite_domain: 'fluxer.gg'};
expect(deriveDomain('invite', config)).toBe('fluxer.gg');
});
test('uses base domain for invite when custom domain not specified', () => {
expect(deriveDomain('invite', baseConfig)).toBe('fluxer.dev');
});
test('uses custom gift domain when specified', () => {
const config = {...baseConfig, gift_domain: 'fluxer.gift'};
expect(deriveDomain('gift', config)).toBe('fluxer.gift');
});
test('uses base domain for gift when custom domain not specified', () => {
expect(deriveDomain('gift', baseConfig)).toBe('fluxer.dev');
test.each([
['static_cdn', {static_cdn_domain: 'cdn.fluxer.dev'}, 'cdn.fluxer.dev'],
['invite', {invite_domain: 'fluxer.gg'}, 'fluxer.gg'],
['gift', {gift_domain: 'fluxer.gift'}, 'fluxer.gift'],
] as const)('uses the custom %s domain', (endpoint, overrides, expected) => {
expect(deriveDomain(endpoint, {...baseConfig, ...overrides})).toBe(expected);
});
});
describe('deriveEndpointsFromDomain', () => {
describe('development environment (localhost)', () => {
const devConfig: DomainConfig = {
interface EndpointScenario {
name: string;
config: DomainConfig;
expected: DerivedEndpoints;
}
const endpointScenarios: Array<EndpointScenario> = [
{
name: 'development',
config: {
base_domain: 'localhost',
public_scheme: 'http',
internal_scheme: 'http',
public_port: 8088,
internal_port: 8088,
};
const endpoints = deriveEndpointsFromDomain(devConfig);
test('derives api endpoint with port', () => {
expect(endpoints.api).toBe('http://localhost:8088/api');
});
test('derives api client endpoint with port', () => {
expect(endpoints.api_client).toBe('http://localhost:8088/api');
});
test('derives app endpoint with port', () => {
expect(endpoints.app).toBe('http://localhost:8088');
});
test('derives gateway endpoint with ws scheme', () => {
expect(endpoints.gateway).toBe('ws://localhost:8088/gateway');
});
test('derives media endpoint with port', () => {
expect(endpoints.media).toBe('http://localhost:8088/media');
});
test('derives static CDN endpoint via public origin', () => {
expect(endpoints.static_cdn).toBe('http://localhost:8088');
});
test('derives admin endpoint with port', () => {
expect(endpoints.admin).toBe('http://localhost:8088/admin');
});
test('derives marketing endpoint with port', () => {
expect(endpoints.marketing).toBe('http://localhost:8088/marketing');
});
test('derives invite endpoint with port', () => {
expect(endpoints.invite).toBe('http://localhost:8088/invite');
});
test('derives gift endpoint with port', () => {
expect(endpoints.gift).toBe('http://localhost:8088/gift');
});
});
describe('production environment (standard HTTPS port)', () => {
const prodConfig: DomainConfig = {
},
expected: {
api: 'http://localhost:8088/api',
api_client: 'http://localhost:8088/api',
app: 'http://localhost:8088',
gateway: 'ws://localhost:8088/gateway',
media: 'http://localhost:8088/media',
static_cdn: 'http://localhost:8088',
admin: 'http://localhost:8088/admin',
docs: 'https://fluxer.dev',
marketing: 'http://localhost:8088/marketing',
invite: 'http://localhost:8088/invite',
gift: 'http://localhost:8088/gift',
},
},
{
name: 'production',
config: {
base_domain: 'fluxer.app',
public_scheme: 'https',
internal_scheme: 'http',
public_port: 443,
internal_port: 8080,
};
const endpoints = deriveEndpointsFromDomain(prodConfig);
test('derives api endpoint without port', () => {
expect(endpoints.api).toBe('https://fluxer.app/api');
});
test('derives api client endpoint without port', () => {
expect(endpoints.api_client).toBe('https://fluxer.app/api');
});
test('derives app endpoint without port', () => {
expect(endpoints.app).toBe('https://fluxer.app');
});
test('derives gateway endpoint with wss scheme without port', () => {
expect(endpoints.gateway).toBe('wss://fluxer.app/gateway');
});
test('derives media endpoint without port', () => {
expect(endpoints.media).toBe('https://fluxer.app/media');
});
test('derives static CDN endpoint without port', () => {
expect(endpoints.static_cdn).toBe('https://fluxer.app');
});
test('derives admin endpoint without port', () => {
expect(endpoints.admin).toBe('https://fluxer.app/admin');
});
test('derives marketing endpoint without port', () => {
expect(endpoints.marketing).toBe('https://fluxer.app/marketing');
});
test('derives invite endpoint without port', () => {
expect(endpoints.invite).toBe('https://fluxer.app/invite');
});
test('derives gift endpoint without port', () => {
expect(endpoints.gift).toBe('https://fluxer.app/gift');
});
});
describe('staging environment (custom port)', () => {
const stagingConfig: DomainConfig = {
},
expected: {
api: 'https://fluxer.app/api',
api_client: 'https://fluxer.app/api',
app: 'https://fluxer.app',
gateway: 'wss://fluxer.app/gateway',
media: 'https://fluxer.app/media',
static_cdn: 'https://fluxer.app',
admin: 'https://fluxer.app/admin',
docs: 'https://fluxer.dev',
marketing: 'https://fluxer.app/marketing',
invite: 'https://fluxer.app/invite',
gift: 'https://fluxer.app/gift',
},
},
{
name: 'staging',
config: {
base_domain: 'staging.fluxer.dev',
public_scheme: 'https',
internal_scheme: 'http',
public_port: 8443,
internal_port: 8080,
};
const endpoints = deriveEndpointsFromDomain(stagingConfig);
test('derives api endpoint with custom port', () => {
expect(endpoints.api).toBe('https://staging.fluxer.dev:8443/api');
});
test('derives api client endpoint with custom port', () => {
expect(endpoints.api_client).toBe('https://staging.fluxer.dev:8443/api');
});
test('derives app endpoint with custom port', () => {
expect(endpoints.app).toBe('https://staging.fluxer.dev:8443');
});
test('derives gateway endpoint with wss and custom port', () => {
expect(endpoints.gateway).toBe('wss://staging.fluxer.dev:8443/gateway');
});
});
describe('custom CDN domain', () => {
const staticCdnConfig: DomainConfig = {
},
expected: {
api: 'https://staging.fluxer.dev:8443/api',
api_client: 'https://staging.fluxer.dev:8443/api',
app: 'https://staging.fluxer.dev:8443',
gateway: 'wss://staging.fluxer.dev:8443/gateway',
media: 'https://staging.fluxer.dev:8443/media',
static_cdn: 'https://staging.fluxer.dev:8443',
admin: 'https://staging.fluxer.dev:8443/admin',
docs: 'https://fluxer.dev',
marketing: 'https://staging.fluxer.dev:8443/marketing',
invite: 'https://staging.fluxer.dev:8443/invite',
gift: 'https://staging.fluxer.dev:8443/gift',
},
},
{
name: 'custom CDN',
config: {
base_domain: 'fluxer.app',
public_scheme: 'https',
internal_scheme: 'http',
public_port: 443,
static_cdn_domain: 'cdn.fluxer.app',
};
const endpoints = deriveEndpointsFromDomain(staticCdnConfig);
test('uses custom CDN domain', () => {
expect(endpoints.static_cdn).toBe('https://cdn.fluxer.app');
});
test('other endpoints use base domain', () => {
expect(endpoints.api).toBe('https://fluxer.app/api');
expect(endpoints.app).toBe('https://fluxer.app');
});
});
describe('custom invite and gift domains', () => {
const customConfig: DomainConfig = {
},
expected: {
api: 'https://fluxer.app/api',
api_client: 'https://fluxer.app/api',
app: 'https://fluxer.app',
gateway: 'wss://fluxer.app/gateway',
media: 'https://fluxer.app/media',
static_cdn: 'https://cdn.fluxer.app',
admin: 'https://fluxer.app/admin',
docs: 'https://fluxer.dev',
marketing: 'https://fluxer.app/marketing',
invite: 'https://fluxer.app/invite',
gift: 'https://fluxer.app/gift',
},
},
{
name: 'custom invite and gift domains',
config: {
base_domain: 'fluxer.app',
public_scheme: 'https',
internal_scheme: 'http',
public_port: 443,
invite_domain: 'fluxer.gg',
gift_domain: 'fluxer.gift',
};
const endpoints = deriveEndpointsFromDomain(customConfig);
test('uses custom invite domain', () => {
expect(endpoints.invite).toBe('https://fluxer.gg/invite');
});
test('uses custom gift domain', () => {
expect(endpoints.gift).toBe('https://fluxer.gift/gift');
});
test('other endpoints use base domain', () => {
expect(endpoints.api).toBe('https://fluxer.app/api');
expect(endpoints.app).toBe('https://fluxer.app');
});
});
describe('WebSocket scheme derivation', () => {
test('derives ws from http', () => {
const config: DomainConfig = {
base_domain: 'localhost',
public_scheme: 'http',
internal_scheme: 'http',
public_port: 8088,
};
const endpoints = deriveEndpointsFromDomain(config);
expect(endpoints.gateway).toBe('ws://localhost:8088/gateway');
});
test('derives wss from https', () => {
const config: DomainConfig = {
base_domain: 'fluxer.app',
public_scheme: 'https',
internal_scheme: 'http',
public_port: 443,
};
const endpoints = deriveEndpointsFromDomain(config);
expect(endpoints.gateway).toBe('wss://fluxer.app/gateway');
});
});
describe('canary environment', () => {
const canaryConfig: DomainConfig = {
},
expected: {
api: 'https://fluxer.app/api',
api_client: 'https://fluxer.app/api',
app: 'https://fluxer.app',
gateway: 'wss://fluxer.app/gateway',
media: 'https://fluxer.app/media',
static_cdn: 'https://fluxer.app',
admin: 'https://fluxer.app/admin',
docs: 'https://fluxer.dev',
marketing: 'https://fluxer.app/marketing',
invite: 'https://fluxer.gg/invite',
gift: 'https://fluxer.gift/gift',
},
},
{
name: 'canary',
config: {
base_domain: 'canary.fluxer.app',
public_scheme: 'https',
internal_scheme: 'http',
public_port: 443,
static_cdn_domain: 'cdn-canary.fluxer.app',
};
const endpoints = deriveEndpointsFromDomain(canaryConfig);
test('derives api endpoint for canary', () => {
expect(endpoints.api).toBe('https://canary.fluxer.app/api');
});
test('derives app endpoint for canary', () => {
expect(endpoints.app).toBe('https://canary.fluxer.app');
});
test('derives gateway endpoint for canary', () => {
expect(endpoints.gateway).toBe('wss://canary.fluxer.app/gateway');
});
test('uses custom CDN domain for canary', () => {
expect(endpoints.static_cdn).toBe('https://cdn-canary.fluxer.app');
});
});
describe('edge cases', () => {
test('handles standard HTTP port (80)', () => {
const config: DomainConfig = {
base_domain: 'example.com',
public_scheme: 'http',
internal_scheme: 'http',
public_port: 80,
};
const endpoints = deriveEndpointsFromDomain(config);
expect(endpoints.api).toBe('http://example.com/api');
expect(endpoints.gateway).toBe('ws://example.com/gateway');
});
test('handles ports when undefined', () => {
const config: DomainConfig = {
base_domain: 'example.com',
public_scheme: 'https',
internal_scheme: 'http',
};
const endpoints = deriveEndpointsFromDomain(config);
expect(endpoints.api).toBe('https://example.com/api');
expect(endpoints.app).toBe('https://example.com');
});
test('handles IPv4 addresses', () => {
const config: DomainConfig = {
base_domain: '127.0.0.1',
public_scheme: 'http',
internal_scheme: 'http',
public_port: 8088,
};
const endpoints = deriveEndpointsFromDomain(config);
expect(endpoints.api).toBe('http://127.0.0.1:8088/api');
});
},
expected: {
api: 'https://canary.fluxer.app/api',
api_client: 'https://canary.fluxer.app/api',
app: 'https://canary.fluxer.app',
gateway: 'wss://canary.fluxer.app/gateway',
media: 'https://canary.fluxer.app/media',
static_cdn: 'https://cdn-canary.fluxer.app',
admin: 'https://canary.fluxer.app/admin',
docs: 'https://fluxer.dev',
marketing: 'https://canary.fluxer.app/marketing',
invite: 'https://canary.fluxer.app/invite',
gift: 'https://canary.fluxer.app/gift',
},
},
{
name: 'standard HTTP port',
config: {
base_domain: 'example.com',
public_scheme: 'http',
internal_scheme: 'http',
public_port: 80,
},
expected: {
api: 'http://example.com/api',
api_client: 'http://example.com/api',
app: 'http://example.com',
gateway: 'ws://example.com/gateway',
media: 'http://example.com/media',
static_cdn: 'http://example.com',
admin: 'http://example.com/admin',
docs: 'https://fluxer.dev',
marketing: 'http://example.com/marketing',
invite: 'http://example.com/invite',
gift: 'http://example.com/gift',
},
},
{
name: 'unspecified port',
config: {
base_domain: 'example.com',
public_scheme: 'https',
internal_scheme: 'http',
},
expected: {
api: 'https://example.com/api',
api_client: 'https://example.com/api',
app: 'https://example.com',
gateway: 'wss://example.com/gateway',
media: 'https://example.com/media',
static_cdn: 'https://example.com',
admin: 'https://example.com/admin',
docs: 'https://fluxer.dev',
marketing: 'https://example.com/marketing',
invite: 'https://example.com/invite',
gift: 'https://example.com/gift',
},
},
{
name: 'IPv4',
config: {
base_domain: '127.0.0.1',
public_scheme: 'http',
internal_scheme: 'http',
public_port: 8088,
},
expected: {
api: 'http://127.0.0.1:8088/api',
api_client: 'http://127.0.0.1:8088/api',
app: 'http://127.0.0.1:8088',
gateway: 'ws://127.0.0.1:8088/gateway',
media: 'http://127.0.0.1:8088/media',
static_cdn: 'http://127.0.0.1:8088',
admin: 'http://127.0.0.1:8088/admin',
docs: 'https://fluxer.dev',
marketing: 'http://127.0.0.1:8088/marketing',
invite: 'http://127.0.0.1:8088/invite',
gift: 'http://127.0.0.1:8088/gift',
},
},
{
name: 'external CDN uses HTTPS without the public port',
config: {
base_domain: 'localhost',
public_scheme: 'http',
internal_scheme: 'http',
public_port: 8088,
static_cdn_domain: 'cdn.example.com',
},
expected: {
api: 'http://localhost:8088/api',
api_client: 'http://localhost:8088/api',
app: 'http://localhost:8088',
gateway: 'ws://localhost:8088/gateway',
media: 'http://localhost:8088/media',
static_cdn: 'https://cdn.example.com',
admin: 'http://localhost:8088/admin',
docs: 'https://fluxer.dev',
marketing: 'http://localhost:8088/marketing',
invite: 'http://localhost:8088/invite',
gift: 'http://localhost:8088/gift',
},
},
];
describe('deriveEndpointsFromDomain', () => {
test.each(endpointScenarios)('$name', ({config, expected}) => {
expect(deriveEndpointsFromDomain(config)).toEqual(expected);
});
});
@@ -433,10 +430,9 @@ describe('parsePublicOrigin', () => {
});
test('normalizes an explicitly written standard port to the portless form', () => {
const origin = parsePublicOrigin('https://chat.example.com:443');
assert.ok(origin);
expect(origin).toEqual({public_scheme: 'https', base_domain: 'chat.example.com', public_port: 443});
expect(buildUrl(origin?.public_scheme ?? 'https', origin?.base_domain ?? '', origin?.public_port)).toBe(
'https://chat.example.com',
);
expect(buildUrl(origin.public_scheme, origin.base_domain, origin.public_port)).toBe('https://chat.example.com');
expect(parsePublicOrigin('http://chat.example.com:80')?.public_port).toBe(80);
});
test('canonicalizes the host', () => {
@@ -472,11 +468,10 @@ describe('parsePublicOrigin', () => {
describe('endpoints derived from a public origin', () => {
test('an origin with a non-standard port ports every derived endpoint', () => {
const origin = parsePublicOrigin('https://chat.example.com:29080');
assert.ok(origin);
const endpoints = deriveEndpointsFromDomain({
base_domain: origin?.base_domain ?? '',
public_scheme: origin?.public_scheme ?? 'https',
...origin,
internal_scheme: 'http',
public_port: origin?.public_port,
});
expect(endpoints.api_client).toBe('https://chat.example.com:29080/api');
expect(endpoints.app).toBe('https://chat.example.com:29080');
@@ -485,11 +480,10 @@ describe('endpoints derived from a public origin', () => {
});
test('an origin written with an explicit :443 derives portless endpoints', () => {
const origin = parsePublicOrigin('https://chat.example.com:443');
assert.ok(origin);
const endpoints = deriveEndpointsFromDomain({
base_domain: origin?.base_domain ?? '',
public_scheme: origin?.public_scheme ?? 'https',
...origin,
internal_scheme: 'http',
public_port: origin?.public_port,
});
expect(endpoints.admin).toBe('https://chat.example.com/admin');
expect(endpoints.app).toBe('https://chat.example.com');
@@ -1,17 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {readFileSync} from 'node:fs';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {loadConfig, resetConfig} from '@fluxer/config/src/ConfigLoader';
import {normalizePublicEndpoint} from '@fluxer/config/src/EndpointDerivation';
import type {MasterConfig} from '@fluxer/config/src/MasterConfig';
import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest';
const SELF_HOSTING = path.join(fileURLToPath(new URL('../../../../', import.meta.url)), 'deploy/self-hosting');
const compose = readFileSync(path.join(SELF_HOSTING, 'docker-compose.yml'), 'utf8');
const lines = compose.split('\n');
import {serviceEnvironment, serviceList, serviceNames} from './SelfHostingCompose';
const DOMAIN = 'chat.example.com';
const PUBLIC_PORT = '19080';
@@ -55,97 +49,8 @@ const ORIGIN_ONLY_ENV: Record<string, string> = {
FLUXER_HTTPS_PORT: ORIGIN_PORT,
};
const indentOf = (line: string): number => line.length - line.trimStart().length;
function unquote(value: string): string {
const trimmed = value.trim();
const quote = trimmed.slice(0, 1);
const quoted = (quote === '"' || quote === "'") && trimmed.length > 1 && trimmed.endsWith(quote);
return quoted ? trimmed.slice(1, -1) : trimmed;
}
function anchorStart(anchor: string): number {
const start = lines.findIndex((line) => line.startsWith('x-') && line.endsWith(`&${anchor}`));
if (start === -1) {
throw new Error(`docker-compose.yml has no &${anchor} anchor`);
}
return start;
}
function mapping(start: number, indent: number): Map<string, string> {
const entries = new Map<string, string>();
for (let index = start; index < lines.length; index += 1) {
const line = lines[index];
if (line.trim().length === 0 || line.trimStart().startsWith('#')) {
continue;
}
if (indentOf(line) < indent) {
break;
}
if (indentOf(line) > indent) {
continue;
}
const merge = /^<<: \*([a-z][a-z0-9-]*)$/u.exec(line.trim());
if (merge) {
for (const [key, value] of mapping(anchorStart(merge[1]) + 1, 2)) {
entries.set(key, value);
}
continue;
}
const entry = /^([A-Za-z_][A-Za-z0-9_]*): (.*)$/u.exec(line.trim());
if (entry) {
entries.set(entry[1], unquote(entry[2]));
}
}
return entries;
}
function serviceSection(service: string, key: string): number {
const start = lines.indexOf(` ${service}:`);
if (start === -1) {
throw new Error(`docker-compose.yml has no ${service} service`);
}
for (let index = start + 1; index < lines.length; index += 1) {
const line = lines[index];
if (line.trim().length > 0 && indentOf(line) <= 2) {
break;
}
if (line === ` ${key}:`) {
return index;
}
}
return -1;
}
function serviceList(service: string, key: string): Array<string> {
const start = serviceSection(service, key);
if (start === -1) {
return [];
}
const items: Array<string> = [];
for (let index = start + 1; index < lines.length; index += 1) {
const line = lines[index];
if (line.trim().length === 0) {
continue;
}
if (indentOf(line) <= 4) {
break;
}
const item = /^- (.*)$/u.exec(line.trim());
if (item) {
items.push(unquote(item[1]));
}
}
return items;
}
const SERVICES_WITHOUT_ENDPOINT_REPAIR = new Set(['edge']);
function serviceNames(): Array<string> {
const section = compose.slice(compose.indexOf('\nservices:\n'), compose.indexOf('\nnetworks:\n'));
return [...section.matchAll(/\n {2}([a-z][a-z0-9_-]*):\n/gu)].map(([, name]) => name);
}
function closingBrace(value: string, open: number): number {
let depth = 0;
for (let index = open; index < value.length; index += 1) {
@@ -196,7 +101,7 @@ function expand(value: string, env: Record<string, string>): string {
function expandedEnvironment(service: string, env: Record<string, string>): Record<string, string> {
const expanded: Record<string, string> = {};
for (const [key, value] of mapping(serviceSection(service, 'environment') + 1, 6)) {
for (const [key, value] of Object.entries(serviceEnvironment(service))) {
expanded[key] = expand(value, env);
}
return expanded;
@@ -257,7 +162,7 @@ function browserFacingUrls(config: MasterConfig): Array<[string, string]> {
return entries.filter(([, value]) => publicUrl(value) !== null);
}
async function loadApiConfig(env: Record<string, string>): Promise<MasterConfig | Error> {
async function loadApiConfig(env: Record<string, string>): Promise<MasterConfig> {
for (const key of Object.keys(process.env)) {
if (key.startsWith('FLUXER_')) {
vi.stubEnv(key, undefined);
@@ -268,16 +173,31 @@ async function loadApiConfig(env: Record<string, string>): Promise<MasterConfig
vi.stubEnv(key, value);
}
}
try {
return await loadConfig();
} catch (error) {
return error instanceof Error ? error : new Error(String(error));
}
return loadConfig();
}
describe('Compose environment interpolation', () => {
test.each([
[`\${VALUE}`, {VALUE: 'set'}, 'set'],
['$VALUE/path', {VALUE: 'set'}, 'set/path'],
[`\${VALUE:-fallback}`, {VALUE: ''}, 'fallback'],
[`\${VALUE:-\${OTHER:-fallback}}`, {OTHER: 'nested'}, 'nested'],
[`\${VALUE:+alternate}`, {VALUE: 'set'}, 'alternate'],
[`\${VALUE:+alternate}`, {}, ''],
[`\${VALUE:?required}`, {VALUE: 'set'}, 'set'],
] as const)('expands %s with %j to %s', (source, environment, expected) => {
expect(expand(source, environment)).toBe(expected);
});
test('rejects a missing required value and unbalanced interpolation', () => {
expect(() => expand(`\${VALUE:?required}`, {})).toThrow('VALUE is required: required');
expect(() => expand('${VALUE:-fallback', {})).toThrow('unbalanced interpolation');
});
});
describe('the shipped compose stack expanded on a non-default port', () => {
test('every service handed a public URL is handed the base domain and port that repair it', () => {
const starved = serviceNames()
const starved = serviceNames
.filter((service) => !SERVICES_WITHOUT_ENDPOINT_REPAIR.has(service))
.filter((service) => {
const environment = expandedEnvironment(service, PORT_ONLY_ENV);
@@ -289,9 +209,10 @@ describe('the shipped compose stack expanded on a non-default port', () => {
});
test('every public URL the stack hands a browser carries the port', () => {
const entries = serviceNames()
const entries = serviceNames
.filter((service) => !SERVICES_WITHOUT_ENDPOINT_REPAIR.has(service))
.flatMap((service) => repairedPublicUrls(service, PORT_ONLY_ENV));
expect(entries.length).toBeGreaterThan(0);
expect(withoutPort(entries, PUBLIC_PORT)).toEqual([]);
});
@@ -319,23 +240,17 @@ describe('a public origin carrying a port while FLUXER_PUBLIC_PORT stays standar
test('the compose overrides all carry the origin port', () => {
const environment = expandedEnvironment('api', ORIGIN_ONLY_ENV);
const entries = publicUrlNames(environment).map((name): [string, string] => [name, environment[name]]);
expect(entries.length).toBeGreaterThan(0);
expect(withoutPort(entries, ORIGIN_PORT)).toEqual([]);
});
test('the loaded config either ports every public URL or refuses the split', async () => {
test('the loaded config ports every public URL using the explicit origin', async () => {
const loaded = await loadApiConfig(ORIGIN_ONLY_ENV);
if (loaded instanceof Error) {
expect(loaded.message).not.toBe('');
return;
}
expect(withoutPort(browserFacingUrls(loaded), ORIGIN_PORT)).toEqual([]);
});
test('the port-only recipe ports every public URL the loaded config exposes', async () => {
const loaded = await loadApiConfig(PORT_ONLY_ENV);
if (loaded instanceof Error) {
throw loaded;
}
expect(withoutPort(browserFacingUrls(loaded), PUBLIC_PORT)).toEqual([]);
});
});
@@ -0,0 +1,55 @@
import assert from 'node:assert/strict';
import {readFileSync} from 'node:fs';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {type ConfigObject, isConfigObject} from '@fluxer/config/src/config_loader/ConfigObject';
import {parse} from 'yaml';
const SELF_HOSTING = fileURLToPath(new URL('../../../../deploy/self-hosting/', import.meta.url));
function readSelfHostingFile(name: string): string {
return readFileSync(path.join(SELF_HOSTING, name), 'utf8');
}
function mapping(value: unknown, name: string): ConfigObject {
assert.ok(isConfigObject(value), `${name} must be a mapping`);
return value;
}
const compose = mapping(parse(readSelfHostingFile('docker-compose.yml'), {merge: true}), 'docker-compose.yml');
const services = mapping(compose.services, 'services');
export const serviceNames = Object.keys(services);
assert.ok(serviceNames.length > 0, 'docker-compose.yml must define services');
export function composeService(name: string): ConfigObject {
return mapping(services[name], `services.${name}`);
}
function environment(value: unknown, name: string): Record<string, string> {
return Object.fromEntries(
Object.entries(mapping(value, name)).map(([key, entry]) => {
assert.ok(
typeof entry === 'string' || typeof entry === 'number' || typeof entry === 'boolean',
`${name}.${key} must have an explicit scalar value`,
);
return [key, String(entry)];
}),
);
}
export function serviceEnvironment(name: string): Record<string, string> {
const value = composeService(name).environment;
return value === undefined ? {} : environment(value, `services.${name}.environment`);
}
export const sharedEnvironment = environment(compose['x-fluxer-env'], 'x-fluxer-env');
export function serviceList(name: string, key: string): Array<string> {
const value = composeService(name)[key];
assert.ok(Array.isArray(value), `services.${name}.${key} must be a list`);
assert.ok(
value.every((item) => typeof item === 'string'),
`services.${name}.${key} must contain strings`,
);
return value;
}
@@ -1,38 +1,28 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {readFileSync} from 'node:fs';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {describe, expect, test} from 'vitest';
const SELF_HOSTING = path.join(fileURLToPath(new URL('../../../../', import.meta.url)), 'deploy/self-hosting');
const compose = readFileSync(path.join(SELF_HOSTING, 'docker-compose.yml'), 'utf8');
const serviceBlock = (name: string): string => {
const start = compose.indexOf(`\n ${name}:\n`);
if (start === -1) {
throw new Error(`docker-compose.yml has no ${name} service`);
}
const rest = compose.slice(start + 1);
const next = rest.slice(1).search(/\n {2}[a-z][a-z0-9_-]*:\n/u);
return next === -1 ? rest : rest.slice(0, next + 1);
};
const sharedEnv = compose.slice(compose.indexOf('x-fluxer-env: &fluxer-env'), compose.indexOf('\nx-fluxer-service:'));
import {serviceEnvironment, serviceNames, sharedEnvironment} from './SelfHostingCompose';
describe('the shipped compose stack wires every service it starts', () => {
test('no service sizes a pool for a connection it cannot make', () => {
for (const [, name] of compose.matchAll(/\n {2}([a-z][a-z0-9_-]*):\n/gu)) {
const block = serviceBlock(name);
if (block.includes('FLUXER_POSTGRES_MAX_CONNECTIONS')) {
expect(block).toMatch(/<<: \*fluxer-(postgres-)?env/u);
}
const pooledServices = serviceNames.filter((name) => 'FLUXER_POSTGRES_MAX_CONNECTIONS' in serviceEnvironment(name));
expect(pooledServices.length).toBeGreaterThan(0);
for (const name of pooledServices) {
expect(serviceEnvironment(name), name).toMatchObject({
FLUXER_DATABASE_BACKEND: 'postgres',
FLUXER_POSTGRES_HOST: expect.stringMatching(/\S/u),
FLUXER_POSTGRES_PORT: expect.stringMatching(/\S/u),
FLUXER_POSTGRES_DATABASE: expect.stringMatching(/\S/u),
FLUXER_POSTGRES_USERNAME: expect.stringMatching(/\S/u),
FLUXER_POSTGRES_PASSWORD: expect.stringMatching(/\S/u),
});
}
});
test('the shared block sets the client-IP trust the merged services read', () => {
expect(sharedEnv).toContain('FLUXER_TRUST_CLIENT_IP_HEADER: "${FLUXER_TRUST_CLIENT_IP_HEADER:-true}"');
expect(sharedEnv).toContain('FLUXER_CLIENT_IP_HEADER_NAME: ${FLUXER_CLIENT_IP_HEADER_NAME:-x-forwarded-for}');
expect(sharedEnvironment).toMatchObject({
FLUXER_TRUST_CLIENT_IP_HEADER: `\${FLUXER_TRUST_CLIENT_IP_HEADER:-true}`,
FLUXER_CLIENT_IP_HEADER_NAME: `\${FLUXER_CLIENT_IP_HEADER_NAME:-x-forwarded-for}`,
});
});
});
@@ -1,28 +1,15 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {readFileSync} from 'node:fs';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {describe, expect, test} from 'vitest';
const SELF_HOSTING = path.join(fileURLToPath(new URL('../../../../', import.meta.url)), 'deploy/self-hosting');
const compose = readFileSync(path.join(SELF_HOSTING, 'docker-compose.yml'), 'utf8');
const serviceBlock = (name: string): string => {
const start = compose.indexOf(`\n ${name}:\n`);
if (start === -1) {
throw new Error(`docker-compose.yml has no ${name} service`);
}
const rest = compose.slice(start + 1);
const next = rest.slice(1).search(/\n {2}[a-z][a-z0-9_-]*:\n/u);
return next === -1 ? rest : rest.slice(0, next + 1);
};
import {composeService, serviceEnvironment, serviceList} from './SelfHostingCompose';
describe('the shipped object store checks the credentials the stack sends', () => {
const init = serviceBlock('seaweedfs-init');
const entrypoint = serviceList('seaweedfs-init', 'entrypoint');
const init = entrypoint[2];
test('seaweedfs-init applies an S3 identity built from the .env credentials', () => {
expect(entrypoint.slice(0, 2)).toEqual(['/bin/sh', '-c']);
expect(entrypoint).toHaveLength(3);
expect(init).toContain(
's3.configure -user=fluxer -access_key=$$FLUXER_S3_ACCESS_KEY -secret_key=$$FLUXER_S3_SECRET_KEY',
);
@@ -31,7 +18,7 @@ describe('the shipped object store checks the credentials the stack sends', () =
test('seaweedfs-init is handed the same credentials the api requires', () => {
for (const name of ['FLUXER_S3_ACCESS_KEY', 'FLUXER_S3_SECRET_KEY']) {
expect(init).toMatch(new RegExp(`${name}: \\$\\{${name}:\\?set ${name} in \\.env\\}`, 'u'));
expect(serviceEnvironment('seaweedfs-init')[name]).toBe(`\${${name}:?set ${name} in .env}`);
}
});
@@ -40,17 +27,20 @@ describe('the shipped object store checks the credentials the stack sends', () =
const ready = init.indexOf('echo "buckets ready"');
expect(configure).toBeGreaterThan(-1);
expect(ready).toBeGreaterThan(configure);
expect(init).toContain('echo "seaweedfs-init could not configure the S3 identity" >&2');
expect(init).toContain('exit 1');
expect(init).toMatch(
/if ! echo "s3\.configure [^"]+"[^;]+; then\s+echo "seaweedfs-init could not configure the S3 identity" >&2;\s+exit 1;\s+fi;/u,
);
});
test('media-proxy signs its reads, which the store now refuses to serve unsigned', () => {
expect(serviceBlock('media-proxy')).toContain('FLUXER_S3_READ_SIGNED: "true"');
expect(serviceEnvironment('media-proxy').FLUXER_S3_READ_SIGNED).toBe('true');
});
test('every service that reaches the store waits for the identity to exist', () => {
for (const name of ['api', 'worker', 'media-proxy']) {
expect(serviceBlock(name)).toContain('seaweedfs-init: {condition: service_completed_successfully}');
expect(composeService(name).depends_on, name).toMatchObject({
'seaweedfs-init': {condition: 'service_completed_successfully'},
});
}
});
});
@@ -0,0 +1,5 @@
export type ConfigObject = Record<string, unknown>;
export function isConfigObject(value: unknown): value is ConfigObject {
return value !== null && typeof value === 'object' && !Array.isArray(value);
}
@@ -1,7 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {type ConfigObject, isConfigObject} from '@fluxer/config/src/config_loader/ConfigObject';
type ConfigPathKey = string | number;
type ConfigObject = Record<string, unknown>;
type ConfigContainer = ConfigObject | Array<unknown>;
type EnvValueParser = (raw: string) => unknown;
@@ -436,12 +437,8 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
FLUXER_GEOIP_DB_PATH: {path: ['geoip', 'maxmind_db_path']},
};
function isPlainObject(value: unknown): value is ConfigObject {
return value !== null && typeof value === 'object' && !Array.isArray(value);
}
function isContainer(value: unknown): value is ConfigContainer {
return isPlainObject(value) || Array.isArray(value);
return isConfigObject(value) || Array.isArray(value);
}
function createChildContainer(nextKey: ConfigPathKey | undefined): ConfigContainer {