pepepper/fluxer
1
0
Fork 0
mirror of https://github.com/fluxerapp/fluxer synced 2026-10-07 19:22:14 +09:00
Code Issues Packages Projects Releases Wiki Activity

feat(api): publish object storage changes to a JetStream feed (#2796)

Browse Source
This commit is contained in:
Hampus
2026-09-15 17:20:26 +02:00
committed by GitHub
parent 83c8e91955
commit b38e7c6433
13 changed files with 905 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/config/src/ConfigLoader.ts
+17
View File
@@ -122,6 +122,10 @@ function defaultConfig(): MasterConfig {
content_moderation: {
nsfw_threshold: 0.7,
},
storage_change_feed: {
enabled: false,
stream: 'STORAGE_CHANGES',
},
},
nats: {
core_url: 'nats://127.0.0.1:4222',
@@ -453,6 +457,18 @@ function validateApiWorkerConfig(config: MasterConfig): void {
}
}
function validateStorageChangeFeedConfig(config: MasterConfig): void {
const feed = config.services.api?.storage_change_feed;
if (!feed?.enabled) {
return;
}
if (feed.stream === undefined || !/^[A-Za-z0-9_-]+$/u.test(feed.stream)) {
throw new Error(
'FLUXER_API_STORAGE_CHANGE_FEED_STREAM must be letters, digits, underscores or hyphens when the storage change feed is enabled',
);
}
}
function validateCachePurgeConfig(config: MasterConfig): void {
const cachePurge = config.integrations.cache_purge;
if (cachePurge.adapter !== 'http') {
@@ -541,6 +557,7 @@ function normalizeConfig(config: MasterConfig): MasterConfig {
validatePostgresConfig(config);
validateCaptchaConfig(config);
validateApiWorkerConfig(config);
validateStorageChangeFeedConfig(config);
validateCachePurgeConfig(config);
assertIntegerInRange(config.services.api.max_inflight_requests, 'FLUXER_API_MAX_INFLIGHT_REQUESTS', 1, 100_000);
assertIntegerInRange(config.services.api.headers_timeout_ms, 'FLUXER_API_HEADERS_TIMEOUT_MS', 1_000, 3_600_000);
packages/config/src/MasterConfig.ts
+5
View File
@@ -127,6 +127,11 @@ export interface MasterConfig {
batch?: number;
};
};
storage_change_feed?: {
enabled?: boolean;
stream?: string;
skip_buckets?: Array<string>;
};
};
nats?: {
core_url?: string;
packages/config/src/__tests__/ConfigLoader.test.ts
+28
View File
@@ -309,6 +309,34 @@ describe('ConfigLoader', () => {
await expect(loadConfig()).rejects.toThrow('FLUXER_API_WORKER_TASK');
});
test('leaves the storage change feed disabled by default', async () => {
stubMinimalEnv();
const config = await loadConfig();
expect(config.services.api.storage_change_feed).toEqual({enabled: false, stream: 'STORAGE_CHANGES'});
});
test('maps the storage change feed environment variables', async () => {
stubMinimalEnv({
FLUXER_API_STORAGE_CHANGE_FEED_ENABLED: 'true',
FLUXER_API_STORAGE_CHANGE_FEED_STREAM: 'BACKUP_CHANGES',
FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS: 'fluxer-uploads, fluxer-harvests',
});
const config = await loadConfig();
expect(config.services.api.storage_change_feed).toEqual({
enabled: true,
stream: 'BACKUP_CHANGES',
skip_buckets: ['fluxer-uploads', 'fluxer-harvests'],
});
});
test('rejects a storage change feed stream name that JetStream cannot use', async () => {
stubMinimalEnv({
FLUXER_API_STORAGE_CHANGE_FEED_ENABLED: 'true',
FLUXER_API_STORAGE_CHANGE_FEED_STREAM: 'storage.changes',
});
await expect(loadConfig()).rejects.toThrow('FLUXER_API_STORAGE_CHANGE_FEED_STREAM');
});
test('rejects invalid Postgres typed environment values', async () => {
stubMinimalEnv({FLUXER_POSTGRES_PORT: 'abc'});
await expect(loadConfig()).rejects.toThrow('FLUXER_POSTGRES_PORT');
packages/config/src/config_loader/EnvironmentOverrides.ts
+9
View File
@@ -112,6 +112,15 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
path: ['services', 'api', 'worker', 'lane_concurrency_overrides'],
parse: parseJsonObject,
},
FLUXER_API_STORAGE_CHANGE_FEED_ENABLED: {
path: ['services', 'api', 'storage_change_feed', 'enabled'],
parse: parseBoolean,
},
FLUXER_API_STORAGE_CHANGE_FEED_STREAM: {path: ['services', 'api', 'storage_change_feed', 'stream']},
FLUXER_API_STORAGE_CHANGE_FEED_SKIP_BUCKETS: {
path: ['services', 'api', 'storage_change_feed', 'skip_buckets'],
parse: parseCsv,
},
FLUXER_API_UNFURL_IGNORED_HOSTS: {path: ['services', 'api', 'unfurl_ignored_hosts'], parse: parseCsv},
FLUXER_API_EMBEDS_OEMBED_HTML_ENABLED: {
path: ['services', 'api', 'embeds', 'oembed_html_enabled'],