mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-10 20:52:12 +09:00
317 lines
12 KiB
TypeScript
317 lines
12 KiB
TypeScript
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
import {Config} from '@app/api/Config';
|
|
import {setDatabaseQueryExecutor} from '@app/api/database/CassandraQueryExecution';
|
|
import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';
|
|
import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService';
|
|
import {shutdownStorageChangeFeed} from '@app/api/infrastructure/StorageServiceFactory';
|
|
import type {InstanceConfigRepository} from '@app/api/instance/InstanceConfigRepository';
|
|
import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
|
|
import {Logger} from '@app/api/Logger';
|
|
import type {LimitConfigService} from '@app/api/limits/LimitConfigService';
|
|
import {
|
|
closeOwnedKVClient,
|
|
createSnowflakeService,
|
|
setInjectedSnowflakeService,
|
|
setInjectedWorkerService,
|
|
shutdownVoiceResources,
|
|
} from '@app/api/middleware/ServiceRegistry';
|
|
import {
|
|
getCacheService,
|
|
getInstanceConfigRepository,
|
|
getLimitConfigService,
|
|
} from '@app/api/middleware/ServiceSingletons';
|
|
import {initializeSearch, shutdownSearch} from '@app/api/SearchFactory';
|
|
import {awaitAll} from '@app/api/utils/ConcurrencyUtils';
|
|
import {CronScheduler} from '@app/api/worker/CronScheduler';
|
|
import {JetStreamWorkerQueue} from '@app/api/worker/JetStreamWorkerQueue';
|
|
import {clearWorkerDependencies, setWorkerDependencies} from '@app/api/worker/WorkerContext';
|
|
import {initializeWorkerDependencies, type WorkerDependencies} from '@app/api/worker/WorkerDependencies';
|
|
import {WorkerHeartbeat} from '@app/api/worker/WorkerHeartbeat';
|
|
import {
|
|
resolveCronSchedulerEnabled,
|
|
resolveWorkerLanes,
|
|
validateLaneCompleteness,
|
|
} from '@app/api/worker/WorkerLaneConfig';
|
|
import {createWorkerProcessErrorHandler} from '@app/api/worker/WorkerProcessErrorHandler';
|
|
import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError';
|
|
import {WorkerRunner} from '@app/api/worker/WorkerRunner';
|
|
import {WorkerService} from '@app/api/worker/WorkerService';
|
|
import {workerTasks} from '@app/api/worker/WorkerTaskRegistry';
|
|
import {setupGracefulShutdown} from '@fluxer/hono/src/Server';
|
|
import {BACKGROUND_READ_TIMEOUT_MS, initCassandra, shutdownCassandra} from '@pkgs/cassandra/src/Client';
|
|
import {JetStreamConnectionManager} from '@pkgs/nats/src/JetStreamConnectionManager';
|
|
import {getDefaultPostgresClient, initPostgres, shutdownPostgres} from '@pkgs/postgres/src/Client';
|
|
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
|
import {ms} from 'itty-time';
|
|
|
|
function registerCronJobs(cron: CronScheduler): void {
|
|
cron.upsert('processAssetDeletionQueue', 'processAssetDeletionQueue', {}, '0 */5 * * * *', {ledger: false});
|
|
if (Config.cachePurge.adapter !== 'none') {
|
|
cron.upsert('processCachePurgeQueue', 'processCachePurgeQueue', {}, '*/10 * * * * *', {ledger: false});
|
|
}
|
|
cron.upsert('processPendingBulkMessageDeletions', 'processPendingBulkMessageDeletions', {}, '0 */10 * * * *', {
|
|
ledger: false,
|
|
});
|
|
cron.upsert('userProcessPendingDeletions', 'userProcessPendingDeletions', {}, '0 * * * * *', {ledger: false});
|
|
cron.upsert('processPremiumStateReconciliationQueue', 'processPremiumStateReconciliationQueue', {}, '0 * * * * *', {
|
|
ledger: false,
|
|
});
|
|
if (!Config.instance.selfHosted) {
|
|
cron.upsert('processExpiredPremiumSweep', 'processExpiredPremiumSweep', {}, '0 0 * * * *', {ledger: false});
|
|
}
|
|
cron.upsert('processInactivityDeletions', 'processInactivityDeletions', {}, '0 0 */6 * * *', {ledger: false});
|
|
cron.upsert('expireAttachments', 'expireAttachments', {}, '0 0 */12 * * *', {ledger: false});
|
|
cron.upsert('prunePostgresKvTtl', 'prunePostgresKvTtl', {}, '0 */5 * * * *', {ledger: false});
|
|
cron.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *', {ledger: false});
|
|
if (Config.blocklistFeeds.enabled) {
|
|
cron.upsert('syncDisposableEmailDomains', 'syncDisposableEmailDomains', {}, '0 0 */6 * * *', {ledger: true});
|
|
cron.upsert('syncUrlBlocklists', 'syncUrlBlocklists', {}, '0 0 */6 * * *', {ledger: true});
|
|
cron.upsert('syncFileShaBlocklists', 'syncFileShaBlocklists', {}, '0 0 */12 * * *', {ledger: true});
|
|
}
|
|
cron.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/10 * * * * *', {ledger: false});
|
|
Logger.info(
|
|
{
|
|
blocklistFeeds: Config.blocklistFeeds.enabled,
|
|
cachePurgeAdapter: Config.cachePurge.adapter,
|
|
selfHosted: Config.instance.selfHosted,
|
|
},
|
|
'Cron jobs registered successfully',
|
|
);
|
|
}
|
|
|
|
export async function startWorkerMain(): Promise<void> {
|
|
Logger.info('Starting worker backend...');
|
|
let cassandraInitialized = false;
|
|
let postgresInitialized = false;
|
|
let jsConnectionManager: JetStreamConnectionManager | null = null;
|
|
let snowflakeService: ISnowflakeService | null = null;
|
|
let limitConfigService: LimitConfigService | null = null;
|
|
let instanceConfigRepository: InstanceConfigRepository | null = null;
|
|
let dependencies: WorkerDependencies | null = null;
|
|
let cron: CronScheduler | null = null;
|
|
const heartbeat = new WorkerHeartbeat({logger: Logger});
|
|
const runners: Array<WorkerRunner> = [];
|
|
let searchInitialized = false;
|
|
let shutdownPromise: Promise<void> | null = null;
|
|
|
|
const cleanupStep = async (label: string, fn: () => Promise<void> | void): Promise<void> => {
|
|
try {
|
|
await fn();
|
|
} catch (error) {
|
|
Logger.error({err: error}, `Error during worker shutdown step: ${label}`);
|
|
}
|
|
};
|
|
|
|
const performShutdown = async (): Promise<void> => {
|
|
Logger.info('Shutting down worker backend...');
|
|
const voiceShutdown = cleanupStep('voice resources', shutdownVoiceResources);
|
|
await cleanupStep('heartbeat', () => heartbeat.stop());
|
|
await cleanupStep('cron', () => cron?.stop());
|
|
await cleanupStep('runners', async () => {
|
|
await awaitAll(
|
|
runners.map((runner) => runner.stop()),
|
|
'Failed to stop worker runners',
|
|
);
|
|
});
|
|
await voiceShutdown;
|
|
await cleanupStep('storage change feed', shutdownStorageChangeFeed);
|
|
await cleanupStep('jetstream', async () => {
|
|
await jsConnectionManager?.drain();
|
|
jsConnectionManager = null;
|
|
});
|
|
await cleanupStep('worker dependencies', () => {
|
|
dependencies = null;
|
|
clearWorkerDependencies();
|
|
setInjectedWorkerService(undefined);
|
|
});
|
|
await cleanupStep('limit config', async () => {
|
|
if (limitConfigService) {
|
|
await limitConfigService.shutdown();
|
|
limitConfigService = null;
|
|
}
|
|
});
|
|
await cleanupStep('instance config repository', async () => {
|
|
if (instanceConfigRepository) {
|
|
await instanceConfigRepository.shutdown();
|
|
instanceConfigRepository = null;
|
|
}
|
|
});
|
|
await cleanupStep('search', async () => {
|
|
if (searchInitialized) {
|
|
await shutdownSearch();
|
|
searchInitialized = false;
|
|
}
|
|
});
|
|
await cleanupStep('cassandra', async () => {
|
|
if (cassandraInitialized) {
|
|
await shutdownCassandra();
|
|
cassandraInitialized = false;
|
|
}
|
|
});
|
|
await cleanupStep('postgres', async () => {
|
|
if (postgresInitialized) {
|
|
setDatabaseQueryExecutor(null);
|
|
await shutdownPostgres();
|
|
postgresInitialized = false;
|
|
}
|
|
});
|
|
await cleanupStep('snowflake', async () => {
|
|
if (snowflakeService) {
|
|
await snowflakeService.shutdown();
|
|
snowflakeService = null;
|
|
}
|
|
setInjectedSnowflakeService(undefined);
|
|
});
|
|
await cleanupStep('key-value client', closeOwnedKVClient);
|
|
};
|
|
const shutdown = (): Promise<void> => {
|
|
shutdownPromise ??= Promise.resolve().then(performShutdown);
|
|
return shutdownPromise;
|
|
};
|
|
|
|
try {
|
|
if (Config.database.backend === 'postgres') {
|
|
await initPostgres(Config.postgres, (diagnostic) => Logger.error(diagnostic, 'Postgres connection error'));
|
|
postgresInitialized = true;
|
|
const postgres = getDefaultPostgresClient();
|
|
await ensurePostgresKvSchema(postgres);
|
|
setDatabaseQueryExecutor(new PostgresKvQueryExecutor(postgres));
|
|
Logger.info('Postgres KV client initialised for worker backend');
|
|
}
|
|
if (Config.database.backend === 'cassandra') {
|
|
await initCassandra({
|
|
hosts: Config.cassandra.hosts.split(',').filter(Boolean),
|
|
port: Config.cassandra.port,
|
|
keyspace: Config.cassandra.keyspace,
|
|
localDc: Config.cassandra.localDc,
|
|
username: Config.cassandra.username || undefined,
|
|
password: Config.cassandra.password || undefined,
|
|
readTimeoutMs: BACKGROUND_READ_TIMEOUT_MS,
|
|
});
|
|
cassandraInitialized = true;
|
|
Logger.info('Cassandra client initialised for worker backend');
|
|
}
|
|
validateLaneCompleteness(workerTasks);
|
|
Logger.info('Worker lane configuration validated');
|
|
const activeWorkerLanes = resolveWorkerLanes({
|
|
mode: Config.worker.mode,
|
|
laneName: Config.worker.laneName,
|
|
taskName: Config.worker.taskName,
|
|
laneConcurrencyOverrides: Config.worker.laneConcurrencyOverrides,
|
|
});
|
|
const cronSchedulerEnabled = resolveCronSchedulerEnabled(Config.worker.mode, Config.worker.enableCronScheduler);
|
|
Logger.info(
|
|
{
|
|
workerMode: Config.worker.mode,
|
|
workerLane: Config.worker.laneName,
|
|
workerTask: Config.worker.taskName,
|
|
lanes: activeWorkerLanes.map((lane) => `${lane.name}(${lane.concurrency})`).join(', '),
|
|
cronSchedulerEnabled,
|
|
},
|
|
'Worker runtime configuration resolved',
|
|
);
|
|
snowflakeService = createSnowflakeService();
|
|
await snowflakeService.initialize();
|
|
setInjectedSnowflakeService(snowflakeService);
|
|
Logger.info('Shared SnowflakeService initialised');
|
|
jsConnectionManager = new JetStreamConnectionManager({
|
|
url: Config.nats.jetStreamUrl,
|
|
token: Config.nats.authToken || undefined,
|
|
name: 'fluxer-worker',
|
|
});
|
|
await jsConnectionManager.connect();
|
|
Logger.info('JetStream connection established');
|
|
const queue = new JetStreamWorkerQueue(jsConnectionManager);
|
|
await queue.ensureInfrastructure(activeWorkerLanes);
|
|
Logger.info('JetStream stream and lane consumers verified');
|
|
const jobLedger = new JobLedgerRepository();
|
|
const workerService = new WorkerService(queue, snowflakeService, jobLedger);
|
|
setInjectedWorkerService(workerService);
|
|
instanceConfigRepository = getInstanceConfigRepository();
|
|
limitConfigService = getLimitConfigService();
|
|
try {
|
|
await initializeSearch(getCacheService());
|
|
searchInitialized = true;
|
|
Logger.info('Search initialised for worker backend');
|
|
} catch (error) {
|
|
Logger.error({err: error}, 'Search initialisation failed for worker backend');
|
|
throw error;
|
|
}
|
|
dependencies = await initializeWorkerDependencies(snowflakeService);
|
|
setWorkerDependencies(dependencies);
|
|
if (Config.blocklistFeeds.enabled) {
|
|
const didClaimEmailSync = await dependencies.kvClient.setnx(
|
|
'sync:email_domains:initialized',
|
|
'1',
|
|
ms('6 hours') / 1000,
|
|
);
|
|
if (didClaimEmailSync) {
|
|
Logger.info('Triggering initial disposable email domain sync');
|
|
try {
|
|
await workerService.addJob('syncDisposableEmailDomains', {});
|
|
} catch (error) {
|
|
if (!(error instanceof WorkerQueueOverflowError)) {
|
|
throw error;
|
|
}
|
|
Logger.warn('Dropped initial disposable email domain sync, jobs stream is at its limit');
|
|
}
|
|
}
|
|
}
|
|
cron = new CronScheduler(workerService, Logger, dependencies.kvClient, heartbeat);
|
|
registerCronJobs(cron);
|
|
for (const lane of activeWorkerLanes) {
|
|
const laneTasks: Record<string, WorkerTaskHandler> = {};
|
|
for (const taskType of lane.taskTypes) {
|
|
const handler = workerTasks[taskType];
|
|
if (handler) {
|
|
laneTasks[taskType] = handler;
|
|
}
|
|
}
|
|
const runner = new WorkerRunner({
|
|
tasks: laneTasks,
|
|
retiredTaskTypes: lane.retiredTaskTypes,
|
|
queue,
|
|
consumerName: lane.consumerName,
|
|
laneName: lane.name,
|
|
ledger: jobLedger,
|
|
concurrency: lane.concurrency,
|
|
maxDeliver: lane.maxDeliver,
|
|
ackWaitMs: lane.ackWaitMs,
|
|
heartbeat,
|
|
});
|
|
runners.push(runner);
|
|
}
|
|
if (cronSchedulerEnabled) {
|
|
cron.start();
|
|
Logger.info('Cron scheduler started');
|
|
} else {
|
|
Logger.info('Cron scheduler disabled for this worker process');
|
|
}
|
|
await Promise.all(runners.map((runner) => runner.start()));
|
|
Logger.info(
|
|
{lanes: activeWorkerLanes.map((l) => `${l.name}(${l.concurrency})`).join(', ')},
|
|
'Worker runners started',
|
|
);
|
|
heartbeat.start();
|
|
setupGracefulShutdown(shutdown, {logger: Logger, timeoutMs: 30000});
|
|
const handleProcessError = createWorkerProcessErrorHandler({
|
|
logger: Logger,
|
|
shutdown,
|
|
exit: (code) => {
|
|
process.exit(code);
|
|
},
|
|
});
|
|
process.on('uncaughtException', (error) => {
|
|
void handleProcessError('uncaughtException', error);
|
|
});
|
|
process.on('unhandledRejection', (reason: unknown) => {
|
|
void handleProcessError('unhandledRejection', reason);
|
|
});
|
|
} catch (error: unknown) {
|
|
Logger.error({err: error}, 'Failed to start worker backend');
|
|
await shutdown();
|
|
process.exit(1);
|
|
}
|
|
}
|