pepepper/fluxer
1
0 Fork 0
mirror of https://github.com/fluxerapp/fluxer synced 2026-10-18 13:35:42 +09:00
Code Issues Packages Projects Releases Wiki Activity

fix(api): eliminate idle Postgres I/O on self-hosted instances (#1593)

Browse Source
This commit is contained in:
Hampus authored and GitHub committed 2026-08-14 19:55:53 +02:00
1 parent 8a9b12e6a1
commit beb906753f
19 files changed
+80 -268
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

fluxer_api/src/api/worker/CronScheduler.ts
+10 -2
View File
@@ -11,6 +11,7 @@ interface CronDefinition {
taskType: WorkerTaskName;
payload: WorkerJobPayload;
cronExpression: string;
ledger: boolean;
lastFired: number;
}
@@ -89,12 +90,19 @@ export class CronScheduler {
this.kvClient = kvClient;
}
upsert(id: string, taskType: WorkerTaskName, payload: WorkerJobPayload, cronExpression: string): void {
upsert(
id: string,
taskType: WorkerTaskName,
payload: WorkerJobPayload,
cronExpression: string,
options: {ledger: boolean},
): void {
this.definitions.set(id, {
id,
taskType,
payload,
cronExpression,
ledger: options.ledger,
lastFired: 0,
});
}
@@ -133,7 +141,7 @@ export class CronScheduler {
if (!acquired) {
continue;
}
await this.workerService.addJob(def.taskType, def.payload, {jobKey});
await this.workerService.addJob(def.taskType, def.payload, {jobKey, skipLedger: !def.ledger});
this.logger.debug({cronId: def.id, taskType: def.taskType}, 'Cron job fired');
} catch (error) {
this.logger.error({err: error, cronId: def.id, taskType: def.taskType}, 'Failed to enqueue cron job');
fluxer_api/src/api/worker/WorkerMain.ts
+31 -19
View File
@@ -41,21 +41,27 @@ const SEARCH_REQUIRED_TASKS = new Set<string>([
]);
function registerCronJobs(cron: CronScheduler): void {
cron.upsert('processAssetDeletionQueue', 'processAssetDeletionQueue', {}, '0 */5 * * * *');
cron.upsert('processBunnyPurgeQueue', 'processBunnyPurgeQueue', {}, '*/10 * * * * *');
cron.upsert('processPendingBulkMessageDeletions', 'processPendingBulkMessageDeletions', {}, '0 */10 * * * *');
cron.upsert('userProcessPendingDeletions', 'userProcessPendingDeletions', {}, '0 * * * * *');
cron.upsert('processPremiumStateReconciliationQueue', 'processPremiumStateReconciliationQueue', {}, '0 * * * * *');
cron.upsert('processExpiredPremiumSweep', 'processExpiredPremiumSweep', {}, '0 0 * * * *');
cron.upsert('processInactivityDeletions', 'processInactivityDeletions', {}, '0 0 */6 * * *');
cron.upsert('expireAttachments', 'expireAttachments', {}, '0 0 */12 * * *');
cron.upsert('prunePostgresKvTtl', 'prunePostgresKvTtl', {}, '0 */5 * * * *');
cron.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *');
cron.upsert('syncDisposableEmailDomains', 'syncDisposableEmailDomains', {}, '0 */30 * * * *');
cron.upsert('syncUrlBlocklists', 'syncUrlBlocklists', {}, '0 0 */6 * * *');
cron.upsert('syncFileShaBlocklists', 'syncFileShaBlocklists', {}, '0 0 */12 * * *');
cron.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/10 * * * * *');
Logger.info('Cron jobs registered successfully');
cron.upsert('processAssetDeletionQueue', 'processAssetDeletionQueue', {}, '0 */5 * * * *', {ledger: false});
cron.upsert('processBunnyPurgeQueue', 'processBunnyPurgeQueue', {}, '*/10 * * * * *', {ledger: false});
cron.upsert('processPendingBulkMessageDeletions', 'processPendingBulkMessageDeletions', {}, '0 */10 * * * *', {
ledger: false,
});
cron.upsert('userProcessPendingDeletions', 'userProcessPendingDeletions', {}, '0 * * * * *', {ledger: false});
cron.upsert('processPremiumStateReconciliationQueue', 'processPremiumStateReconciliationQueue', {}, '0 * * * * *', {
ledger: false,
});
cron.upsert('processExpiredPremiumSweep', 'processExpiredPremiumSweep', {}, '0 0 * * * *', {ledger: false});
cron.upsert('processInactivityDeletions', 'processInactivityDeletions', {}, '0 0 */6 * * *', {ledger: false});
cron.upsert('expireAttachments', 'expireAttachments', {}, '0 0 */12 * * *', {ledger: false});
cron.upsert('prunePostgresKvTtl', 'prunePostgresKvTtl', {}, '0 */5 * * * *', {ledger: false});
cron.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *', {ledger: false});
if (Config.blocklistFeeds.enabled) {
cron.upsert('syncDisposableEmailDomains', 'syncDisposableEmailDomains', {}, '0 0 */6 * * *', {ledger: true});
cron.upsert('syncUrlBlocklists', 'syncUrlBlocklists', {}, '0 0 */6 * * *', {ledger: true});
cron.upsert('syncFileShaBlocklists', 'syncFileShaBlocklists', {}, '0 0 */12 * * *', {ledger: true});
}
cron.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/10 * * * * *', {ledger: false});
Logger.info({blocklistFeeds: Config.blocklistFeeds.enabled}, 'Cron jobs registered successfully');
}
function workerLanesRequireSearch(activeWorkerLanes: ReadonlyArray<WorkerLaneDefinition>): boolean {
@@ -191,10 +197,16 @@ export async function startWorkerMain(): Promise<void> {
setInjectedWorkerService(workerService);
dependencies = await initializeWorkerDependencies(snowflakeService);
setWorkerDependencies(dependencies);
const didClaimEmailSync = await dependencies.kvClient.setnx('sync:email_domains:initialized', '1');
if (didClaimEmailSync) {
Logger.info('Triggering initial disposable email domain sync');
await workerService.addJob('syncDisposableEmailDomains', {});
if (Config.blocklistFeeds.enabled) {
const didClaimEmailSync = await dependencies.kvClient.setnx(
'sync:email_domains:initialized',
'1',
ms('6 hours') / 1000,
);
if (didClaimEmailSync) {
Logger.info('Triggering initial disposable email domain sync');
await workerService.addJob('syncDisposableEmailDomains', {});
}
}
cron = new CronScheduler(workerService, Logger, dependencies.kvClient);
registerCronJobs(cron);
fluxer_api/src/api/worker/tasks/SyncDisposableEmailDomains.ts
+1 -11
View File
@@ -21,7 +21,6 @@ const SOURCES = [
'https://raw.githubusercontent.com/vrittech/disposable-email/main/disposable_domains.txt',
'https://raw.githubusercontent.com/martenson/disposable-email-domains/master/disposable_email_blocklist.conf',
];
const CURRENT_DOMAIN_PAGE_SIZE = 10000;
const WRITE_PROGRESS_INTERVAL = 500;
const DOMAIN_REGEX = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/;
@@ -120,16 +119,7 @@ function normaliseDomain(raw: string): string | null {
async function loadCurrentDisposableEmailDomains(): Promise<Set<string>> {
const {adminRepository} = getWorkerDependencies();
const currentSet = new Set<string>();
let pageState: string | null = null;
do {
const page = await adminRepository.listDisposableEmailDomainsPage(CURRENT_DOMAIN_PAGE_SIZE, pageState);
for (const domain of page.domains) {
currentSet.add(domain);
}
pageState = page.pageState;
} while (pageState !== null);
return currentSet;
return new Set(await adminRepository.listDisposableEmailDomains());
}
async function throwIfCancelled(helpers: WorkerTaskHelpers): Promise<void> {