fix(api): honour default TTLs and expire stale job ledger rows (#2887)

deploy/self-hosting/.env.example

@@ -281,6 +281,7 @@ FLUXER_DISCOVERY_ENABLED=true
 #FLUXER_POSTGRES_WORK_MEM=8MB
 #FLUXER_POSTGRES_MAINTENANCE_WORK_MEM=256MB
 #FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM=128MB
+#FLUXER_POSTGRES_SHM_SIZE=1gb
 
 # The bundled Valkey holds durable state as well as cache, so it runs with an
 # append-only file and with noeviction, which fails an over-limit write instead

deploy/self-hosting/docker-compose.yml

@@ -188,7 +188,7 @@ services:
       -c autovacuum_vacuum_cost_limit=2000
       -c track_io_timing=on
       -c shared_preload_libraries=pg_stat_statements
-    shm_size: 256mb
+    shm_size: ${FLUXER_POSTGRES_SHM_SIZE:-1gb}
     environment:
       POSTGRES_DB: fluxer
       POSTGRES_USER: fluxer

fluxer_api/pkgs/geoip/src/PostgresIpInfoKv.ts

@@ -11,6 +11,8 @@ interface PostgresIpInfoOptions {
 }
 
 const VALUE_SEPARATOR = '\u001f';
+export const IPINFO_CACHE_TTL_SECONDS = 14 * 24 * 60 * 60;
+export const IPINFO_REQUEST_AUDIT_TTL_SECONDS = 90 * 24 * 60 * 60;
 
 function getClient(options: PostgresIpInfoOptions): IPostgresClient | null {
 	return options.client ?? options.getClient?.() ?? null;
@@ -34,12 +36,9 @@ async function upsertKvRow(
 	partitionKey: string,
 	key: string,
 	row: Record<string, unknown>,
-	ttlSeconds?: number,
+	ttlSeconds: number,
 ): Promise<void> {
-	const expiresAt =
-		ttlSeconds != null && Number.isFinite(ttlSeconds) && ttlSeconds > 0
-			? new Date(Date.now() + ttlSeconds * 1000)
-			: null;
+	const expiresAt = new Date(Date.now() + ttlSeconds * 1000);
 	await client.query(
 		`INSERT INTO ${table(client)} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
 VALUES ($1, $2, $3, $4::jsonb, $5, now())
@@ -77,7 +76,14 @@ export function createPostgresIpInfoCache(options: PostgresIpInfoOptions): IpInf
 			try {
 				const client = getClient(options);
 				if (!client) return;
-				await upsertKvRow(client, 'ipinfo_cache', rowKey([key]), rowKey([key]), {cache_key: key, payload}, ttlSeconds);
+				await upsertKvRow(
+					client,
+					'ipinfo_cache',
+					rowKey([key]),
+					rowKey([key]),
+					{cache_key: key, payload},
+					ttlSeconds != null && Number.isFinite(ttlSeconds) && ttlSeconds > 0 ? ttlSeconds : IPINFO_CACHE_TTL_SECONDS,
+				);
 			} catch (error) {
 				options.onError?.(error, 'ipinfo_cache_set');
 			}
@@ -124,6 +130,7 @@ export function createPostgresIpInfoRequestAuditLogger(options: PostgresIpInfoOp
 						is_residential_proxy: event.isResidentialProxy,
 						metadata_json: serializeMetadata(event.metadata),
 					},
+					IPINFO_REQUEST_AUDIT_TTL_SECONDS,
 				);
 			} catch (error) {
 				options.onError?.(error, 'ipinfo_request_audit_record');

fluxer_api/pkgs/postgres/src/Client.ts

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import assert from 'node:assert/strict';
-import type {Pool, PoolClient, QueryResult, QueryResultRow} from 'pg';
+import type {Pool, PoolClient, PoolConfig, QueryResult, QueryResultRow} from 'pg';
 import pg from 'pg';
 
 const MAX_DIAGNOSTIC_FIELD_LENGTH = 128;
@@ -131,7 +131,7 @@ class PostgresClient implements IPostgresClient {
 	}
 
 	private async openPool(): Promise<void> {
-		const pool = new pg.Pool({
+		const poolConfig: PoolConfig & {scramMaxIterations: number} = {
 			connectionString: this.config.url || undefined,
 			host: this.config.url ? undefined : (this.config.host ?? '127.0.0.1'),
 			port: this.config.url ? undefined : (this.config.port ?? 5432),
@@ -140,7 +140,9 @@ class PostgresClient implements IPostgresClient {
 			password: this.config.url ? undefined : (this.config.password ?? 'fluxer'),
 			ssl: this.config.ssl ? {rejectUnauthorized: true, ca: normalizePem(this.config.sslCa)} : undefined,
 			max: this.config.maxConnections ?? 20,
-		});
+			scramMaxIterations: 0,
+		};
+		const pool = new pg.Pool(poolConfig);
 		this.observePoolConnections(pool);
 		try {
 			const client = await pool.connect();

fluxer_api/src/api/Tables.ts

@@ -341,6 +341,7 @@ import {
 	type UsersPendingDeletionRow,
 } from '@app/api/database/types/UserTypes';
 import {ATTACHMENT_DECAY_COLUMNS, type AttachmentDecayRow} from '@app/api/types/AttachmentDecayTypes';
+import {seconds} from 'itty-time';
 
 export const Users = defineTable<UserRow, 'user_id'>({
 	name: 'users',
@@ -499,16 +500,19 @@ export const GuildAuditLogs = defineTable<GuildAuditLogRow, 'guild_id' | 'log_id
 	name: 'guild_audit_logs_v2',
 	columns: GUILD_AUDIT_LOG_COLUMNS,
 	primaryKey: ['guild_id', 'log_id'],
+	defaultTtlSeconds: seconds('45 days'),
 });
 export const GuildAuditLogsByUser = defineTable<GuildAuditLogRow, 'guild_id' | 'user_id' | 'log_id'>({
 	name: 'guild_audit_logs_v2_by_user',
 	columns: GUILD_AUDIT_LOG_COLUMNS,
 	primaryKey: ['guild_id', 'user_id', 'log_id'],
+	defaultTtlSeconds: seconds('45 days'),
 });
 export const GuildAuditLogsByAction = defineTable<GuildAuditLogRow, 'guild_id' | 'action_type' | 'log_id'>({
 	name: 'guild_audit_logs_v2_by_action',
 	columns: GUILD_AUDIT_LOG_COLUMNS,
 	primaryKey: ['guild_id', 'action_type', 'log_id'],
+	defaultTtlSeconds: seconds('45 days'),
 });
 export const GuildAuditLogsByUserAction = defineTable<
 	GuildAuditLogRow,
@@ -517,6 +521,7 @@ export const GuildAuditLogsByUserAction = defineTable<
 	name: 'guild_audit_logs_v2_by_user_action',
 	columns: GUILD_AUDIT_LOG_COLUMNS,
 	primaryKey: ['guild_id', 'user_id', 'action_type', 'log_id'],
+	defaultTtlSeconds: seconds('45 days'),
 });
 export const GuildMembershipMetadata = defineTable<GuildMembershipMetadataRow, 'guild_id' | 'user_id'>({
 	name: 'guild_membership_metadata',
@@ -655,6 +660,7 @@ export const RecentMentions = defineTable<RecentMentionRow, 'user_id' | 'message
 	name: 'recent_mentions',
 	columns: RECENT_MENTION_COLUMNS,
 	primaryKey: ['user_id', 'message_id'],
+	defaultTtlSeconds: seconds('7 days'),
 });
 
 interface RecentMentionsByGuildRow {
@@ -678,6 +684,7 @@ export const RecentMentionsByGuild = defineTable<RecentMentionsByGuildRow, 'user
 	name: 'recent_mentions_by_guild',
 	columns: RECENT_MENTIONS_BY_GUILD_COLUMNS,
 	primaryKey: ['user_id', 'guild_id', 'message_id'],
+	defaultTtlSeconds: seconds('7 days'),
 });
 export const SavedMessages = defineTable<SavedMessageRow, 'user_id' | 'message_id'>({
 	name: 'saved_messages',
@@ -688,6 +695,7 @@ export const PushSubscriptions = defineTable<PushSubscriptionRow, 'user_id' | 's
 	name: 'push_subscriptions',
 	columns: PUSH_SUBSCRIPTION_COLUMNS,
 	primaryKey: ['user_id', 'subscription_id'],
+	defaultTtlSeconds: seconds('90 days'),
 });
 export const Payments = defineTable<PaymentRow, 'checkout_session_id'>({
 	name: 'payments',
@@ -854,11 +862,13 @@ export const EmailVerificationTokens = defineTable<EmailVerificationTokenRow, 't
 	name: 'email_verification_tokens',
 	columns: EMAIL_VERIFICATION_TOKEN_COLUMNS,
 	primaryKey: ['token_', 'user_id'],
+	defaultTtlSeconds: seconds('24 hours'),
 });
 export const PasswordResetTokens = defineTable<PasswordResetTokenRow, 'token_' | 'user_id'>({
 	name: 'password_reset_tokens',
 	columns: PASSWORD_RESET_TOKEN_COLUMNS,
 	primaryKey: ['token_', 'user_id'],
+	defaultTtlSeconds: seconds('24 hours'),
 });
 export const PasswordResetTokensByUserId = defineTable<
 	{
@@ -870,16 +880,19 @@ export const PasswordResetTokensByUserId = defineTable<
 	name: 'password_reset_tokens_by_user_id',
 	columns: ['user_id', 'token_'],
 	primaryKey: ['user_id', 'token_'],
+	defaultTtlSeconds: seconds('24 hours'),
 });
 export const EmailRevertTokens = defineTable<EmailRevertTokenRow, 'token_' | 'user_id'>({
 	name: 'email_revert_tokens',
 	columns: EMAIL_REVERT_TOKEN_COLUMNS,
 	primaryKey: ['token_', 'user_id'],
+	defaultTtlSeconds: seconds('48 hours'),
 });
 export const PhoneTokens = defineTable<PhoneTokenRow, 'token_'>({
 	name: 'phone_tokens',
 	columns: PHONE_TOKEN_COLUMNS,
 	primaryKey: ['token_'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const AuthSessions = defineTable<AuthSessionRow, 'session_id_hash'>({
 	name: 'auth_sessions',
@@ -901,11 +914,13 @@ export const AuthSessionTombstones = defineTable<AuthSessionTombstoneRow, 'user_
 	name: 'auth_session_tombstones',
 	columns: AUTH_SESSION_TOMBSTONE_COLUMNS,
 	primaryKey: ['user_id', 'session_id_hash'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const UserCountryHistory = defineTable<UserCountryHistoryRow, 'user_id' | 'country'>({
 	name: 'user_country_history',
 	columns: USER_COUNTRY_HISTORY_COLUMNS,
 	primaryKey: ['user_id', 'country'],
+	defaultTtlSeconds: seconds('365 days'),
 });
 export const MfaBackupCodes = defineTable<MfaBackupCodeRow, 'user_id' | 'code'>({
 	name: 'mfa_backup_codes',
@@ -932,6 +947,7 @@ export const IpAuthorizationTokens = defineTable<IpAuthorizationTokenRow, 'token
 	name: 'ip_authorization_tokens',
 	columns: IP_AUTHORIZATION_TOKEN_COLUMNS,
 	primaryKey: ['token_', 'user_id'],
+	defaultTtlSeconds: seconds('30 minutes'),
 });
 export const AuthorizedIps = defineTable<AuthorizedIpRow, 'user_id' | 'ip'>({
 	name: 'authorized_ips_v2',
@@ -1057,26 +1073,31 @@ export const OAuth2AuthorizationCodes = defineTable<OAuth2AuthorizationCodeRow,
 	name: 'oauth2_authorization_codes',
 	columns: OAUTH2_AUTHORIZATION_CODE_COLUMNS,
 	primaryKey: ['code'],
+	defaultTtlSeconds: seconds('10 minutes'),
 });
 export const OAuth2AccessTokens = defineTable<OAuth2AccessTokenRow, 'token_'>({
 	name: 'oauth2_access_tokens',
 	columns: OAUTH2_ACCESS_TOKEN_COLUMNS,
 	primaryKey: ['token_'],
+	defaultTtlSeconds: seconds('7 days'),
 });
 export const OAuth2AccessTokensByUser = defineTable<OAuth2AccessTokenByUserRow, 'user_id' | 'token_'>({
 	name: 'oauth2_access_tokens_by_user',
 	columns: OAUTH2_ACCESS_TOKENS_BY_USER_COLUMNS,
 	primaryKey: ['user_id', 'token_'],
+	defaultTtlSeconds: seconds('7 days'),
 });
 export const OAuth2RefreshTokens = defineTable<OAuth2RefreshTokenRow, 'token_'>({
 	name: 'oauth2_refresh_tokens',
 	columns: OAUTH2_REFRESH_TOKEN_COLUMNS,
 	primaryKey: ['token_'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const OAuth2RefreshTokensByUser = defineTable<OAuth2RefreshTokenByUserRow, 'user_id' | 'token_'>({
 	name: 'oauth2_refresh_tokens_by_user',
 	columns: OAUTH2_REFRESH_TOKENS_BY_USER_COLUMNS,
 	primaryKey: ['user_id', 'token_'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 
 interface WebhooksByChannelRow {
@@ -1117,12 +1138,14 @@ export const JobsById = defineTable<JobByIdRow, 'job_id'>({
 	name: 'jobs_by_id',
 	columns: JOB_BY_ID_COLUMNS,
 	primaryKey: ['job_id'],
+	defaultTtlSeconds: seconds('90 days'),
 });
 export const JobsByDayBucket = defineTable<JobByDayBucketRow, 'bucket_day' | 'created_at' | 'job_id'>({
 	name: 'jobs_by_day_bucket',
 	columns: JOB_BY_DAY_BUCKET_COLUMNS,
 	primaryKey: ['bucket_day', 'created_at', 'job_id'],
 	partitionKey: ['bucket_day'],
+	defaultTtlSeconds: seconds('90 days'),
 });
 export const JobsActive = defineTable<JobActiveRow, 'job_id'>({
 	name: 'jobs_active',
@@ -1133,11 +1156,13 @@ export const AttachmentUploadTracesByKey = defineTable<AttachmentUploadTraceByKe
 	name: 'attachment_upload_traces_by_key',
 	columns: ATTACHMENT_UPLOAD_TRACE_BY_KEY_COLUMNS,
 	primaryKey: ['upload_key'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const AttachmentUploadTracesByAttachment = defineTable<AttachmentUploadTraceByAttachmentRow, 'attachment_id'>({
 	name: 'attachment_upload_traces_by_attachment',
 	columns: ATTACHMENT_UPLOAD_TRACE_BY_ATTACHMENT_COLUMNS,
 	primaryKey: ['attachment_id'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const NcmecAttachmentSubmissions = defineTable<NcmecAttachmentSubmissionRow, 'attachment_id'>({
 	name: 'ncmec_attachment_submissions',
@@ -1154,6 +1179,7 @@ export const RegistrationEventsByIp = defineTable<RegistrationEventByIpRow, 'ip'
 	columns: REGISTRATION_EVENT_BY_IP_COLUMNS,
 	primaryKey: ['ip', 'created_at', 'user_id'],
 	partitionKey: ['ip'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const RegistrationEventsBySubnet = defineTable<
 	RegistrationEventBySubnetRow,
@@ -1164,6 +1190,7 @@ export const RegistrationEventsBySubnet = defineTable<
 	columns: REGISTRATION_EVENT_BY_SUBNET_COLUMNS,
 	primaryKey: ['subnet', 'created_at', 'user_id'],
 	partitionKey: ['subnet'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const RegistrationEventsByEmailDomain = defineTable<
 	RegistrationEventByEmailDomainRow,
@@ -1174,6 +1201,7 @@ export const RegistrationEventsByEmailDomain = defineTable<
 	columns: REGISTRATION_EVENT_BY_EMAIL_DOMAIN_COLUMNS,
 	primaryKey: ['email_domain', 'created_at', 'user_id'],
 	partitionKey: ['email_domain'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const RegistrationEventsByPlusAddressBase = defineTable<
 	RegistrationEventByPlusAddressBaseRow,
@@ -1184,6 +1212,7 @@ export const RegistrationEventsByPlusAddressBase = defineTable<
 	columns: REGISTRATION_EVENT_BY_PLUS_ADDRESS_BASE_COLUMNS,
 	primaryKey: ['plus_address_base', 'created_at', 'user_id'],
 	partitionKey: ['plus_address_base'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const LatestRiskContextByUser = defineTable<LatestRiskContextByUserRow, 'user_id'>({
 	name: 'latest_risk_context_by_user',
@@ -1194,6 +1223,7 @@ export const SuspiciousIps = defineTable<SuspiciousIpRow, 'ip'>({
 	name: 'suspicious_ips',
 	columns: SUSPICIOUS_IP_COLUMNS,
 	primaryKey: ['ip'],
+	defaultTtlSeconds: seconds('180 days'),
 });
 export const RiskOutcomesByIp = defineTable<RiskOutcomeByIpRow, 'ip' | 'created_at' | 'user_id' | 'outcome_code', 'ip'>(
 	{
@@ -1201,6 +1231,7 @@ export const RiskOutcomesByIp = defineTable<RiskOutcomeByIpRow, 'ip' | 'created_
 		columns: RISK_OUTCOME_BY_IP_COLUMNS,
 		primaryKey: ['ip', 'created_at', 'user_id', 'outcome_code'],
 		partitionKey: ['ip'],
+		defaultTtlSeconds: seconds('180 days'),
 	},
 );
 export const RiskOutcomesBySubnet = defineTable<
@@ -1212,6 +1243,7 @@ export const RiskOutcomesBySubnet = defineTable<
 	columns: RISK_OUTCOME_BY_SUBNET_COLUMNS,
 	primaryKey: ['subnet', 'created_at', 'user_id', 'outcome_code'],
 	partitionKey: ['subnet'],
+	defaultTtlSeconds: seconds('180 days'),
 });
 export const RiskOutcomesByEmailDomain = defineTable<
 	RiskOutcomeByEmailDomainRow,
@@ -1222,6 +1254,7 @@ export const RiskOutcomesByEmailDomain = defineTable<
 	columns: RISK_OUTCOME_BY_EMAIL_DOMAIN_COLUMNS,
 	primaryKey: ['email_domain', 'created_at', 'user_id', 'outcome_code'],
 	partitionKey: ['email_domain'],
+	defaultTtlSeconds: seconds('180 days'),
 });
 export const RiskOutcomesByAsn = defineTable<
 	RiskOutcomeByAsnRow,
@@ -1232,6 +1265,7 @@ export const RiskOutcomesByAsn = defineTable<
 	columns: RISK_OUTCOME_BY_ASN_COLUMNS,
 	primaryKey: ['asn', 'created_at', 'user_id', 'outcome_code'],
 	partitionKey: ['asn'],
+	defaultTtlSeconds: seconds('180 days'),
 });
 export const RiskAssessments = defineTable<RiskAssessmentRow, 'assessment_id'>({
 	name: 'risk_assessments',
@@ -1248,6 +1282,7 @@ export const InboundSmsChallenges = defineTable<InboundSmsChallengeRow, 'challen
 	name: 'inbound_sms_challenges',
 	columns: INBOUND_SMS_CHALLENGE_COLUMNS,
 	primaryKey: ['challenge_code'],
+	defaultTtlSeconds: seconds('15 minutes'),
 });
 export const InboundSmsChallengesByUser = defineTable<
 	InboundSmsChallengeByUserRow,
@@ -1258,16 +1293,19 @@ export const InboundSmsChallengesByUser = defineTable<
 	columns: INBOUND_SMS_CHALLENGE_BY_USER_COLUMNS,
 	primaryKey: ['user_id', 'created_at'],
 	partitionKey: ['user_id'],
+	defaultTtlSeconds: seconds('15 minutes'),
 });
 export const PhoneLookupCache = defineTable<PhoneLookupCacheRow, 'phone'>({
 	name: 'phone_lookup_cache',
 	columns: PHONE_LOOKUP_CACHE_COLUMNS,
 	primaryKey: ['phone'],
+	defaultTtlSeconds: seconds('7 days'),
 });
 export const PhoneVerificationAttempts = defineTable<PhoneVerificationAttemptRow, 'attempt_id'>({
 	name: 'phone_verification_attempts',
 	columns: PHONE_VERIFICATION_ATTEMPT_COLUMNS,
 	primaryKey: ['attempt_id'],
+	defaultTtlSeconds: seconds('90 days'),
 });
 export const BillingCustomers = defineTable<BillingCustomerRow, 'provider_id'>({
 	name: 'billing_customers',

fluxer_api/src/api/admin/AdminRepository.ts

@@ -2,7 +2,15 @@
 
 import type {AdminAuditLog, BannedIpEntry, BannedIpKind, IAdminRepository} from '@app/api/admin/IAdminRepository';
 import {createUserID} from '@app/api/BrandedTypes';
-import {deleteOneOrMany, fetchMany, fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
+import {Config} from '@app/api/Config';
+import {ContentBlocklistCategory} from '@app/api/constants/ContentModeration';
+import {
+	deleteOneOrMany,
+	executeConditional,
+	fetchMany,
+	fetchOne,
+	upsertOne,
+} from '@app/api/database/CassandraQueryExecution';
 import type {
 	AdminAuditLogRow,
 	BannedAvatarHashRow,
@@ -282,6 +290,7 @@ export class AdminRepository implements IAdminRepository {
 	}
 
 	async isEmailDomainDisposable(domain: string): Promise<boolean> {
+		if (!Config.blocklistFeeds.enabled) return false;
 		const domainLower = domain.toLowerCase();
 		if (isAccountPolicyContactDomainReputationExempt(domainLower)) return false;
 		const result = await fetchOne<{
@@ -395,6 +404,15 @@ export class AdminRepository implements IAdminRepository {
 		await deleteOneOrMany(BannedFileShas.deleteByPk({sha256_hex: sha256Hex.toLowerCase()}));
 	}
 
+	async unbanFeedFileSha(sha256Hex: string): Promise<boolean> {
+		return executeConditional(
+			BannedFileShas.conditionalDeleteByPk(
+				{sha256_hex: sha256Hex.toLowerCase()},
+				{added_by: null, category: ContentBlocklistCategory.MALWARE_BAZAAR},
+			),
+		);
+	}
+
 	async loadAllBannedFileShas(): Promise<Array<BannedFileShaRow>> {
 		return fetchMany<BannedFileShaRow>(LOAD_ALL_BANNED_FILE_SHAS_QUERY.bind({}));
 	}

fluxer_api/src/api/admin/IAdminRepository.ts

@@ -109,6 +109,8 @@ export abstract class IAdminRepository {
 
 	abstract unbanFileSha(sha256Hex: string): Promise<void>;
 
+	abstract unbanFeedFileSha(sha256Hex: string): Promise<boolean>;
+
 	abstract loadAllBannedFileShas(): Promise<Array<BannedFileShaRow>>;
 
 	abstract isAvatarHashBanned(hashShort: string): Promise<boolean>;

fluxer_api/src/api/constants/ContentModeration.ts

@@ -1,5 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import type {BannedFileShaRow} from '@app/api/database/types/AdminArchiveTypes';
+
 export const BANNED_URLS_REFRESH_CHANNEL = 'banned_urls_refresh';
 export const BANNED_URL_DOMAINS_REFRESH_CHANNEL = 'banned_url_domains_refresh';
 export const BANNED_FILE_SHAS_REFRESH_CHANNEL = 'banned_file_shas_refresh';
@@ -23,3 +25,7 @@ export const ContentBlocklistCategory = {
 	GIFCT: 'gifct',
 	STOP_NCII: 'stop_ncii',
 } as const;
+
+export function isBlocklistFeedFileSha(row: Pick<BannedFileShaRow, 'category' | 'added_by'>): boolean {
+	return row.added_by == null && row.category === ContentBlocklistCategory.MALWARE_BAZAAR;
+}

fluxer_api/src/api/database/CassandraDefaultTtlParity.test.ts

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import fs from 'node:fs';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {DEFAULT_TTL_TABLES} from '@app/api/database/PostgresKvDefaultTtlExpiry';
+import * as DonationTables from '@app/api/donation/DonationTables';
+import * as Tables from '@app/api/Tables';
+import {IPINFO_CACHE_TTL_SECONDS, IPINFO_REQUEST_AUDIT_TTL_SECONDS} from '@pkgs/geoip/src/PostgresIpInfoKv';
+import {describe, expect, it} from 'vitest';
+
+const THIS_DIR = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(THIS_DIR, '../../../..');
+
+interface SchemaTable {
+	name: string;
+	options: string;
+}
+
+const SCHEMA = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'tools/dev/cassandra_target_schema.json'), 'utf8')) as {
+	tables: Array<SchemaTable>;
+};
+
+const SCHEMA_DEFAULTS = new Map<string, number>(
+	SCHEMA.tables.flatMap((table): Array<[string, number]> => {
+		const match = /default_time_to_live = (\d+)/.exec(table.options);
+		return match ? [[table.name, Number(match[1])]] : [];
+	}),
+);
+
+const DSL_TABLES = [...Object.values(Tables), ...Object.values(DonationTables)];
+const DSL_NAMES = new Set<string>(DSL_TABLES.map((table) => table.name));
+
+const NON_DSL_DEFAULTS: Record<string, number | null> = {
+	ipinfo_cache: IPINFO_CACHE_TTL_SECONDS,
+	ipinfo_requests_by_hour: IPINFO_REQUEST_AUDIT_TTL_SECONDS,
+	billing_webhook_events: null,
+	forensic_identifier_by_key_day: null,
+	forensic_identifier_by_request: null,
+	forensic_request_meta_by_actor_day: null,
+	forensic_request_meta_by_id: null,
+	forensic_request_meta_by_route_day_shard: null,
+	forensic_resource_exposure_by_request: null,
+	forensic_resource_exposure_by_route_day_shard: null,
+	forensic_resource_exposure_by_subject_day: null,
+};
+
+const OWN_EXPIRY_PASS = new Set(['jobs_by_id', 'jobs_by_day_bucket']);
+
+function schemaDefault(name: string): number {
+	return SCHEMA_DEFAULTS.get(name) ?? 0;
+}
+
+function byName(left: {name: string}, right: {name: string}): number {
+	return left.name.localeCompare(right.name);
+}
+
+describe('Cassandra default TTL parity', () => {
+	it('declares every Cassandra default TTL on the matching table', () => {
+		const mismatches = DSL_TABLES.flatMap((table) => {
+			const declared = table.defaultTtlSeconds ?? 0;
+			return declared === schemaDefault(table.name)
+				? []
+				: [{table: table.name, declared, schema: schemaDefault(table.name)}];
+		});
+		expect(mismatches).toEqual([]);
+	});
+
+	it('declares a writer or no writer for every other table with a default', () => {
+		const undeclared = [...SCHEMA_DEFAULTS]
+			.filter(([name, ttl]) => ttl > 0 && !DSL_NAMES.has(name) && !Object.hasOwn(NON_DSL_DEFAULTS, name))
+			.map(([name]) => name);
+		expect(undeclared).toEqual([]);
+		const stale = Object.keys(NON_DSL_DEFAULTS).filter((name) => schemaDefault(name) === 0 || DSL_NAMES.has(name));
+		expect(stale).toEqual([]);
+		const mismatched = Object.entries(NON_DSL_DEFAULTS)
+			.filter(([name, ttl]) => ttl !== null && ttl !== schemaDefault(name))
+			.map(([name]) => name);
+		expect(mismatched).toEqual([]);
+	});
+
+	it('the Postgres expiry pass covers every table with a default except the job ledger', () => {
+		const expected = [...SCHEMA_DEFAULTS]
+			.filter(([name, ttl]) => ttl > 0 && NON_DSL_DEFAULTS[name] !== null && !OWN_EXPIRY_PASS.has(name))
+			.map(([name, ttl]) => ({name, defaultTtlSeconds: ttl}))
+			.sort(byName);
+		expect([...DEFAULT_TTL_TABLES].sort(byName)).toEqual(expected);
+	});
+});

fluxer_api/src/api/database/CassandraMetaRegistry.ts

@@ -12,6 +12,7 @@ interface TableMetadata {
 	columns: ReadonlyArray<string>;
 	primaryKey: ReadonlyArray<string>;
 	partitionKey: ReadonlyArray<string>;
+	defaultTtlSeconds?: number;
 }
 
 const kvMetaRegistry = new Map<string, KvQueryMeta<Record<string, unknown>>>();
@@ -24,6 +25,7 @@ export function registerTableSpec<Row extends object>(tableSpec: KvTableSpec<Row
 		columns: tableSpec.columns as ReadonlyArray<string>,
 		primaryKey: tableSpec.primaryKey as ReadonlyArray<string>,
 		partitionKey: tableSpec.partitionKey as ReadonlyArray<string>,
+		defaultTtlSeconds: tableSpec.defaultTtlSeconds,
 	};
 	tableRegistry.set(tableSpec.name, metadata);
 }

fluxer_api/src/api/database/CassandraTableDsl.test.ts

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import {getTableMetadata} from '@app/api/database/CassandraMetaRegistry';
 import {defineTable} from '@app/api/database/CassandraTableDsl';
 import {Db, type PreparedQuery} from '@app/api/database/CassandraTypes';
 import {describe, expect, it} from 'vitest';
@@ -76,3 +77,41 @@ describe('CassandraTableDsl select templates', () => {
 		expect(longQuery.cql).not.toContain('LIMIT 20');
 	});
 });
+
+describe('CassandraTableDsl default TTL', () => {
+	it('keeps the CQL of a table with a default TTL free of USING TTL', () => {
+		const DefaultTtlRows = defineTable<TtlHelperTestRow, 'id'>({
+			name: 'default_ttl_dsl_rows',
+			columns: ['id', 'value'],
+			primaryKey: ['id'],
+			defaultTtlSeconds: 600,
+		});
+		expect(DefaultTtlRows.defaultTtlSeconds).toBe(600);
+		const queries = [
+			DefaultTtlRows.insert({id: 'insert', value: 'a'}),
+			DefaultTtlRows.upsertAll({id: 'upsert', value: 'b'}),
+			DefaultTtlRows.patchByPk({id: 'patch'}, {value: Db.set('c')}),
+		];
+		for (const query of queries) {
+			expect(query.cql).not.toContain('USING TTL');
+			expect(query.kvMeta?.table.defaultTtlSeconds).toBe(600);
+		}
+		expect(getTableMetadata('default_ttl_dsl_rows')?.defaultTtlSeconds).toBe(600);
+		expect(TtlHelperTestRows.defaultTtlSeconds).toBeUndefined();
+		expect(getTableMetadata('ttl_helper_test_rows')?.defaultTtlSeconds).toBeUndefined();
+	});
+
+	it('rejects a default TTL of zero, a fraction or past the maximum', () => {
+		for (const defaultTtlSeconds of [0, 1.5, 630_720_001]) {
+			expect(() =>
+				defineTable<TtlHelperTestRow, 'id'>({
+					name: 'default_ttl_dsl_rejected_rows',
+					columns: ['id', 'value'],
+					primaryKey: ['id'],
+					defaultTtlSeconds,
+				}),
+			).toThrow();
+		}
+		expect(getTableMetadata('default_ttl_dsl_rejected_rows')).toBeUndefined();
+	});
+});

fluxer_api/src/api/database/CassandraTableDsl.ts

@@ -83,6 +83,7 @@ export function defineTable<Row extends object, PK extends ColumnName<Row>, Part
 	columns: ReadonlyArray<ColumnName<Row>>;
 	primaryKey: ReadonlyArray<PK>;
 	partitionKey?: ReadonlyArray<PartKey>;
+	defaultTtlSeconds?: number;
 }): Table<Row, PK, PartKey> {
 	const columns = [...def.columns];
 	const pk = [...def.primaryKey];
@@ -91,11 +92,15 @@ export function defineTable<Row extends object, PK extends ColumnName<Row>, Part
 	for (const c of columns) assertCqlIdentifier(c as string);
 	for (const k of pk) assertCqlIdentifier(k as string);
 	for (const k of partitionKey) assertCqlIdentifier(k as string);
+	if (def.defaultTtlSeconds !== undefined && validateTtlSeconds(def.defaultTtlSeconds) === 0) {
+		throw new Error(`Table "${def.name}" needs a positive default TTL`);
+	}
 	const tableSpec: KvTableSpec<Row> = {
 		name: def.name,
 		columns,
 		primaryKey: pk as ReadonlyArray<ColumnName<Row>>,
 		partitionKey: partitionKey as ReadonlyArray<ColumnName<Row>>,
+		defaultTtlSeconds: def.defaultTtlSeconds,
 	};
 	registerTableSpec(tableSpec);
 	const nonPkColumns = columns.filter((c) => !pk.includes(c as PK)) as Array<Exclude<ColumnName<Row>, PK>>;
@@ -685,6 +690,7 @@ WHERE ${pk.map((k) => `${k} = :${k}`).join(' AND ')};
 		columns: def.columns,
 		primaryKey: def.primaryKey,
 		partitionKey: partitionKey,
+		defaultTtlSeconds: def.defaultTtlSeconds,
 		selectCql,
 		select,
 		updateAllCql() {

fluxer_api/src/api/database/CassandraTypes.ts

@@ -56,6 +56,7 @@ export interface KvTableSpec<Row extends object = Record<string, unknown>> {
 	columns: ReadonlyArray<ColumnName<Row>>;
 	primaryKey: ReadonlyArray<ColumnName<Row>>;
 	partitionKey: ReadonlyArray<ColumnName<Row>>;
+	defaultTtlSeconds?: number;
 }
 
 export interface KvColumnParam<Row extends object = Record<string, unknown>> {
@@ -190,6 +191,7 @@ export interface Table<Row extends object, PK extends ColumnName<Row>, PartKey e
 	columns: ReadonlyArray<ColumnName<Row>>;
 	primaryKey: ReadonlyArray<PK>;
 	partitionKey: ReadonlyArray<PartKey>;
+	defaultTtlSeconds: number | undefined;
 	selectCql(opts?: {
 		columns?: ReadonlyArray<ColumnName<Row>>;
 		where?: WhereExpr<Row> | ReadonlyArray<WhereExpr<Row>>;

fluxer_api/src/api/database/PostgresKvDefaultTtl.test.ts

@@ -0,0 +1,490 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {defineTable} from '@app/api/database/CassandraTableDsl';
+import {Db} from '@app/api/database/CassandraTypes';
+import {
+	DEFAULT_TTL_EXPIRY_RESUME,
+	DEFAULT_TTL_TABLES,
+	expireLegacyDefaultTtlRows,
+} from '@app/api/database/PostgresKvDefaultTtlExpiry';
+import {
+	ensurePostgresKvSchema,
+	PostgresKvQueryExecutor,
+	pruneExpiredPostgresKvRows,
+} from '@app/api/database/PostgresKvQueryExecutor';
+import {startDockerContainer} from '@app/api/test/DockerTestContainer';
+import {
+	getDefaultPostgresClient,
+	type IPostgresClient,
+	initPostgres,
+	shutdownPostgres,
+} from '@pkgs/postgres/src/Client';
+import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
+
+const KV_TABLE = 'kv_default_ttl';
+const CONTAINER = `fluxer-kvttl-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+const DEFAULT_TTL_SECONDS = 600;
+
+interface ProbeRow {
+	id: string;
+	value: string | null;
+	note: string | null;
+}
+
+interface OwnedProbeRow {
+	owner: string;
+	id: string;
+	value: string | null;
+}
+
+const DefaultTtlProbe = defineTable<ProbeRow, 'id'>({
+	name: 'default_ttl_probe',
+	columns: ['id', 'value', 'note'],
+	primaryKey: ['id'],
+	defaultTtlSeconds: DEFAULT_TTL_SECONDS,
+});
+
+const DefaultTtlProbeRows = defineTable<OwnedProbeRow, 'owner' | 'id', 'owner'>({
+	name: 'default_ttl_probe_rows',
+	columns: ['owner', 'id', 'value'],
+	primaryKey: ['owner', 'id'],
+	partitionKey: ['owner'],
+	defaultTtlSeconds: DEFAULT_TTL_SECONDS,
+});
+
+const NoTtlProbe = defineTable<ProbeRow, 'id'>({
+	name: 'no_ttl_probe',
+	columns: ['id', 'value', 'note'],
+	primaryKey: ['id'],
+});
+
+async function sleep(ms: number): Promise<void> {
+	await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function freePort(): Promise<number> {
+	return new Promise((resolve, reject) => {
+		const server = createServer();
+		server.on('error', reject);
+		server.listen(0, '127.0.0.1', () => {
+			const address = server.address();
+			if (typeof address === 'string' || address === null) {
+				reject(new Error('no port'));
+				return;
+			}
+			const port = address.port;
+			server.close(() => resolve(port));
+		});
+	});
+}
+
+function expectExpiresIn(value: Date | number | null, ttlSeconds: number): void {
+	expect(value).toBeInstanceOf(Date);
+	const remainingSeconds = ((value as Date).getTime() - Date.now()) / 1000;
+	expect(remainingSeconds).toBeGreaterThan(ttlSeconds - 60);
+	expect(remainingSeconds).toBeLessThanOrEqual(ttlSeconds);
+}
+
+describe.skipIf(!dockerAvailable)('Postgres KV default TTL', () => {
+	let raw: IPostgresClient;
+	let executor: PostgresKvQueryExecutor;
+
+	async function stored(table: string, id: string): Promise<{expires_at: Date | number | null; row_data: object}> {
+		const result = await raw.query<{expires_at: Date | number | null; row_data: object}>(
+			`SELECT expires_at, row_data FROM ${KV_TABLE} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
+			[table, id],
+		);
+		expect(result.rows).toHaveLength(1);
+		return result.rows[0]!;
+	}
+
+	async function expiresAt(table: string, id: string): Promise<Date | number | null> {
+		return (await stored(table, id)).expires_at;
+	}
+
+	async function neverExpires(table: string, id: string): Promise<boolean> {
+		const result = await raw.query<{forever: boolean}>(
+			`SELECT expires_at = 'infinity'::timestamptz AS forever FROM ${KV_TABLE} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
+			[table, id],
+		);
+		return result.rows[0]?.forever === true;
+	}
+
+	async function setExpiry(table: string, id: string, expression: string): Promise<void> {
+		await raw.query(
+			`UPDATE ${KV_TABLE} SET expires_at = ${expression} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
+			[table, id],
+		);
+	}
+
+	async function seed(table: string, key: string, age: string, expires: Date | string | null = null): Promise<string> {
+		const result = await raw.query<{updated_at: string}>(
+			`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
+VALUES ($1, $2, $2, '{}'::jsonb, $3::timestamptz, now() - $4::interval)
+RETURNING updated_at::text`,
+			[table, key, expires, age],
+		);
+		return result.rows[0]!.updated_at;
+	}
+
+	async function remaining(): Promise<Array<{table_name: string; row_key: string}>> {
+		const result = await raw.query<{table_name: string; row_key: string}>(
+			`SELECT table_name, row_key FROM ${KV_TABLE} WHERE table_name <> '__fluxer_schema_migrations' ORDER BY table_name, row_key`,
+		);
+		return result.rows;
+	}
+
+	async function ageMarker(): Promise<void> {
+		await raw.query(
+			`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('applied_at', now() - interval '2 days') WHERE table_name = '__fluxer_schema_migrations' AND row_key = 'default_ttl_expiry_v1'`,
+		);
+	}
+
+	async function resumePoint(): Promise<object | null> {
+		const result = await raw.query<{row_data: object}>(
+			`SELECT row_data FROM ${KV_TABLE} WHERE table_name = '__fluxer_schema_migrations' AND row_key = $1`,
+			[DEFAULT_TTL_EXPIRY_RESUME],
+		);
+		return result.rows[0]?.row_data ?? null;
+	}
+
+	async function markerCount(): Promise<number> {
+		const result = await raw.query<{n: number}>(
+			`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = '__fluxer_schema_migrations' AND row_key = 'default_ttl_expiry_v1'`,
+		);
+		return result.rows[0]!.n;
+	}
+
+	beforeAll(async () => {
+		const port = await freePort();
+		startDockerContainer([
+			'run',
+			'-d',
+			'--name',
+			CONTAINER,
+			'-e',
+			'POSTGRES_USER=fluxer',
+			'-e',
+			'POSTGRES_PASSWORD=fluxer',
+			'-e',
+			'POSTGRES_DB=fluxer',
+			'-p',
+			`127.0.0.1:${port}:5432`,
+			'postgres:16-alpine',
+			'-c',
+			'fsync=off',
+		]);
+		let ready = false;
+		for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+			await sleep(500);
+			const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
+				stdio: 'ignore',
+			});
+			if (probe.status !== 0) continue;
+			try {
+				await initPostgres({
+					url: `postgres://fluxer:[email protected]:${port}/fluxer`,
+					maxConnections: 4,
+					kvTable: KV_TABLE,
+				});
+				await getDefaultPostgresClient().query('SELECT 1');
+				ready = true;
+			} catch {
+				await shutdownPostgres().catch(() => {});
+			}
+		}
+		if (!ready) throw new Error('postgres never came up');
+		raw = getDefaultPostgresClient();
+		await ensurePostgresKvSchema(raw);
+		executor = new PostgresKvQueryExecutor(raw);
+	}, 900_000);
+
+	beforeEach(async () => {
+		await raw.query(`DELETE FROM ${KV_TABLE}`);
+	});
+
+	afterAll(async () => {
+		await shutdownPostgres().catch(() => {});
+		spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+	});
+
+	it('gives every full-row write without a TTL the table default', async () => {
+		await executor.executeQuery(DefaultTtlProbe.insert({id: 'insert', value: 'a', note: null}));
+		await executor.executeQuery(DefaultTtlProbe.upsertAll({id: 'upsert', value: 'b', note: 'n'}));
+		expect(
+			await executor.executeQuery(DefaultTtlProbe.insertIfNotExists({id: 'claimed', value: 'c', note: null})),
+		).toEqual([{'[applied]': true}]);
+		expect(
+			await executor.executeQuery(
+				DefaultTtlProbeRows.conditionalBatch([{action: 'insert', row: {owner: 'o', id: 'batched', value: 'd'}}]),
+			),
+		).toEqual([{'[applied]': true}]);
+
+		for (const id of ['insert', 'upsert', 'claimed']) {
+			expectExpiresIn(await expiresAt('default_ttl_probe', id), DEFAULT_TTL_SECONDS);
+		}
+		expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'batched'), DEFAULT_TTL_SECONDS);
+	});
+
+	it('keeps an explicit TTL ahead of the default', async () => {
+		await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'short', value: 'a', note: null}, 60));
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'short'), 60);
+
+		await executor.executeQuery(DefaultTtlProbe.insert({id: 'patched', value: 'a', note: null}));
+		await executor.executeQuery(DefaultTtlProbe.patchByPkWithTtl({id: 'patched'}, {value: Db.set('b')}, 60));
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'patched'), 60);
+	});
+
+	it('keeps an explicit TTL of zero as no expiry', async () => {
+		await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'forever', value: 'a', note: null}, 0));
+		expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
+		expect(
+			await executor.executeQuery(
+				DefaultTtlProbe.select({where: DefaultTtlProbe.where.eq('id')}).bind({id: 'forever'}),
+			),
+		).toEqual([{id: 'forever', value: 'a', note: null}]);
+
+		await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'forever'}, {note: Db.set('patched')}));
+		expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
+
+		await pruneExpiredPostgresKvRows(raw);
+		expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
+	});
+
+	it('raises a patched row to the default but never lowers it', async () => {
+		await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'longer', value: 'a', note: null}, 3600));
+		await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'longer'}, {note: Db.set('patched')}));
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'longer'), 3600);
+
+		await executor.executeQuery(DefaultTtlProbe.insert({id: 'soon', value: 'a', note: null}));
+		await setExpiry('default_ttl_probe', 'soon', "now() + interval '5 seconds'");
+		await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'soon'}, {note: Db.set('patched')}));
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'soon'), DEFAULT_TTL_SECONDS);
+
+		await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'missing'}, {note: Db.set('created')}));
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'missing'), DEFAULT_TTL_SECONDS);
+
+		await executor.executeQuery(DefaultTtlProbe.insert({id: 'unset', value: 'a', note: null}));
+		await setExpiry('default_ttl_probe', 'unset', 'NULL');
+		await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'unset'}, {note: Db.set('patched')}));
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'unset'), DEFAULT_TTL_SECONDS);
+
+		await executor.executeQuery(DefaultTtlProbe.insert({id: 'expired', value: 'a', note: null}));
+		await setExpiry('default_ttl_probe', 'expired', "now() - interval '1 second'");
+		await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'expired'}, {note: Db.set('patched')}));
+		const revived = await stored('default_ttl_probe', 'expired');
+		expect(revived.row_data).toEqual({id: 'expired', note: 'patched'});
+		expectExpiresIn(revived.expires_at, DEFAULT_TTL_SECONDS);
+	});
+
+	it('raises conditional patches the same way', async () => {
+		await executor.executeQuery(DefaultTtlProbe.insert({id: 'soon', value: 'a', note: null}));
+		await setExpiry('default_ttl_probe', 'soon', "now() + interval '5 seconds'");
+		expect(
+			await executor.executeQuery(
+				DefaultTtlProbe.conditionalPatchByPk({id: 'soon'}, {note: Db.set('patched')}, {value: 'a'}),
+			),
+		).toEqual([{'[applied]': true}]);
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'soon'), DEFAULT_TTL_SECONDS);
+
+		await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'longer', value: 'a', note: null}, 3600));
+		expect(
+			await executor.executeQuery(
+				DefaultTtlProbe.conditionalPatchByPk({id: 'longer'}, {note: Db.set('patched')}, {value: 'a'}),
+			),
+		).toEqual([{'[applied]': true}]);
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'longer'), 3600);
+
+		await executor.executeQuery(DefaultTtlProbeRows.insert({owner: 'o', id: 'existing', value: 'old'}));
+		await setExpiry('default_ttl_probe_rows', 'existing', 'NULL');
+		expect(
+			await executor.executeQuery(
+				DefaultTtlProbeRows.conditionalBatch([
+					{action: 'insert', row: {owner: 'o', id: 'added', value: 'new'}},
+					{
+						action: 'patch',
+						pk: {owner: 'o', id: 'existing'},
+						patch: {value: Db.set('updated')},
+						expected: {value: 'old'},
+					},
+				]),
+			),
+		).toEqual([{'[applied]': true}]);
+		expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'added'), DEFAULT_TTL_SECONDS);
+		expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'existing'), DEFAULT_TTL_SECONDS);
+	});
+
+	it('leaves tables without a default untouched', async () => {
+		await executor.executeQuery(NoTtlProbe.insert({id: 'plain', value: 'a', note: null}));
+		expect(await expiresAt('no_ttl_probe', 'plain')).toBeNull();
+		await executor.executeQuery(NoTtlProbe.patchByPk({id: 'plain'}, {note: Db.set('patched')}));
+		expect(await expiresAt('no_ttl_probe', 'plain')).toBeNull();
+		await executor.executeQuery(NoTtlProbe.insertWithTtl({id: 'zero', value: 'a', note: null}, 0));
+		expect(await expiresAt('no_ttl_probe', 'zero')).toBeNull();
+	});
+
+	it('gives rows an older image wrote the expiry of their last write and deletes the ones past it', async () => {
+		const mentionWrittenAt = await seed('recent_mentions', 'rm-day', '1 day');
+		await seed('recent_mentions', 'rm-week', '8 days');
+		await seed('attachment_upload_traces_by_key', 'at-31', '31 days');
+		await seed('attachment_upload_traces_by_key', 'at-29', '29 days');
+		await seed('phone_lookup_cache', 'pl-8', '8 days');
+		await seed('donor_magic_link_tokens', 'dm-hour', '1 hour');
+		await seed('ipinfo_requests_by_hour', 'ip-day', '1 day');
+		await seed('jobs_by_id', 'job', '100 days');
+		await seed('users', 'user', '100 days');
+		await seed('recent_mentions', 'rm-forever', '1 day', 'infinity');
+		await seed('recent_mentions', 'rm-hour', '30 days', new Date(Date.now() + 3_600_000));
+
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 4,
+			expiring: 3,
+			complete: true,
+		});
+		expect(await remaining()).toEqual([
+			{table_name: 'attachment_upload_traces_by_key', row_key: 'at-29'},
+			{table_name: 'ipinfo_requests_by_hour', row_key: 'ip-day'},
+			{table_name: 'jobs_by_id', row_key: 'job'},
+			{table_name: 'recent_mentions', row_key: 'rm-day'},
+			{table_name: 'recent_mentions', row_key: 'rm-forever'},
+			{table_name: 'recent_mentions', row_key: 'rm-hour'},
+			{table_name: 'users', row_key: 'user'},
+		]);
+
+		const exact = await raw.query<{row_key: string; exact: boolean; unchanged: boolean | null}>(
+			`SELECT row_key,
+	expires_at = updated_at + CASE table_name WHEN 'recent_mentions' THEN interval '7 days' WHEN 'attachment_upload_traces_by_key' THEN interval '30 days' ELSE interval '90 days' END AS exact,
+	CASE WHEN row_key = 'rm-day' THEN updated_at = $1::timestamptz END AS unchanged
+FROM ${KV_TABLE}
+WHERE row_key IN ('rm-day', 'at-29', 'ip-day')
+ORDER BY row_key`,
+			[mentionWrittenAt],
+		);
+		expect(exact.rows).toEqual([
+			{row_key: 'at-29', exact: true, unchanged: null},
+			{row_key: 'ip-day', exact: true, unchanged: null},
+			{row_key: 'rm-day', exact: true, unchanged: true},
+		]);
+		const untouched = await raw.query<{row_key: string; state: string}>(
+			`SELECT row_key, CASE WHEN expires_at IS NULL THEN 'unset' WHEN expires_at = 'infinity' THEN 'forever' ELSE 'set' END AS state
+FROM ${KV_TABLE}
+WHERE row_key IN ('job', 'user', 'rm-forever', 'rm-hour')
+ORDER BY row_key`,
+		);
+		expect(untouched.rows).toEqual([
+			{row_key: 'job', state: 'unset'},
+			{row_key: 'rm-forever', state: 'forever'},
+			{row_key: 'rm-hour', state: 'set'},
+			{row_key: 'user', state: 'unset'},
+		]);
+
+		expect(await markerCount()).toBe(0);
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await markerCount()).toBe(1);
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
+	});
+
+	it('checks again a day after a clean pass', async () => {
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
+
+		await seed('recent_mentions', 'rm-rolled-back', '1 day');
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
+		const before = await raw.query(`SELECT expires_at FROM ${KV_TABLE} WHERE row_key = 'rm-rolled-back'`);
+		expect(before.rows).toEqual([{expires_at: null}]);
+
+		await ageMarker();
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 1,
+			complete: true,
+		});
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
+	});
+
+	it('pages through more rows than one page holds and stops at its deadline', async () => {
+		await raw.query(
+			`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data, updated_at)
+SELECT 'recent_mentions', 'rm-' || lpad(g::text, 5, '0'), 'rm-' || lpad(g::text, 5, '0'), '{}'::jsonb, now() - interval '1 day'
+FROM generate_series(1, 2300) g`,
+		);
+
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() - 1)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: false,
+		});
+		expect(await markerCount()).toBe(0);
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 2300,
+			complete: true,
+		});
+		const unset = await raw.query<{n: number}>(
+			`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = 'recent_mentions' AND expires_at IS NULL`,
+		);
+		expect(unset.rows[0]).toEqual({n: 0});
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
+	});
+
+	it('saves where a run stopped and starts the next run there', async () => {
+		const first = DEFAULT_TTL_TABLES[0]!.name;
+		const last = DEFAULT_TTL_TABLES.at(-1)!.name;
+		await seed(first, 'a', '1 hour');
+		await seed(first, 'z', '1 hour');
+		await seed(last, 'k', '1 hour');
+
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() - 1)).toEqual({deleted: 0, expiring: 0, complete: false});
+		expect(await resumePoint()).toEqual({table: first, row_key: '', unset: 0});
+
+		await raw.query(
+			`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('table', $1::text, 'row_key', 'm', 'unset', 0) WHERE table_name = '__fluxer_schema_migrations' AND row_key = $2`,
+			[first, DEFAULT_TTL_EXPIRY_RESUME],
+		);
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 2,
+			complete: true,
+		});
+		const untouched = await raw.query<{expires_at: Date | null}>(
+			`SELECT expires_at FROM ${KV_TABLE} WHERE table_name = $1 AND row_key = 'a'`,
+			[first],
+		);
+		expect(untouched.rows).toEqual([{expires_at: null}]);
+		expect(await resumePoint()).toBeNull();
+		expect(await markerCount()).toBe(0);
+
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 1,
+			complete: true,
+		});
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await markerCount()).toBe(1);
+	});
+});

fluxer_api/src/api/database/PostgresKvDefaultTtlExpiry.ts

@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {
+	POSTGRES_KV_MIGRATION_TABLE,
+	postgresKvPassIsFresh,
+	recordPostgresKvCleanPass,
+} from '@app/api/database/PostgresKvQueryExecutor';
+import * as DonationTables from '@app/api/donation/DonationTables';
+import * as Tables from '@app/api/Tables';
+import {IPINFO_CACHE_TTL_SECONDS, IPINFO_REQUEST_AUDIT_TTL_SECONDS} from '@pkgs/geoip/src/PostgresIpInfoKv';
+import {type IPostgresClient, quoteIdentifier} from '@pkgs/postgres/src/Client';
+import {ms} from 'itty-time';
+
+const DEFAULT_TTL_EXPIRY_MARKER = 'default_ttl_expiry_v1';
+export const DEFAULT_TTL_EXPIRY_RESUME = 'default_ttl_expiry_v1_resume';
+const PAGE_SIZE = 2000;
+const CLEAN_PASS_INTERVAL_MS = ms('1 day');
+const OWN_EXPIRY_PASS = new Set<string>([Tables.JobsById.name, Tables.JobsByDayBucket.name]);
+
+export const DEFAULT_TTL_TABLES: ReadonlyArray<{name: string; defaultTtlSeconds: number}> = [
+	...[...Object.values(Tables), ...Object.values(DonationTables)].flatMap((table) =>
+		table.defaultTtlSeconds === undefined || OWN_EXPIRY_PASS.has(table.name)
+			? []
+			: [{name: table.name, defaultTtlSeconds: table.defaultTtlSeconds}],
+	),
+	{name: 'ipinfo_cache', defaultTtlSeconds: IPINFO_CACHE_TTL_SECONDS},
+	{name: 'ipinfo_requests_by_hour', defaultTtlSeconds: IPINFO_REQUEST_AUDIT_TTL_SECONDS},
+];
+
+export interface LegacyDefaultTtlExpiryResult {
+	deleted: number;
+	expiring: number;
+	complete: boolean;
+}
+
+interface ResumePoint {
+	table: string;
+	rowKey: string;
+	unset: number;
+}
+
+async function readResumePoint(client: IPostgresClient, kvTable: string): Promise<ResumePoint | null> {
+	const result = await client.query<{row_data: Record<string, unknown>}>(
+		`SELECT row_data FROM ${kvTable} WHERE table_name = $1 AND row_key = $2`,
+		[POSTGRES_KV_MIGRATION_TABLE, DEFAULT_TTL_EXPIRY_RESUME],
+	);
+	const data = result.rows[0]?.row_data;
+	if (typeof data?.table !== 'string' || typeof data.row_key !== 'string' || typeof data.unset !== 'number') {
+		return null;
+	}
+	return {table: data.table, rowKey: data.row_key, unset: data.unset};
+}
+
+async function writeResumePoint(client: IPostgresClient, kvTable: string, point: ResumePoint | null): Promise<void> {
+	if (point === null) {
+		await client.query(`DELETE FROM ${kvTable} WHERE table_name = $1 AND row_key = $2`, [
+			POSTGRES_KV_MIGRATION_TABLE,
+			DEFAULT_TTL_EXPIRY_RESUME,
+		]);
+		return;
+	}
+	await client.query(
+		`INSERT INTO ${kvTable} (table_name, partition_key, row_key, row_data)
+VALUES ($1, $2, $2, jsonb_build_object('table', $3::text, 'row_key', $4::text, 'unset', $5::bigint))
+ON CONFLICT (table_name, row_key) DO UPDATE SET row_data = EXCLUDED.row_data, updated_at = now()`,
+		[POSTGRES_KV_MIGRATION_TABLE, DEFAULT_TTL_EXPIRY_RESUME, point.table, point.rowKey, point.unset],
+	);
+}
+
+function pageSql(table: string): string {
+	return `
+WITH page AS (
+	SELECT kv.row_key, kv.expires_at IS NULL AS unset
+	FROM ${table} kv
+	WHERE kv.table_name = $1 AND kv.row_key > $2
+	ORDER BY kv.row_key
+	LIMIT $3
+), removed AS (
+	DELETE FROM ${table} kv
+	USING page
+	WHERE kv.table_name = $1 AND kv.row_key = page.row_key AND kv.expires_at IS NULL
+		AND kv.updated_at + make_interval(secs => $4::double precision) <= now()
+	RETURNING 1
+), expiring AS (
+	UPDATE ${table} kv
+	SET expires_at = kv.updated_at + make_interval(secs => $4::double precision)
+	FROM page
+	WHERE kv.table_name = $1 AND kv.row_key = page.row_key AND kv.expires_at IS NULL
+		AND kv.updated_at + make_interval(secs => $4::double precision) > now()
+	RETURNING 1
+)
+SELECT
+	(SELECT max(row_key) FROM page) AS last_row_key,
+	(SELECT count(*) FROM page WHERE unset) AS unset,
+	(SELECT count(*) FROM removed) AS deleted,
+	(SELECT count(*) FROM expiring) AS expiring`;
+}
+
+export async function expireLegacyDefaultTtlRows(
+	client: IPostgresClient,
+	deadlineMs: number,
+): Promise<LegacyDefaultTtlExpiryResult | null> {
+	if (await postgresKvPassIsFresh(client, DEFAULT_TTL_EXPIRY_MARKER, CLEAN_PASS_INTERVAL_MS)) {
+		return null;
+	}
+	const kvTable = quoteIdentifier(client.kvTable());
+	const sql = pageSql(kvTable);
+	const resume = await readResumePoint(client, kvTable);
+	const resumeIndex = resume === null ? -1 : DEFAULT_TTL_TABLES.findIndex((target) => target.name === resume.table);
+	let unset = resumeIndex < 0 ? 0 : resume!.unset;
+	let deleted = 0;
+	let expiring = 0;
+	for (let index = Math.max(resumeIndex, 0); index < DEFAULT_TTL_TABLES.length; index += 1) {
+		const target = DEFAULT_TTL_TABLES[index]!;
+		let cursor = index === resumeIndex ? resume!.rowKey : '';
+		for (;;) {
+			if (Date.now() >= deadlineMs) {
+				await writeResumePoint(client, kvTable, {table: target.name, rowKey: cursor, unset});
+				return {deleted, expiring, complete: false};
+			}
+			const result = await client.query<{
+				last_row_key: string | null;
+				unset: string;
+				deleted: string;
+				expiring: string;
+			}>(sql, [target.name, cursor, PAGE_SIZE, target.defaultTtlSeconds]);
+			const page = result.rows[0];
+			if (!page || page.last_row_key === null) {
+				break;
+			}
+			unset += Number(page.unset);
+			deleted += Number(page.deleted);
+			expiring += Number(page.expiring);
+			cursor = page.last_row_key;
+		}
+	}
+	await writeResumePoint(client, kvTable, null);
+	if (unset === 0) {
+		await recordPostgresKvCleanPass(client, DEFAULT_TTL_EXPIRY_MARKER);
+	}
+	return {deleted, expiring, complete: true};
+}

fluxer_api/src/api/database/PostgresKvQueryExecutor.ts

@@ -89,6 +89,28 @@ const NUMERIC_ROW_KEY_NUMBER_PATTERN = '^(-?[0-9]+(?:\\.[0-9]+)?(?:[eE][-+]?[0-9
 const EXPIRED_STORED_ROW = 'kv.expires_at IS NOT NULL AND kv.expires_at <= now()';
 const MERGED_ROW_DATA = `CASE WHEN ${EXPIRED_STORED_ROW} THEN EXCLUDED.row_data ELSE kv.row_data || EXCLUDED.row_data END`;
 const KEPT_EXPIRES_AT = `CASE WHEN ${EXPIRED_STORED_ROW} THEN NULL ELSE kv.expires_at END`;
+const NO_EXPIRY = 'infinity';
+
+export async function postgresKvPassIsFresh(
+	client: IPostgresClient,
+	marker: string,
+	maxAgeMs: number,
+): Promise<boolean> {
+	const result = await client.query(
+		`SELECT 1 FROM ${quoteIdentifier(client.kvTable())} WHERE table_name = $1 AND row_key = $2 AND (row_data ->> 'applied_at')::timestamptz > now() - make_interval(secs => $3::double precision)`,
+		[POSTGRES_KV_MIGRATION_TABLE, marker, maxAgeMs / 1000],
+	);
+	return result.rows.length > 0;
+}
+
+export async function recordPostgresKvCleanPass(client: IPostgresClient, marker: string): Promise<void> {
+	await client.query(
+		`INSERT INTO ${quoteIdentifier(client.kvTable())} (table_name, partition_key, row_key, row_data)
+VALUES ($1, $2, $2, jsonb_build_object('applied_at', now()))
+ON CONFLICT (table_name, row_key) DO UPDATE SET row_data = EXCLUDED.row_data, updated_at = now()`,
+		[POSTGRES_KV_MIGRATION_TABLE, marker],
+	);
+}
 
 function numericRowKeyExpr(column: string): string {
 	return `(COALESCE(substring(${column} from '${NUMERIC_ROW_KEY_BIGINT_PATTERN}'), substring(${column} from '${NUMERIC_ROW_KEY_NUMBER_PATTERN}'))::numeric)`;
@@ -333,20 +355,21 @@ function projectRow(row: Row, columns: ReadonlyArray<string> | undefined): Row {
 	return projected;
 }
 
-function rowComparator(meta: KvQueryMeta): (left: Row, right: Row) => number {
-	if (meta.orderBy) {
-		const column = meta.orderBy.col as string;
-		const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
-		return (left, right) => compareValues(left[column], right[column]) * direction;
-	}
-	const columns = meta.table.primaryKey as ReadonlyArray<string>;
-	return (left, right) => {
+function compareColumns(columns: ReadonlyArray<string>, left: Row, right: Row): number {
 	for (const column of columns) {
 		const cmp = compareValues(left[column], right[column]);
 		if (cmp !== 0) return cmp;
 	}
 	return 0;
-	};
+}
+
+function rowComparator(meta: KvQueryMeta): (left: Row, right: Row) => number {
+	const primaryKey = meta.table.primaryKey as ReadonlyArray<string>;
+	if (!meta.orderBy) return (left, right) => compareColumns(primaryKey, left, right);
+	const column = meta.orderBy.col as string;
+	const columns = [column, ...primaryKey.slice(primaryKey.indexOf(column) + 1)];
+	const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
+	return (left, right) => compareColumns(columns, left, right) * direction;
 }
 
 function sortRows(meta: KvQueryMeta, rows: Array<Row>): Array<Row> {
@@ -679,7 +702,7 @@ function logFullScan(meta: KvQueryMeta): void {
 	logWarn({table: meta.table.name, action: meta.action, where: shape.summary || 'none'}, 'Postgres KV full table scan');
 }
 
-function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null | undefined {
+function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | typeof NO_EXPIRY | null | undefined {
 	const ttlParam = meta.ttlParamName;
 	if (!ttlParam) return undefined;
 	const ttlRaw = params[ttlParam];
@@ -687,7 +710,13 @@ function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null |
 		throw new Error(`TTL parameter ${ttlParam} must be a number`);
 	}
 	const ttlSeconds = validateTtlSeconds(ttlRaw);
-	return ttlSeconds === 0 ? null : new Date(Date.now() + ttlSeconds * 1000);
+	if (ttlSeconds === 0) return meta.table.defaultTtlSeconds === undefined ? null : NO_EXPIRY;
+	return new Date(Date.now() + ttlSeconds * 1000);
+}
+
+function defaultExpiresAt(meta: KvQueryMeta): Date | undefined {
+	const ttlSeconds = meta.table.defaultTtlSeconds;
+	return ttlSeconds === undefined ? undefined : new Date(Date.now() + ttlSeconds * 1000);
 }
 
 function encodePageState(pageState: PageState): string {
@@ -1191,7 +1220,8 @@ export class PostgresKvQueryExecutor {
 				'kv_del_expired',
 			);
 		}
-		const expiresAt = ttlExpiresAt(meta, params) ?? null;
+		const explicit = ttlExpiresAt(meta, params);
+		const expiresAt = explicit === undefined ? (defaultExpiresAt(meta) ?? null) : explicit;
 		const result = await db.query(
 			`INSERT INTO ${this.table} AS kv (table_name, partition_key, row_key, row_data, expires_at, updated_at)
 VALUES ($1, $2, $3, $4::jsonb, $5, now())
@@ -1244,10 +1274,14 @@ WHERE NOT $6`,
 			}
 			bindings.push(JSON.stringify(encodeRow(paramsRow(params, meta.patchKeys))));
 			const assignments = [`row_data = kv.row_data || $${bindings.length}::jsonb`, 'updated_at = now()'];
-			const expiresAt = ttlExpiresAt(meta, params);
-			if (expiresAt !== undefined) {
-				bindings.push(expiresAt);
+			const explicit = ttlExpiresAt(meta, params);
+			const fallback = explicit === undefined ? defaultExpiresAt(meta) : undefined;
+			if (explicit !== undefined) {
+				bindings.push(explicit);
 				assignments.push(`expires_at = $${bindings.length}`);
+			} else if (fallback !== undefined) {
+				bindings.push(fallback);
+				assignments.push(`expires_at = GREATEST(kv.expires_at, $${bindings.length}::timestamptz)`);
 			}
 			sql = `UPDATE ${this.table} kv SET ${assignments.join(', ')} WHERE ${where}`;
 		}
@@ -1346,15 +1380,27 @@ WHERE NOT $6`,
 		for (const column of meta.patchKeys ?? []) {
 			incoming[column] = column in params ? params[column] : null;
 		}
-		const ttl = ttlExpiresAt(meta, params);
-		const expiresAtExpr = ttl === undefined ? KEPT_EXPIRES_AT : 'EXCLUDED.expires_at';
+		const explicit = ttlExpiresAt(meta, params);
+		const fallback = explicit === undefined ? defaultExpiresAt(meta) : undefined;
+		const [expiresAtExpr, statementName] =
+			explicit !== undefined
+				? ['EXCLUDED.expires_at', 'kv_patch_set_ttl']
+				: fallback !== undefined
+					? ['GREATEST(kv.expires_at, EXCLUDED.expires_at)', 'kv_patch_default_ttl']
+					: [KEPT_EXPIRES_AT, 'kv_patch_keep_ttl'];
 		await db.query(
 			`INSERT INTO ${this.table} AS kv (table_name, partition_key, row_key, row_data, expires_at, updated_at)
 VALUES ($1, $2, $3, $4::jsonb, $5, now())
 ON CONFLICT (table_name, row_key)
 DO UPDATE SET partition_key = EXCLUDED.partition_key, row_data = ${MERGED_ROW_DATA}, expires_at = ${expiresAtExpr}, updated_at = now()`,
-			[meta.table.name, partitionKey(meta, incoming), key, JSON.stringify(encodeRow(incoming)), ttl ?? null],
-			ttl === undefined ? 'kv_patch_keep_ttl' : 'kv_patch_set_ttl',
+			[
+				meta.table.name,
+				partitionKey(meta, incoming),
+				key,
+				JSON.stringify(encodeRow(incoming)),
+				explicit ?? fallback ?? null,
+			],
+			statementName,
 		);
 	}
 

fluxer_api/src/api/database/PostgresKvStatementNames.test.ts

@@ -52,6 +52,8 @@ const Composite: KvTableSpec<Row> = {
 	partitionKey: ['owner_id'],
 };
 
+const Expiring: KvTableSpec<Row> = {...Composite, name: 'stmt_expiring', defaultTtlSeconds: 600};
+
 const Bucketed: KvTableSpec<Row> = {
 	name: 'stmt_bucketed',
 	columns: ['bucket', 'item_id', 'payload'],
@@ -118,6 +120,7 @@ async function runShapes(): Promise<Array<Statement>> {
 			meta(Composite, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload'], ttlParamName: 'ttl_'}),
 			{...OWNER_ITEM, ttl_: 600} as CassandraParams,
 		],
+		[meta(Expiring, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload']}), OWNER_ITEM],
 	];
 	for (const [kvMeta, params] of cases) {
 		await executor.executeQuery({cql: `__stmt_${kvMeta.action}`, params, kvMeta: kvMeta as KvQueryMeta});
@@ -142,6 +145,7 @@ describe('PostgresKvQueryExecutor statement names', () => {
 			'kv_del_keys',
 			'kv_del_rowkeys',
 			'kv_get_row',
+			'kv_patch_default_ttl',
 			'kv_patch_keep_ttl',
 			'kv_patch_set_ttl',
 			'kv_sel_range',
@@ -231,6 +235,17 @@ async function exerciseKvShapes(executor: PostgresKvQueryExecutor): Promise<void
 		kvMeta: meta(Composite, 'select', [eq('owner_id'), eq('item_id')]) as KvQueryMeta,
 	});
 	expect(patched.map((row) => row.payload)).toEqual(['patched']);
+	await executor.executeQuery({
+		cql: '__stmt_patch_default_ttl',
+		params: {owner_id: 'o5', item_id: 'i5', payload: 'defaulted'} as CassandraParams,
+		kvMeta: meta(Expiring, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload']}) as KvQueryMeta,
+	});
+	const defaulted = await executor.executeQuery<Row>({
+		cql: '__stmt_point',
+		params: {owner_id: 'o5', item_id: 'i5'} as CassandraParams,
+		kvMeta: meta(Expiring, 'select', [eq('owner_id'), eq('item_id')]) as KvQueryMeta,
+	});
+	expect(defaulted.map((row) => row.payload)).toEqual(['defaulted']);
 	await executor.executeQuery({
 		cql: '__stmt_delete',
 		params: {owner_id: 'o0', item_id: 'i0'} as CassandraParams,
@@ -323,6 +338,7 @@ describe.skipIf(!dockerAvailable)('PostgresKvQueryExecutor statement names again
 			'kv_del_expired',
 			'kv_del_rowkeys',
 			'kv_get_row',
+			'kv_patch_default_ttl',
 			'kv_patch_keep_ttl',
 			'kv_patch_set_ttl',
 			'kv_sel_range',

fluxer_api/src/api/database/PostgresScramIterations.test.ts

@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {startDockerContainer} from '@app/api/test/DockerTestContainer';
+import {getDefaultPostgresClient, initPostgres, shutdownPostgres} from '@pkgs/postgres/src/Client';
+import {afterAll, beforeAll, describe, expect, it} from 'vitest';
+
+const CONTAINER = `fluxer-kvscram-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+const SCRAM_ITERATIONS = 200_000;
+
+async function sleep(ms: number): Promise<void> {
+	await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function freePort(): Promise<number> {
+	return new Promise((resolve, reject) => {
+		const server = createServer();
+		server.on('error', reject);
+		server.listen(0, '127.0.0.1', () => {
+			const address = server.address();
+			if (typeof address === 'string' || address === null) {
+				reject(new Error('no port'));
+				return;
+			}
+			const port = address.port;
+			server.close(() => resolve(port));
+		});
+	});
+}
+
+describe.skipIf(!dockerAvailable)('postgres client against a server with raised SCRAM iterations', () => {
+	let port: number;
+
+	beforeAll(async () => {
+		port = await freePort();
+		startDockerContainer([
+			'run',
+			'-d',
+			'--name',
+			CONTAINER,
+			'-e',
+			'POSTGRES_USER=fluxer',
+			'-e',
+			'POSTGRES_PASSWORD=fluxer',
+			'-e',
+			'POSTGRES_DB=fluxer',
+			'-p',
+			`127.0.0.1:${port}:5432`,
+			'postgres:16-alpine',
+			'-c',
+			'fsync=off',
+			'-c',
+			`scram_iterations=${SCRAM_ITERATIONS}`,
+		]);
+		let ready = false;
+		for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+			await sleep(500);
+			const probe = spawnSync(
+				'docker',
+				['exec', CONTAINER, 'psql', '-h', '127.0.0.1', '-U', 'fluxer', '-d', 'fluxer', '-Atc', 'SELECT 1'],
+				{stdio: 'ignore'},
+			);
+			ready = probe.status === 0;
+		}
+		if (!ready) throw new Error('postgres never came up');
+		const rehash = spawnSync(
+			'docker',
+			['exec', CONTAINER, 'psql', '-U', 'fluxer', '-d', 'fluxer', '-Atc', "ALTER ROLE fluxer PASSWORD 'fluxer'"],
+			{
+				stdio: 'ignore',
+			},
+		);
+		if (rehash.status !== 0) throw new Error('could not re-hash the role password');
+	}, 900_000);
+
+	afterAll(async () => {
+		await shutdownPostgres().catch(() => {});
+		spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+	});
+
+	it('connects when the role verifier uses more iterations than the driver default allows', async () => {
+		await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 1});
+		const verifier = await getDefaultPostgresClient().query<{rolpassword: string}>(
+			"SELECT rolpassword FROM pg_authid WHERE rolname = 'fluxer'",
+		);
+		expect(verifier.rows[0]?.rolpassword.startsWith(`SCRAM-SHA-256$${SCRAM_ITERATIONS}:`)).toBe(true);
+	});
+});

fluxer_api/src/api/donation/DonationTables.ts

@@ -13,6 +13,7 @@ import {
 	type DonorMagicLinkTokenRow,
 	type DonorRow,
 } from '@app/api/database/types/DonationTypes';
+import {seconds} from 'itty-time';
 
 export const Donors = defineTable<DonorRow, 'email'>({
 	name: 'donors',
@@ -43,9 +44,11 @@ export const DonorMagicLinkTokens = defineTable<DonorMagicLinkTokenRow, 'token_'
 	name: 'donor_magic_link_tokens',
 	columns: DONOR_MAGIC_LINK_TOKEN_COLUMNS,
 	primaryKey: ['token_'],
+	defaultTtlSeconds: seconds('15 minutes'),
 });
 export const DonorMagicLinkTokensByEmail = defineTable<DonorMagicLinkTokenByEmailRow, 'donor_email' | 'token_'>({
 	name: 'donor_magic_link_tokens_by_email',
 	columns: DONOR_MAGIC_LINK_TOKEN_BY_EMAIL_COLUMNS,
 	primaryKey: ['donor_email', 'token_'],
+	defaultTtlSeconds: seconds('15 minutes'),
 });

fluxer_api/src/api/guild/GuildAuditLogService.ts

@@ -15,6 +15,8 @@ import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType';
 import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
 import {ms} from 'itty-time';
 
+const MESSAGE_DELETE_BATCH_DELAY_MS = ms('30 seconds');
+
 interface MessageDeleteBatchGroup {
 	logs: Array<GuildAuditLog>;
 	userId: UserID;
@@ -81,14 +83,15 @@ export class GuildAuditLogService {
 	}
 
 	async scheduleMessageDeleteBatchJob(guildId: GuildID): Promise<void> {
-		const runAt = new Date(Date.now() + ms('30 seconds'));
+		const batchWindow = Math.floor(Date.now() / MESSAGE_DELETE_BATCH_DELAY_MS);
 		await this.workerService.addJob(
 			'batchGuildAuditLogMessageDeletes',
 			{guildId: guildId.toString()},
 			{
-				jobKey: `batch-audit-log-message-deletes:${guildId}`,
-				runAt,
+				jobKey: `batch-audit-log-message-deletes:${guildId}:${batchWindow}`,
+				runAt: new Date((batchWindow + 2) * MESSAGE_DELETE_BATCH_DELAY_MS),
 				maxAttempts: 3,
+				skipLedger: true,
 			},
 		);
 	}

fluxer_api/src/api/guild/tests/GuildAuditLogService.test.ts

@@ -60,7 +60,7 @@ function createService(roleNames: Map<string, string> = new Map()) {
 		{addJob} as unknown as IWorkerService<WorkerTaskName>,
 		{dispatchGuild} as unknown as IGatewayService,
 	);
-	return {service, createAuditLog, batchDeleteAndCreateAuditLogs, getRole, dispatchGuild};
+	return {service, createAuditLog, batchDeleteAndCreateAuditLogs, getRole, dispatchGuild, addJob};
 }
 
 function overwrites(
@@ -376,3 +376,26 @@ describe('GuildAuditLogService.recordPermissionOverwriteDiff', () => {
 		expect(dispatchGuild).not.toHaveBeenCalled();
 	});
 });
+
+describe('GuildAuditLogService.scheduleMessageDeleteBatchJob', () => {
+	it('gives every delete in one 30 second window a single batch job that runs after the window closes, without a ledger row', async () => {
+		vi.useFakeTimers({toFake: ['Date']});
+		try {
+			const {service, addJob} = createService();
+			for (const at of ['2026-09-21T12:00:00.000Z', '2026-09-21T12:00:29.999Z', '2026-09-21T12:00:40.000Z']) {
+				vi.setSystemTime(new Date(at));
+				await service.scheduleMessageDeleteBatchJob(GUILD_ID);
+			}
+			const options = addJob.mock.calls.map((call) => call[2] as {jobKey: string; runAt: Date; skipLedger: boolean});
+			expect(options.every((option) => option.skipLedger)).toBe(true);
+			expect(options[0]!.jobKey).toBe(options[1]!.jobKey);
+			expect(options[2]!.jobKey).not.toBe(options[1]!.jobKey);
+			expect(options[0]!.runAt.getTime()).toBeGreaterThan(new Date('2026-09-21T12:00:29.999Z').getTime());
+			expect(options[1]!.runAt).toEqual(options[0]!.runAt);
+			expect(options[2]!.runAt).toEqual(new Date('2026-09-21T12:01:30.000Z'));
+			expect(options[2]!.runAt.getTime() - options[0]!.runAt.getTime()).toBe(30_000);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+});

fluxer_api/src/api/jobs/IJobLedgerRepository.ts

@@ -32,10 +32,12 @@ export interface ListJobsResult {
 }
 
 export abstract class IJobLedgerRepository {
-	abstract createJob(input: CreateJobInput): Promise<void>;
+	abstract createJob(input: CreateJobInput): Promise<Date>;
 
 	abstract getJob(jobId: bigint): Promise<JobByIdRow | null>;
 
+	abstract discardJob(jobId: bigint, createdAt: Date): Promise<void>;
+
 	abstract markRunning(jobId: bigint, lane: string): Promise<void>;
 
 	abstract markSucceeded(jobId: bigint, result: Record<string, unknown> | null): Promise<void>;

fluxer_api/src/api/jobs/JobLedgerRepository.test.ts

@@ -1,9 +1,24 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import {setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
+import {
+	type CassandraQueryExecutorForTesting,
+	executeQuery,
+	fetchMany,
+	fetchOne,
+	setCassandraQueryExecutorForTesting,
+} from '@app/api/database/CassandraQueryExecution';
+import {Db, type PreparedQuery} from '@app/api/database/CassandraTypes';
 import type {JobStatus} from '@app/api/database/types/JobLedgerTypes';
-import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
+import {
+	EXPIRED_JOB_ERROR,
+	JOB_LEDGER_TTL_SECONDS,
+	JOB_STALE_AFTER_MS,
+	JobLedgerRepository,
+} from '@app/api/jobs/JobLedgerRepository';
+import {describeListJobsPaging} from '@app/api/jobs/ListJobsPagingSuite';
+import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
 import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
+import {createSnowflake} from '@fluxer/snowflake/src/Snowflake';
 import {afterEach, beforeEach, describe, expect, it} from 'vitest';
 
 let executor: InMemoryCassandraQueryExecutor;
@@ -22,6 +37,20 @@ async function createJob(repository: JobLedgerRepository, jobId: bigint, taskTyp
 	});
 }
 
+async function createJobAt(repository: JobLedgerRepository, jobId: bigint): Promise<Date> {
+	return repository.createJob({
+		jobId,
+		taskType: 'batchGuildAuditLogMessageDeletes',
+		payload: {},
+		requestedByUserId: null,
+		auditLogReason: null,
+		maxAttempts: 3,
+		runAt: null,
+		jetStreamLane: null,
+		jetStreamSeq: null,
+	});
+}
+
 async function listJobIdsByStatus(repository: JobLedgerRepository, status: JobStatus): Promise<Array<bigint>> {
 	const result = await repository.listJobs({limit: 50, cursor: null, filters: {status}, maxLookbackDays: 1});
 	return result.jobs.map((job) => job.job_id);
@@ -130,6 +159,197 @@ describe('JobLedgerRepository listJobs pagination', () => {
 			maxLookbackDays: 1,
 		});
 
-		expect(result.jobs.map((job) => job.job_id)).toEqual([1_000n, 1_001n, 1_002n, 1_003n, 1_004n]);
+		expect(result.jobs.map((job) => job.job_id)).toEqual([1_004n, 1_003n, 1_002n, 1_001n, 1_000n]);
+	});
+});
+
+describe('JobLedgerRepository listJobs on the in-memory executor', () => {
+	beforeEach(() => {
+		executor = new InMemoryCassandraQueryExecutor();
+		setCassandraQueryExecutorForTesting(executor);
+	});
+
+	afterEach(() => {
+		executor.reset();
+		setCassandraQueryExecutorForTesting(null);
+	});
+
+	describeListJobsPaging();
+});
+
+let staleSequence = 0;
+
+function jobIdAgedDays(days: number): bigint {
+	staleSequence += 1;
+	return createSnowflake({timestamp: Date.now() - days * 86_400_000, sequence: staleSequence % 4096, workerId: 1});
+}
+
+async function createAgedJob(repository: JobLedgerRepository, days: number): Promise<bigint> {
+	const jobId = jobIdAgedDays(days);
+	await createJob(repository, jobId, 'syncUrlBlocklists');
+	return jobId;
+}
+
+async function activeJobIds(repository: JobLedgerRepository): Promise<Array<bigint>> {
+	return (await repository.listActiveJobs()).map((job) => job.job_id).sort((a, b) => (a < b ? -1 : 1));
+}
+
+function sweep(repository: JobLedgerRepository, maxCleared = 100) {
+	return repository.expireStaleActiveJobs({
+		staleBeforeMs: Date.now() - JOB_STALE_AFTER_MS,
+		pageSize: 100,
+		maxCleared,
+	});
+}
+
+describe('JobLedgerRepository expireStaleActiveJobs', () => {
+	beforeEach(() => {
+		executor = new InMemoryCassandraQueryExecutor();
+		setCassandraQueryExecutorForTesting(executor);
+	});
+
+	afterEach(() => {
+		executor.reset();
+		setCassandraQueryExecutorForTesting(null);
+	});
+
+	it('dead-letters queued and running jobs the jobs stream has outlived and keeps younger ones active', async () => {
+		const repository = new JobLedgerRepository();
+		const staleQueued = await createAgedJob(repository, 9);
+		const staleRunning = await createAgedJob(repository, 9);
+		await repository.markRunning(staleRunning, 'batch');
+		const weekOld = await createAgedJob(repository, 7);
+		const fresh = await createAgedJob(repository, 0);
+
+		expect(await sweep(repository)).toEqual({cleared: 2, expired: 2, complete: true});
+
+		for (const jobId of [staleQueued, staleRunning]) {
+			const job = await repository.getJob(jobId);
+			expect(job?.status).toBe('deadletter');
+			expect(job?.error_message).toBe(EXPIRED_JOB_ERROR);
+			expect(job?.completed_at).toBeInstanceOf(Date);
+		}
+		expect(await activeJobIds(repository)).toEqual([weekOld, fresh].sort((a, b) => (a < b ? -1 : 1)));
+		expect((await repository.getJob(weekOld))?.status).toBe('queued');
+	});
+
+	it('drops a stale active row without touching a finished or missing job', async () => {
+		const repository = new JobLedgerRepository();
+		const finished = await createAgedJob(repository, 9);
+		await repository.markSucceeded(finished, null);
+		const orphan = jobIdAgedDays(9);
+		for (const jobId of [finished, orphan]) {
+			await executeQuery(
+				JobsActive.patchByPkWithTtl({job_id: jobId}, {status: Db.set('running')}, JOB_LEDGER_TTL_SECONDS),
+			);
+		}
+
+		expect(await sweep(repository)).toEqual({cleared: 2, expired: 0, complete: true});
+
+		const job = await repository.getJob(finished);
+		expect(job?.status).toBe('succeeded');
+		expect(job?.error_message).toBeNull();
+		expect(await fetchOne(JobsById.select({where: JobsById.where.eq('job_id')}).bind({job_id: orphan}))).toBeNull();
+		expect(await fetchMany(JobsActive.select().bind({}))).toEqual([]);
+	});
+
+	it('stops at its per-run cap and picks up the rest on the next run', async () => {
+		const repository = new JobLedgerRepository();
+		for (let index = 0; index < 3; index += 1) {
+			await createAgedJob(repository, 9);
+		}
+
+		expect(await sweep(repository, 2)).toEqual({cleared: 2, expired: 2, complete: false});
+		expect(await activeJobIds(repository)).toHaveLength(1);
+		expect(await sweep(repository, 2)).toEqual({cleared: 1, expired: 1, complete: true});
+		expect(await activeJobIds(repository)).toEqual([]);
+	});
+});
+
+describe('JobLedgerRepository getJob', () => {
+	beforeEach(() => {
+		executor = new InMemoryCassandraQueryExecutor();
+		setCassandraQueryExecutorForTesting(executor);
+	});
+
+	afterEach(() => {
+		executor.reset();
+		setCassandraQueryExecutorForTesting(null);
+	});
+
+	it('hides a job row that lost its status, creation time or task type', async () => {
+		const repository = new JobLedgerRepository();
+		await createJob(repository, 9n, 'syncUrlBlocklists');
+		expect((await repository.getJob(9n))?.status).toBe('queued');
+		for (const column of ['status', 'created_at', 'task_type'] as const) {
+			await createJob(repository, 9n, 'syncUrlBlocklists');
+			await executeQuery(JobsById.patchByPk({job_id: 9n}, {[column]: Db.clear()}));
+			expect(await repository.getJob(9n)).toBeNull();
+		}
+	});
+});
+
+describe('JobLedgerRepository discardJob', () => {
+	let inner: InMemoryCassandraQueryExecutor;
+	let log: Array<string>;
+	let failDeleteOn: string | null;
+
+	beforeEach(() => {
+		inner = new InMemoryCassandraQueryExecutor();
+		log = [];
+		failDeleteOn = null;
+		const wrapper: CassandraQueryExecutorForTesting = {
+			async executeQuery<T>(query: PreparedQuery) {
+				const meta = query.kvMeta;
+				if (meta) log.push(`${meta.action} ${meta.table.name}`);
+				if (meta?.action === 'delete' && meta.table.name === failDeleteOn) {
+					throw new Error('write timeout');
+				}
+				return inner.executeQuery<T>(query);
+			},
+			executeBatch: (queries, atomic) => inner.executeBatch(queries, atomic),
+		};
+		setCassandraQueryExecutorForTesting(wrapper);
+	});
+
+	afterEach(() => {
+		inner.reset();
+		setCassandraQueryExecutorForTesting(null);
+	});
+
+	it('removes every ledger row of a duplicate with three deletes and no read', async () => {
+		const repository = new JobLedgerRepository();
+		const createdAt = await createJobAt(repository, 7n);
+		log.length = 0;
+
+		await repository.discardJob(7n, createdAt);
+
+		expect([...log].sort()).toEqual(['delete jobs_active', 'delete jobs_by_day_bucket', 'delete jobs_by_id']);
+		expect(await repository.getJob(7n)).toBeNull();
+		expect(await repository.listActiveJobs()).toEqual([]);
+		expect(
+			await fetchMany(
+				JobsByDayBucket.select({where: JobsByDayBucket.where.eq('bucket_day')}).bind({
+					bucket_day: createdAt.toISOString().slice(0, 10),
+				}),
+			),
+		).toEqual([]);
+	});
+
+	it('keeps deleting the other ledger rows when one delete fails', async () => {
+		const repository = new JobLedgerRepository();
+		const createdAt = await createJobAt(repository, 8n);
+		failDeleteOn = 'jobs_by_id';
+
+		await expect(repository.discardJob(8n, createdAt)).rejects.toThrow('write timeout');
+
+		expect(await repository.listActiveJobs()).toEqual([]);
+		expect(
+			await fetchMany(
+				JobsByDayBucket.select({where: JobsByDayBucket.where.eq('bucket_day')}).bind({
+					bucket_day: createdAt.toISOString().slice(0, 10),
+				}),
+			),
+		).toEqual([]);
 	});
 });

fluxer_api/src/api/jobs/JobLedgerRepository.ts

@@ -1,6 +1,14 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import {BatchBuilder, deleteOneOrMany, fetchMany, fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
+import {
+	BatchBuilder,
+	deleteOneOrMany,
+	fetchMany,
+	fetchOne,
+	fetchPage,
+	type PagedQueryResult,
+	upsertOne,
+} from '@app/api/database/CassandraQueryExecution';
 import {Db} from '@app/api/database/CassandraTypes';
 import type {JobActiveRow, JobByDayBucketRow, JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
 import {
@@ -11,6 +19,17 @@ import {
 	type ListJobsResult,
 } from '@app/api/jobs/IJobLedgerRepository';
 import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
+import {awaitAll} from '@app/api/utils/ConcurrencyUtils';
+import {JOBS_STREAM_MAX_AGE_MS} from '@app/api/worker/JetStreamWorkerQueue';
+import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';
+import {ms, seconds} from 'itty-time';
+
+export const JOB_LEDGER_TTL_SECONDS = seconds('90 days');
+export const JOB_STALE_AFTER_MS = JOBS_STREAM_MAX_AGE_MS + ms('1 day');
+export const EXPIRED_JOB_ERROR = 'Expired from the job queue';
+
+const JOB_LEDGER_RETENTION_DAYS = JOB_LEDGER_TTL_SECONDS / seconds('1 day');
+const NEWEST_FIRST = {col: 'created_at', direction: 'DESC'} as const;
 
 const FETCH_JOB_BY_ID_QUERY = JobsById.select({
 	where: JobsById.where.eq('job_id'),
@@ -19,13 +38,58 @@ const FETCH_CANCEL_REQUESTED_QUERY = JobsById.select({
 	where: JobsById.where.eq('job_id'),
 });
 const ACTIVE_JOBS_QUERY = JobsActive.select();
+const ACTIVE_JOB_IDS_QUERY = JobsActive.select({columns: ['job_id']});
+const JOBS_AFTER_IN_TIE_QUERY = JobsByDayBucket.select({
+	where: [
+		JobsByDayBucket.where.eq('bucket_day'),
+		JobsByDayBucket.where.eq('created_at'),
+		JobsByDayBucket.where.lt('job_id'),
+	],
+	orderBy: NEWEST_FIRST,
+});
+
+type LedgerPosition = Pick<ListJobsCursor, 'createdAt' | 'jobId'>;
 
 function bucketDayFor(d: Date): string {
 	return d.toISOString().slice(0, 10);
 }
 
+function previousBucketDay(day: string): string {
+	const date = new Date(`${day}T00:00:00Z`);
+	date.setUTCDate(date.getUTCDate() - 1);
+	return bucketDayFor(date);
+}
+
+function dayJobsQuery(olderThanPosition: boolean, limit: number | null) {
+	return JobsByDayBucket.select({
+		where: olderThanPosition
+			? [JobsByDayBucket.where.eq('bucket_day'), JobsByDayBucket.where.lt('created_at')]
+			: JobsByDayBucket.where.eq('bucket_day'),
+		orderBy: NEWEST_FIRST,
+		...(limit === null ? {} : {limit}),
+	});
+}
+
+async function fetchDayAfter(
+	bucketDay: string,
+	after: LedgerPosition | null,
+	limit: number | null,
+): Promise<{rows: Array<JobByDayBucketRow>; exhausted: boolean}> {
+	if (after === null) {
+		const rows = await fetchMany<JobByDayBucketRow>(dayJobsQuery(false, limit).bind({bucket_day: bucketDay}));
+		return {rows, exhausted: limit === null || rows.length < limit};
+	}
+	const ties = await fetchMany<JobByDayBucketRow>(
+		JOBS_AFTER_IN_TIE_QUERY.bind({bucket_day: bucketDay, created_at: after.createdAt, job_id: after.jobId}),
+	);
+	const older = await fetchMany<JobByDayBucketRow>(
+		dayJobsQuery(true, limit).bind({bucket_day: bucketDay, created_at: after.createdAt}),
+	);
+	return {rows: [...ties, ...older], exhausted: limit === null || older.length < limit};
+}
+
 export class JobLedgerRepository extends IJobLedgerRepository {
-	async createJob(input: CreateJobInput): Promise<void> {
+	async createJob(input: CreateJobInput): Promise<Date> {
 		const now = new Date();
 		const status: JobStatus = 'queued';
 		const idRow: JobByIdRow = {
@@ -68,39 +132,62 @@ export class JobLedgerRepository extends IJobLedgerRepository {
 			started_at: null,
 		};
 		const batch = new BatchBuilder();
-		batch.addPrepared(JobsById.insert(idRow));
-		batch.addPrepared(JobsByDayBucket.insert(bucketRow));
-		batch.addPrepared(JobsActive.insert(activeRow));
+		batch.addPrepared(JobsById.insertWithTtl(idRow, JOB_LEDGER_TTL_SECONDS));
+		batch.addPrepared(JobsByDayBucket.insertWithTtl(bucketRow, JOB_LEDGER_TTL_SECONDS));
+		batch.addPrepared(JobsActive.insertWithTtl(activeRow, JOB_LEDGER_TTL_SECONDS));
 		await batch.executeChunked(10, false);
+		return now;
 	}
 
 	async getJob(jobId: bigint): Promise<JobByIdRow | null> {
-		return fetchOne<JobByIdRow>(FETCH_JOB_BY_ID_QUERY.bind({job_id: jobId}));
+		const row = await fetchOne<JobByIdRow>(FETCH_JOB_BY_ID_QUERY.bind({job_id: jobId}));
+		return row?.created_at && row.task_type && row.status ? row : null;
+	}
+
+	async discardJob(jobId: bigint, createdAt: Date): Promise<void> {
+		await awaitAll(
+			[
+				deleteOneOrMany(
+					JobsByDayBucket.deleteByPk({bucket_day: bucketDayFor(createdAt), created_at: createdAt, job_id: jobId}),
+				),
+				deleteOneOrMany(JobsById.deleteByPk({job_id: jobId})),
+				deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId})),
+			],
+			'Ledger discard left rows behind',
+		);
 	}
 
 	async markRunning(jobId: bigint, lane: string): Promise<void> {
 		const startedAt = new Date();
 		const status: JobStatus = 'running';
 		await upsertOne(
-			JobsById.patchByPk(
+			JobsById.patchByPkWithTtl(
 				{job_id: jobId},
 				{status: Db.set(status), started_at: Db.set(startedAt), jet_stream_lane: Db.set(lane)},
+				JOB_LEDGER_TTL_SECONDS,
+			),
+		);
+		await upsertOne(
+			JobsActive.patchByPkWithTtl(
+				{job_id: jobId},
+				{status: Db.set(status), started_at: Db.set(startedAt)},
+				JOB_LEDGER_TTL_SECONDS,
 			),
 		);
-		await upsertOne(JobsActive.patchByPk({job_id: jobId}, {status: Db.set(status), started_at: Db.set(startedAt)}));
 	}
 
 	async markSucceeded(jobId: bigint, result: Record<string, unknown> | null): Promise<void> {
 		const completedAt = new Date();
 		const status: JobStatus = 'succeeded';
 		await upsertOne(
-			JobsById.patchByPk(
+			JobsById.patchByPkWithTtl(
 				{job_id: jobId},
 				{
 					status: Db.set(status),
 					completed_at: Db.set(completedAt),
 					result: result === null ? Db.clear() : Db.set(JSON.stringify(result)),
 				},
+				JOB_LEDGER_TTL_SECONDS,
 			),
 		);
 		await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
@@ -109,7 +196,13 @@ export class JobLedgerRepository extends IJobLedgerRepository {
 	async markCancelled(jobId: bigint): Promise<void> {
 		const completedAt = new Date();
 		const status: JobStatus = 'cancelled';
-		await upsertOne(JobsById.patchByPk({job_id: jobId}, {status: Db.set(status), completed_at: Db.set(completedAt)}));
+		await upsertOne(
+			JobsById.patchByPkWithTtl(
+				{job_id: jobId},
+				{status: Db.set(status), completed_at: Db.set(completedAt)},
+				JOB_LEDGER_TTL_SECONDS,
+			),
+		);
 		await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
 	}
 
@@ -117,9 +210,10 @@ export class JobLedgerRepository extends IJobLedgerRepository {
 		const completedAt = new Date();
 		const status: JobStatus = 'deadletter';
 		await upsertOne(
-			JobsById.patchByPk(
+			JobsById.patchByPkWithTtl(
 				{job_id: jobId},
 				{status: Db.set(status), completed_at: Db.set(completedAt), error_message: Db.set(errorMessage)},
+				JOB_LEDGER_TTL_SECONDS,
 			),
 		);
 		await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
@@ -127,27 +221,30 @@ export class JobLedgerRepository extends IJobLedgerRepository {
 
 	async reportProgress(jobId: bigint, current: number, total: number | null, message: string | null): Promise<void> {
 		await upsertOne(
-			JobsById.patchByPk(
+			JobsById.patchByPkWithTtl(
 				{job_id: jobId},
 				{
 					progress_current: Db.set(BigInt(current)),
 					progress_total: total === null ? Db.clear() : Db.set(BigInt(total)),
 					progress_message: message === null ? Db.clear() : Db.set(message),
 				},
+				JOB_LEDGER_TTL_SECONDS,
 			),
 		);
 	}
 
 	async setContextLink(jobId: bigint, link: string): Promise<void> {
-		await upsertOne(JobsById.patchByPk({job_id: jobId}, {context_link: Db.set(link)}));
+		await upsertOne(JobsById.patchByPkWithTtl({job_id: jobId}, {context_link: Db.set(link)}, JOB_LEDGER_TTL_SECONDS));
 	}
 
 	async setJetStreamSeq(jobId: bigint, seq: string): Promise<void> {
-		await upsertOne(JobsById.patchByPk({job_id: jobId}, {jet_stream_seq: Db.set(seq)}));
+		await upsertOne(JobsById.patchByPkWithTtl({job_id: jobId}, {jet_stream_seq: Db.set(seq)}, JOB_LEDGER_TTL_SECONDS));
 	}
 
 	async requestCancel(jobId: bigint): Promise<void> {
-		await upsertOne(JobsById.patchByPk({job_id: jobId}, {cancel_requested: Db.set(true)}));
+		await upsertOne(
+			JobsById.patchByPkWithTtl({job_id: jobId}, {cancel_requested: Db.set(true)}, JOB_LEDGER_TTL_SECONDS),
+		);
 	}
 
 	async isCancelRequested(jobId: bigint): Promise<boolean> {
@@ -160,7 +257,9 @@ export class JobLedgerRepository extends IJobLedgerRepository {
 	async incrementAttempts(jobId: bigint): Promise<void> {
 		const row = await this.getJob(jobId);
 		if (!row) return;
-		await upsertOne(JobsById.patchByPk({job_id: jobId}, {attempts: Db.set(row.attempts + 1)}));
+		await upsertOne(
+			JobsById.patchByPkWithTtl({job_id: jobId}, {attempts: Db.set(row.attempts + 1)}, JOB_LEDGER_TTL_SECONDS),
+		);
 	}
 
 	async listJobs(opts: {
@@ -169,59 +268,74 @@ export class JobLedgerRepository extends IJobLedgerRepository {
 		filters: ListJobsFilters;
 		maxLookbackDays: number;
 	}): Promise<ListJobsResult> {
-		const {limit, cursor, filters, maxLookbackDays} = opts;
-		const startBucket = cursor ? new Date(`${cursor.bucketDay}T00:00:00Z`) : new Date();
-		const hasFilters = Boolean(
-			filters.status ||
-				filters.taskType ||
-				(filters.requestedByUserId !== undefined && filters.requestedByUserId !== null),
-		);
-		const collected: Array<JobByIdRow> = [];
-		let nextCursor: ListJobsCursor | null = null;
-		for (let dayOffset = 0; dayOffset <= maxLookbackDays && collected.length < limit; dayOffset++) {
-			const bucketDate = new Date(startBucket);
-			bucketDate.setUTCDate(bucketDate.getUTCDate() - dayOffset);
-			const bucketDay = bucketDayFor(bucketDate);
-			const remaining = limit - collected.length + 1;
-			const bucketLimit = hasFilters ? {} : {limit: remaining};
-			const useCursor = dayOffset === 0 && cursor !== null;
-			let bucketRows: Array<JobByDayBucketRow>;
-			if (useCursor && cursor) {
-				const query = JobsByDayBucket.select({
-					where: [JobsByDayBucket.where.eq('bucket_day'), JobsByDayBucket.where.lt('created_at')],
-					...bucketLimit,
-				});
-				bucketRows = await fetchMany<JobByDayBucketRow>(
-					query.bind({bucket_day: bucketDay, created_at: cursor.createdAt}),
+		const {limit, cursor, filters} = opts;
+		const now = Date.now();
+		const lookbackDays = Math.min(opts.maxLookbackDays, JOB_LEDGER_RETENTION_DAYS);
+		const oldestDay = bucketDayFor(new Date(now - lookbackDays * ms('1 day')));
+		const requestedBy = filters.requestedByUserId ?? null;
+		const wholeDays = Boolean(filters.status || filters.taskType || requestedBy !== null);
+		const jobs: Array<JobByIdRow> = [];
+		let lastRow: JobByDayBucketRow | null = null;
+		let day = bucketDayFor(new Date(Math.min(cursor ? cursor.createdAt.getTime() : now, now)));
+		let after: LedgerPosition | null = cursor;
+		while (jobs.length < limit && day >= oldestDay) {
+			const {rows, exhausted} = await fetchDayAfter(day, after, wholeDays ? null : limit - jobs.length);
+			for (const row of rows) {
+				after = {createdAt: row.created_at, jobId: row.job_id};
+				if (filters.taskType && row.task_type !== filters.taskType) continue;
+				if (requestedBy !== null && row.requested_by_user_id !== requestedBy) continue;
+				const job = await this.getJob(row.job_id);
+				if (!job || (filters.status && job.status !== filters.status)) continue;
+				jobs.push(job);
+				lastRow = row;
+				if (jobs.length === limit) break;
+			}
+			if (exhausted) {
+				day = previousBucketDay(day);
+				after = null;
+			}
+		}
+		return {
+			jobs,
+			nextCursor:
+				lastRow && jobs.length === limit
+					? {bucketDay: lastRow.bucket_day, createdAt: lastRow.created_at, jobId: lastRow.job_id}
+					: null,
+		};
+	}
+
+	async expireStaleActiveJobs(opts: {
+		staleBeforeMs: number;
+		pageSize: number;
+		maxCleared: number;
+	}): Promise<{cleared: number; expired: number; complete: boolean}> {
+		let cleared = 0;
+		let expired = 0;
+		let pageState: string | null = null;
+		do {
+			const page: PagedQueryResult<Pick<JobActiveRow, 'job_id'>> = await fetchPage(
+				ACTIVE_JOB_IDS_QUERY.bind({}),
+				undefined,
+				{
+					pageSize: opts.pageSize,
+					pageState,
+				},
 			);
+			for (const {job_id: jobId} of page.rows) {
+				if (snowflakeToDate(jobId).getTime() >= opts.staleBeforeMs) continue;
+				if (cleared >= opts.maxCleared) return {cleared, expired, complete: false};
+				const job = await this.getJob(jobId);
+				if (job?.status === 'queued' || job?.status === 'running') {
+					await this.markDeadletter(jobId, EXPIRED_JOB_ERROR);
+					expired += 1;
 				} else {
-				const query = JobsByDayBucket.select({
-					where: JobsByDayBucket.where.eq('bucket_day'),
-					...bucketLimit,
-				});
-				bucketRows = await fetchMany<JobByDayBucketRow>(query.bind({bucket_day: bucketDay}));
+					await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
 				}
-			for (const r of bucketRows) {
-				if (filters.taskType && r.task_type !== filters.taskType) continue;
-				if (filters.requestedByUserId !== undefined && filters.requestedByUserId !== null) {
-					if (r.requested_by_user_id !== filters.requestedByUserId) continue;
+				cleared += 1;
 			}
-				const fullRow = await this.getJob(r.job_id);
-				if (!fullRow) continue;
-				if (filters.status && fullRow.status !== filters.status) continue;
-				if (collected.length >= limit) {
-					nextCursor = {bucketDay, createdAt: r.created_at, jobId: r.job_id};
-					break;
-				}
-				collected.push(fullRow);
-			}
-			if (nextCursor) break;
-		}
-		if (nextCursor === null && collected.length >= limit) {
-			const last = collected[collected.length - 1];
-			nextCursor = {bucketDay: bucketDayFor(last.created_at), createdAt: last.created_at, jobId: last.job_id};
-		}
-		return {jobs: collected, nextCursor};
+			pageState = page.pageState;
+		} while (pageState !== null);
+		return {cleared, expired, complete: true};
 	}
 
 	async listActiveJobs(): Promise<Array<JobByIdRow>> {

fluxer_api/src/api/jobs/ListJobsPagingSuite.ts

@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {upsertOne} from '@app/api/database/CassandraQueryExecution';
+import type {JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
+import type {ListJobsCursor, ListJobsFilters} from '@app/api/jobs/IJobLedgerRepository';
+import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
+import {JobsByDayBucket, JobsById} from '@app/api/Tables';
+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
+
+const DAY_MS = 86_400_000;
+const HOUR_MS = 3_600_000;
+const TODAY_NOON = new Date(`${new Date().toISOString().slice(0, 10)}T12:00:00.000Z`);
+const ADMIN_USER_ID = 4_242n;
+
+interface ListedPage {
+	ids: Array<bigint>;
+	cursor: ListJobsCursor | null;
+}
+
+function at(daysAgo: number, hour: number): Date {
+	return new Date(TODAY_NOON.getTime() - daysAgo * DAY_MS + (hour - 12) * HOUR_MS);
+}
+
+async function seedJob(
+	jobId: bigint,
+	createdAt: Date,
+	opts: {taskType?: string; requestedBy?: bigint | null; status?: JobStatus; bucketOnly?: boolean} = {},
+): Promise<void> {
+	const taskType = opts.taskType ?? 'A';
+	const requestedBy = opts.requestedBy ?? null;
+	await upsertOne(
+		JobsByDayBucket.insert({
+			bucket_day: createdAt.toISOString().slice(0, 10),
+			created_at: createdAt,
+			job_id: jobId,
+			task_type: taskType,
+			status: 'queued',
+			requested_by_user_id: requestedBy,
+		}),
+	);
+	if (opts.bucketOnly) return;
+	const row: JobByIdRow = {
+		job_id: jobId,
+		task_type: taskType,
+		status: opts.status ?? 'queued',
+		progress_current: null,
+		progress_total: null,
+		progress_message: null,
+		payload: '{}',
+		result: null,
+		error_message: null,
+		created_at: createdAt,
+		started_at: null,
+		completed_at: null,
+		requested_by_user_id: requestedBy,
+		audit_log_reason: null,
+		jet_stream_seq: null,
+		jet_stream_lane: 'batch',
+		attempts: 0,
+		max_attempts: 5,
+		run_at: null,
+		cancel_requested: false,
+		context_link: null,
+	};
+	await upsertOne(JobsById.insert(row));
+}
+
+async function listPages(opts: {
+	limit: number;
+	filters?: ListJobsFilters;
+	maxLookbackDays?: number;
+	cursor?: ListJobsCursor | null;
+}): Promise<Array<ListedPage>> {
+	const repository = new JobLedgerRepository();
+	const pages: Array<ListedPage> = [];
+	let cursor = opts.cursor ?? null;
+	for (let page = 0; page < 50; page += 1) {
+		const result = await repository.listJobs({
+			limit: opts.limit,
+			cursor,
+			filters: opts.filters ?? {},
+			maxLookbackDays: opts.maxLookbackDays ?? 14,
+		});
+		pages.push({ids: result.jobs.map((job) => job.job_id), cursor: result.nextCursor});
+		if (result.nextCursor === null) return pages;
+		cursor = {
+			bucketDay: result.nextCursor.bucketDay,
+			createdAt: new Date(result.nextCursor.createdAt.toISOString()),
+			jobId: BigInt(result.nextCursor.jobId.toString()),
+		};
+	}
+	throw new Error('listJobs never stopped paging');
+}
+
+function expectPages(pages: Array<ListedPage>, limit: number, expected: Array<Array<bigint>>): void {
+	expect(pages.map((page) => page.ids)).toEqual(expected);
+	const ids = pages.flatMap((page) => page.ids);
+	expect(new Set(ids).size).toBe(ids.length);
+	for (const page of pages.slice(0, -1)) {
+		expect(page.ids).toHaveLength(limit);
+		expect(page.cursor?.jobId).toBe(page.ids.at(-1));
+	}
+	expect(pages.at(-1)?.cursor).toBeNull();
+}
+
+export function describeListJobsPaging(): void {
+	describe('listJobs paging', () => {
+		beforeEach(() => {
+			vi.useFakeTimers({toFake: ['Date']});
+			vi.setSystemTime(TODAY_NOON);
+		});
+
+		afterEach(() => {
+			vi.useRealTimers();
+		});
+
+		it('walks one day newest first through a tie group larger than the page', async () => {
+			await seedJob(20n, at(0, 11));
+			for (const jobId of [11n, 12n, 13n, 14n, 15n]) {
+				await seedJob(jobId, at(0, 10));
+			}
+			await seedJob(9n, at(0, 9));
+			await seedJob(5n, at(0, 8));
+			await seedJob(6n, at(0, 8));
+
+			const pages = await listPages({limit: 2});
+
+			expectPages(pages, 2, [[20n, 15n], [14n, 13n], [12n, 11n], [9n, 6n], [5n]]);
+			expect(pages[0]?.cursor).toEqual({
+				bucketDay: at(0, 10).toISOString().slice(0, 10),
+				createdAt: at(0, 10),
+				jobId: 15n,
+			});
+		});
+
+		it('crosses days and keeps the lookback window anchored on today', async () => {
+			await seedJob(41n, at(0, 11));
+			await seedJob(40n, at(0, 10));
+			await seedJob(32n, at(1, 11));
+			await seedJob(31n, at(1, 10));
+			await seedJob(30n, at(1, 9));
+			await seedJob(21n, at(2, 11));
+			await seedJob(20n, at(2, 10));
+			await seedJob(10n, at(3, 11));
+
+			expectPages(await listPages({limit: 2, maxLookbackDays: 2}), 2, [[41n, 40n], [32n, 31n], [30n, 21n], [20n]]);
+			expectPages(await listPages({limit: 2, maxLookbackDays: 3}), 2, [
+				[41n, 40n],
+				[32n, 31n],
+				[30n, 21n],
+				[20n, 10n],
+				[],
+			]);
+			const outside = await new JobLedgerRepository().listJobs({
+				limit: 2,
+				cursor: {bucketDay: at(3, 12).toISOString().slice(0, 10), createdAt: at(3, 12), jobId: 1n},
+				filters: {},
+				maxLookbackDays: 2,
+			});
+			expect(outside).toEqual({jobs: [], nextCursor: null});
+		});
+
+		it('starts a cursor dated in the future at today', async () => {
+			await seedJob(2n, at(0, 2));
+			await seedJob(1n, at(1, 2));
+			const farFuture = new Date('9999-12-31T00:00:00.000Z');
+
+			expectPages(await listPages({limit: 5, cursor: {bucketDay: '9999-12-31', createdAt: farFuture, jobId: 1n}}), 5, [
+				[2n, 1n],
+			]);
+		}, 2_000);
+
+		it('never lists a day past the 90-day retention', async () => {
+			await seedJob(890n, at(89, 11));
+			await seedJob(910n, at(91, 11));
+
+			expectPages(await listPages({limit: 10, maxLookbackDays: 120}), 10, [[890n]]);
+		});
+
+		it('fills a page past rows whose job record is missing instead of leaving the day', async () => {
+			for (let hour = 1; hour <= 8; hour += 1) {
+				await seedJob(BigInt(hour), at(0, hour), {bucketOnly: [3, 6, 7].includes(hour)});
+			}
+
+			expectPages(await listPages({limit: 3}), 3, [
+				[8n, 5n, 4n],
+				[2n, 1n],
+			]);
+		});
+
+		it('fills pages through task type, requester and status filters across days', async () => {
+			await seedJob(60n, at(0, 11));
+			await seedJob(59n, at(0, 10.5), {taskType: 'B'});
+			await seedJob(58n, at(0, 10));
+			await seedJob(57n, at(0, 9.5), {taskType: 'B', requestedBy: ADMIN_USER_ID});
+			await seedJob(56n, at(0, 9), {status: 'succeeded'});
+			await seedJob(55n, at(0, 8.5), {taskType: 'B'});
+			await seedJob(50n, at(1, 11));
+			await seedJob(49n, at(1, 10.5), {taskType: 'B', requestedBy: ADMIN_USER_ID});
+			await seedJob(48n, at(1, 10), {bucketOnly: true});
+			await seedJob(47n, at(1, 9.5));
+
+			expectPages(await listPages({limit: 2, filters: {taskType: 'A'}}), 2, [[60n, 58n], [56n, 50n], [47n]]);
+			expectPages(await listPages({limit: 1, filters: {status: 'succeeded'}}), 1, [[56n], []]);
+			expectPages(await listPages({limit: 5, filters: {requestedByUserId: ADMIN_USER_ID}}), 5, [[57n, 49n]]);
+		});
+
+		it('resumes after a cursor whose job was discarded between pages', async () => {
+			for (let hour = 1; hour <= 5; hour += 1) {
+				await seedJob(BigInt(hour), at(0, hour));
+			}
+			const repository = new JobLedgerRepository();
+			const first = await repository.listJobs({limit: 2, cursor: null, filters: {}, maxLookbackDays: 14});
+			expect(first.jobs.map((job) => job.job_id)).toEqual([5n, 4n]);
+
+			await repository.discardJob(4n, at(0, 4));
+
+			expectPages(await listPages({limit: 2, cursor: first.nextCursor}), 2, [[3n, 2n], [1n]]);
+		});
+
+		it('returns an empty page after a page that filled exactly', async () => {
+			await seedJob(1n, at(0, 1));
+			await seedJob(2n, at(0, 2));
+
+			expectPages(await listPages({limit: 2}), 2, [[2n, 1n], []]);
+		});
+	});
+}

fluxer_api/src/api/jobs/PostgresJobLedgerExpiry.test.ts

@@ -0,0 +1,600 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {BatchBuilder, setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
+import {Db} from '@app/api/database/CassandraTypes';
+import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';
+import type {JobActiveRow, JobByDayBucketRow, JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
+import {
+	EXPIRED_JOB_ERROR,
+	JOB_LEDGER_TTL_SECONDS,
+	JOB_STALE_AFTER_MS,
+	JobLedgerRepository,
+} from '@app/api/jobs/JobLedgerRepository';
+import {describeListJobsPaging} from '@app/api/jobs/ListJobsPagingSuite';
+import {expireLegacyJobLedgerRows} from '@app/api/jobs/PostgresJobLedgerExpiry';
+import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
+import {startDockerContainer} from '@app/api/test/DockerTestContainer';
+import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
+import {createSnowflake} from '@fluxer/snowflake/src/Snowflake';
+import {
+	getDefaultPostgresClient,
+	type IPostgresClient,
+	initPostgres,
+	shutdownPostgres,
+} from '@pkgs/postgres/src/Client';
+import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
+
+const KV_TABLE = 'kv_job_ledger_expiry';
+const CONTAINER = `fluxer-kvjobs-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+const DAY_MS = 86_400_000;
+const ADMIN_USER_ID = 1_234_567_890_123n;
+
+async function sleep(ms: number): Promise<void> {
+	await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function freePort(): Promise<number> {
+	return new Promise((resolve, reject) => {
+		const server = createServer();
+		server.on('error', reject);
+		server.listen(0, '127.0.0.1', () => {
+			const address = server.address();
+			if (typeof address === 'string' || address === null) {
+				reject(new Error('no port'));
+				return;
+			}
+			const port = address.port;
+			server.close(() => resolve(port));
+		});
+	});
+}
+
+let sequence = 0;
+
+function jobIdAgedDays(days: number): bigint {
+	sequence += 1;
+	return createSnowflake({timestamp: Date.now() - days * DAY_MS, sequence: sequence % 4096, workerId: 1});
+}
+
+interface LegacyJob {
+	jobId: bigint;
+	createdAt: Date;
+}
+
+async function seedLegacyJob(
+	executor: PostgresKvQueryExecutor,
+	opts: {ageDays: number; status: JobStatus; requestedBy: bigint | null; active: boolean},
+): Promise<LegacyJob> {
+	const jobId = jobIdAgedDays(opts.ageDays);
+	const createdAt = new Date(Date.now() - opts.ageDays * DAY_MS);
+	const idRow: JobByIdRow = {
+		job_id: jobId,
+		task_type: opts.requestedBy === null ? 'flushUserActivityBuffer' : 'refreshSearchIndex',
+		status: opts.status,
+		progress_current: null,
+		progress_total: null,
+		progress_message: null,
+		payload: '{}',
+		result: null,
+		error_message: null,
+		created_at: createdAt,
+		started_at: null,
+		completed_at: opts.status === 'succeeded' ? createdAt : null,
+		requested_by_user_id: opts.requestedBy,
+		audit_log_reason: null,
+		jet_stream_seq: '1',
+		jet_stream_lane: 'batch',
+		attempts: 0,
+		max_attempts: 5,
+		run_at: null,
+		cancel_requested: false,
+		context_link: null,
+	};
+	const bucketRow: JobByDayBucketRow = {
+		bucket_day: createdAt.toISOString().slice(0, 10),
+		created_at: createdAt,
+		job_id: jobId,
+		task_type: idRow.task_type,
+		status: 'queued',
+		requested_by_user_id: opts.requestedBy,
+	};
+	await executor.executeQuery(JobsById.insert(idRow));
+	await executor.executeQuery(JobsByDayBucket.insert(bucketRow));
+	if (opts.active) {
+		const activeRow: JobActiveRow = {
+			job_id: jobId,
+			task_type: idRow.task_type,
+			status: opts.status,
+			requested_by_user_id: opts.requestedBy,
+			created_at: createdAt,
+			started_at: null,
+		};
+		await executor.executeQuery(JobsActive.insert(activeRow));
+	}
+	return {jobId, createdAt};
+}
+
+describe.skipIf(!dockerAvailable)('job ledger expiry against postgres', () => {
+	let raw: IPostgresClient;
+	let executor: PostgresKvQueryExecutor;
+
+	async function jobRows(
+		jobId: bigint,
+	): Promise<Array<{table_name: string; expires_at: Date | null; row_data: never}>> {
+		const result = await raw.query<{table_name: string; expires_at: Date | null; row_data: never}>(
+			`SELECT table_name, expires_at, row_data FROM ${KV_TABLE}
+WHERE table_name IN ('jobs_by_id', 'jobs_active', 'jobs_by_day_bucket')
+	AND (row_key = $1 OR split_part(row_key, chr(31), 3) = $1)
+ORDER BY table_name`,
+			[JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()})],
+		);
+		return result.rows;
+	}
+
+	async function waitForLockWait(): Promise<void> {
+		for (let attempt = 0; attempt < 400; attempt += 1) {
+			const waiting = await raw.query<{n: number}>(
+				`SELECT count(*)::int AS n FROM pg_stat_activity WHERE datname = current_database() AND wait_event_type = 'Lock'`,
+			);
+			if (waiting.rows[0]!.n > 0) return;
+			await sleep(25);
+		}
+		throw new Error('the pass never waited on the writer');
+	}
+
+	async function forgetLedgerExpiry(): Promise<void> {
+		await raw.query(
+			`UPDATE ${KV_TABLE} SET expires_at = NULL WHERE table_name IN ('jobs_by_id', 'jobs_by_day_bucket')`,
+		);
+	}
+
+	async function forgetExpiryOf(...jobIds: Array<bigint>): Promise<void> {
+		const keys = jobIds.map((jobId) => JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()}));
+		await raw.query(
+			`UPDATE ${KV_TABLE} SET expires_at = NULL
+WHERE table_name IN ('jobs_by_id', 'jobs_by_day_bucket')
+	AND (row_key = ANY($1::text[]) OR split_part(row_key, chr(31), 3) = ANY($1::text[]))`,
+			[keys],
+		);
+	}
+
+	beforeAll(async () => {
+		const port = await freePort();
+		startDockerContainer([
+			'run',
+			'-d',
+			'--name',
+			CONTAINER,
+			'-e',
+			'POSTGRES_USER=fluxer',
+			'-e',
+			'POSTGRES_PASSWORD=fluxer',
+			'-e',
+			'POSTGRES_DB=fluxer',
+			'-p',
+			`127.0.0.1:${port}:5432`,
+			'postgres:16-alpine',
+			'-c',
+			'fsync=off',
+		]);
+		let ready = false;
+		for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+			await sleep(500);
+			const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
+				stdio: 'ignore',
+			});
+			if (probe.status !== 0) continue;
+			try {
+				await initPostgres({
+					url: `postgres://fluxer:[email protected]:${port}/fluxer`,
+					maxConnections: 4,
+					kvTable: KV_TABLE,
+				});
+				await getDefaultPostgresClient().query('SELECT 1');
+				ready = true;
+			} catch {
+				await shutdownPostgres().catch(() => {});
+			}
+		}
+		if (!ready) throw new Error('postgres never came up');
+		raw = getDefaultPostgresClient();
+		await ensurePostgresKvSchema(raw);
+		executor = new PostgresKvQueryExecutor(raw);
+	}, 900_000);
+
+	beforeEach(async () => {
+		await raw.query(`DELETE FROM ${KV_TABLE}`);
+		setCassandraQueryExecutorForTesting(executor);
+	});
+
+	afterAll(async () => {
+		setCassandraQueryExecutorForTesting(new InMemoryCassandraQueryExecutor());
+		await shutdownPostgres().catch(() => {});
+		spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+	});
+
+	it('writes every ledger row with an expiry and keeps it through the job lifecycle', async () => {
+		const repository = new JobLedgerRepository();
+		const jobId = jobIdAgedDays(0);
+		await repository.createJob({
+			jobId,
+			taskType: 'refreshSearchIndex',
+			payload: {},
+			requestedByUserId: ADMIN_USER_ID,
+			auditLogReason: null,
+			maxAttempts: 5,
+			runAt: null,
+			jetStreamLane: 'batch',
+			jetStreamSeq: null,
+		});
+		await repository.setJetStreamSeq(jobId, '7');
+		await repository.markRunning(jobId, 'batch');
+		await repository.reportProgress(jobId, 1, 2, 'half');
+
+		const running = await jobRows(jobId);
+		expect(running.map((row) => row.table_name)).toEqual(['jobs_active', 'jobs_by_day_bucket', 'jobs_by_id']);
+		for (const row of running) {
+			expect(row.expires_at).not.toBeNull();
+			const remainingSeconds = (row.expires_at!.getTime() - Date.now()) / 1000;
+			expect(remainingSeconds).toBeGreaterThan(JOB_LEDGER_TTL_SECONDS - 60);
+			expect(remainingSeconds).toBeLessThanOrEqual(JOB_LEDGER_TTL_SECONDS);
+		}
+
+		await repository.markSucceeded(jobId, {ok: true});
+		const done = await jobRows(jobId);
+		expect(done.map((row) => row.table_name)).toEqual(['jobs_by_day_bucket', 'jobs_by_id']);
+		expect(done.every((row) => row.expires_at !== null)).toBe(true);
+		expect((await repository.getJob(jobId))?.status).toBe('succeeded');
+	});
+
+	it('never leaves a row without an expiry when a patch lands on a job that is gone', async () => {
+		const repository = new JobLedgerRepository();
+		const patches: Array<(jobId: bigint) => Promise<void>> = [
+			(jobId) => repository.markRunning(jobId, 'batch'),
+			(jobId) => repository.markSucceeded(jobId, null),
+			(jobId) => repository.markCancelled(jobId),
+			(jobId) => repository.markDeadletter(jobId, 'boom'),
+			(jobId) => repository.reportProgress(jobId, 1, null, null),
+			(jobId) => repository.setContextLink(jobId, '/admin/jobs'),
+			(jobId) => repository.setJetStreamSeq(jobId, '1'),
+			(jobId) => repository.requestCancel(jobId),
+		];
+		for (const patch of patches) {
+			const jobId = jobIdAgedDays(0);
+			await patch(jobId);
+			const rows = await jobRows(jobId);
+			expect(rows.length).toBeGreaterThan(0);
+			expect(rows.every((row) => row.expires_at !== null)).toBe(true);
+			expect(await repository.getJob(jobId)).toBeNull();
+		}
+		expect(await repository.listActiveJobs()).toEqual([]);
+	});
+
+	it('discards every row of a job that never reached the stream', async () => {
+		const repository = new JobLedgerRepository();
+		const jobId = jobIdAgedDays(0);
+		const createdAt = await repository.createJob({
+			jobId,
+			taskType: 'batchGuildAuditLogMessageDeletes',
+			payload: {guildId: '1'},
+			requestedByUserId: null,
+			auditLogReason: null,
+			maxAttempts: 3,
+			runAt: new Date(Date.now() + 30_000),
+			jetStreamLane: 'batch',
+			jetStreamSeq: null,
+		});
+		expect(await jobRows(jobId)).toHaveLength(3);
+
+		await repository.discardJob(jobId, createdAt);
+
+		expect(await jobRows(jobId)).toEqual([]);
+	});
+
+	it('clears legacy rows by the same rules the expiry now enforces', async () => {
+		const repository = new JobLedgerRepository();
+		const cronDone = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'succeeded',
+			requestedBy: null,
+			active: false,
+		});
+		const cronStuck = await seedLegacyJob(executor, {ageDays: 20, status: 'queued', requestedBy: null, active: true});
+		const partialId = jobIdAgedDays(20);
+		await executor.executeQuery(
+			JobsById.patchByPk({job_id: partialId}, {status: Db.set('succeeded'), completed_at: Db.set(new Date())}),
+		);
+		const adminDone = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'succeeded',
+			requestedBy: ADMIN_USER_ID,
+			active: false,
+		});
+		const adminStuck = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'queued',
+			requestedBy: ADMIN_USER_ID,
+			active: true,
+		});
+		const adminRunning = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'running',
+			requestedBy: ADMIN_USER_ID,
+			active: true,
+		});
+		const adminAncient = await seedLegacyJob(executor, {
+			ageDays: 100,
+			status: 'succeeded',
+			requestedBy: ADMIN_USER_ID,
+			active: false,
+		});
+		const cronInFlight = await seedLegacyJob(executor, {
+			ageDays: 2,
+			status: 'queued',
+			requestedBy: null,
+			active: true,
+		});
+		await forgetLedgerExpiry();
+		const fresh = jobIdAgedDays(0);
+		await repository.createJob({
+			jobId: fresh,
+			taskType: 'syncUrlBlocklists',
+			payload: {},
+			requestedByUserId: null,
+			auditLogReason: null,
+			maxAttempts: 5,
+			runAt: null,
+			jetStreamLane: 'batch',
+			jetStreamSeq: null,
+		});
+		const freshBefore = await jobRows(fresh);
+		await raw.query(
+			`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data) VALUES ('users', 'u1', 'u1', '{}'::jsonb)`,
+		);
+
+		expect(await repository.getJob(partialId)).toBeNull();
+
+		const first = await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
+		expect(first).toEqual({deleted: 10, expiring: 6, complete: true});
+
+		expect(await jobRows(cronDone.jobId)).toEqual([]);
+		expect(await jobRows(cronStuck.jobId)).toEqual([]);
+		expect(await jobRows(partialId)).toEqual([]);
+		expect(await jobRows(adminAncient.jobId)).toEqual([]);
+
+		for (const kept of [adminDone, adminStuck, adminRunning]) {
+			const rows = await jobRows(kept.jobId);
+			expect(rows.map((row) => row.table_name)).toEqual(['jobs_by_day_bucket', 'jobs_by_id']);
+			for (const row of rows) {
+				const expected = kept.createdAt.getTime() + JOB_LEDGER_TTL_SECONDS * 1000;
+				expect(Math.abs(row.expires_at!.getTime() - expected)).toBeLessThan(2000);
+			}
+		}
+		expect((await repository.getJob(adminDone.jobId))?.status).toBe('succeeded');
+		for (const stuck of [adminStuck, adminRunning]) {
+			const expired = await repository.getJob(stuck.jobId);
+			expect(expired?.status).toBe('deadletter');
+			expect(expired?.error_message).toBe('Expired from the job queue');
+			expect(expired?.completed_at).toBeInstanceOf(Date);
+		}
+
+		const inFlight = await jobRows(cronInFlight.jobId);
+		expect(inFlight.map((row) => row.table_name)).toEqual(['jobs_active', 'jobs_by_day_bucket', 'jobs_by_id']);
+		expect(inFlight.every((row) => row.expires_at === null)).toBe(true);
+		expect((await repository.getJob(cronInFlight.jobId))?.status).toBe('queued');
+
+		expect(await jobRows(fresh)).toEqual(freshBefore);
+		const users = await raw.query(`SELECT expires_at FROM ${KV_TABLE} WHERE table_name = 'users'`);
+		expect(users.rows).toEqual([{expires_at: null}]);
+
+		const listed = await repository.listJobs({limit: 50, cursor: null, filters: {}, maxLookbackDays: 30});
+		expect(listed.jobs.map((job) => job.job_id).sort()).toEqual(
+			[adminDone.jobId, adminStuck.jobId, adminRunning.jobId, cronInFlight.jobId, fresh].sort(),
+		);
+
+		for (let pass = 0; pass < 2; pass += 1) {
+			expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
+				deleted: 0,
+				expiring: 0,
+				complete: true,
+			});
+		}
+	});
+
+	it('runs again a day after a clean pass and clears rows an older image wrote in between', async () => {
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		const rolledBack = await seedLegacyJob(executor, {ageDays: 30, status: 'queued', requestedBy: null, active: true});
+		await forgetLedgerExpiry();
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
+
+		await raw.query(
+			`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('applied_at', now() - interval '2 days') WHERE row_key = 'job_ledger_expiry_v1'`,
+		);
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 3,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await jobRows(rolledBack.jobId)).toEqual([]);
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
+	});
+
+	it('expires stale active jobs page by page without fighting the legacy pass', async () => {
+		const repository = new JobLedgerRepository();
+		const legacyFirst = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'queued',
+			requestedBy: ADMIN_USER_ID,
+			active: true,
+		});
+		await forgetExpiryOf(legacyFirst.jobId);
+		await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
+		const legacyFirstRows = await jobRows(legacyFirst.jobId);
+
+		const stale: Array<bigint> = [];
+		for (let index = 0; index < 4; index += 1) {
+			const jobId = jobIdAgedDays(9);
+			await repository.createJob({
+				jobId,
+				taskType: 'syncUrlBlocklists',
+				payload: {},
+				requestedByUserId: null,
+				auditLogReason: null,
+				maxAttempts: 5,
+				runAt: null,
+				jetStreamLane: 'batch',
+				jetStreamSeq: null,
+			});
+			stale.push(jobId);
+		}
+		await repository.markRunning(stale[0]!, 'batch');
+		const fresh = jobIdAgedDays(0);
+		await repository.createJob({
+			jobId: fresh,
+			taskType: 'syncUrlBlocklists',
+			payload: {},
+			requestedByUserId: null,
+			auditLogReason: null,
+			maxAttempts: 5,
+			runAt: null,
+			jetStreamLane: 'batch',
+			jetStreamSeq: null,
+		});
+		const legacyStale = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'running',
+			requestedBy: ADMIN_USER_ID,
+			active: true,
+		});
+		const legacyYoung = await seedLegacyJob(executor, {ageDays: 2, status: 'queued', requestedBy: null, active: true});
+		await forgetExpiryOf(legacyStale.jobId, legacyYoung.jobId);
+		const orphan = jobIdAgedDays(9);
+		await executor.executeQuery(
+			JobsActive.patchByPkWithTtl({job_id: orphan}, {status: Db.set('running')}, JOB_LEDGER_TTL_SECONDS),
+		);
+
+		const sweep = () =>
+			repository.expireStaleActiveJobs({staleBeforeMs: Date.now() - JOB_STALE_AFTER_MS, pageSize: 2, maxCleared: 100});
+		expect(await sweep()).toEqual({cleared: 6, expired: 5, complete: true});
+
+		const active = (await repository.listActiveJobs()).map((job) => job.job_id).sort();
+		expect(active).toEqual([fresh, legacyYoung.jobId].sort());
+		for (const jobId of [...stale, legacyStale.jobId]) {
+			const job = await repository.getJob(jobId);
+			expect(job?.status).toBe('deadletter');
+			expect(job?.error_message).toBe(EXPIRED_JOB_ERROR);
+			const byId = (await jobRows(jobId)).find((row) => row.table_name === 'jobs_by_id');
+			expect(Math.abs(byId!.expires_at!.getTime() - (Date.now() + JOB_LEDGER_TTL_SECONDS * 1000))).toBeLessThan(60_000);
+		}
+		expect(await jobRows(orphan)).toEqual([]);
+		expect(await jobRows(legacyFirst.jobId)).toEqual(legacyFirstRows);
+
+		const legacyStaleById = (await jobRows(legacyStale.jobId)).find((row) => row.table_name === 'jobs_by_id');
+		await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
+		expect((await jobRows(legacyStale.jobId)).find((row) => row.table_name === 'jobs_by_id')).toEqual(legacyStaleById);
+		expect((await repository.listActiveJobs()).map((job) => job.job_id)).toContain(legacyYoung.jobId);
+		expect(await sweep()).toEqual({cleared: 0, expired: 0, complete: true});
+	});
+
+	it('leaves rows alone when a live writer gives them an expiry while the pass waits on them', async () => {
+		const cronDone = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'succeeded',
+			requestedBy: null,
+			active: false,
+		});
+		const adminDone = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'succeeded',
+			requestedBy: ADMIN_USER_ID,
+			active: false,
+		});
+		await forgetLedgerExpiry();
+		const liveKeys = [cronDone.jobId, adminDone.jobId].map((jobId) =>
+			JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()}),
+		);
+		let written!: () => void;
+		const writerHoldsRows = new Promise<void>((resolve) => {
+			written = resolve;
+		});
+		let release!: () => void;
+		const released = new Promise<void>((resolve) => {
+			release = resolve;
+		});
+		const writer = raw.transaction(async (db) => {
+			await db.query(
+				`UPDATE ${KV_TABLE} SET expires_at = now() + interval '1 hour', updated_at = now() WHERE table_name = 'jobs_by_id' AND row_key = ANY($1::text[])`,
+				[liveKeys],
+			);
+			written();
+			await released;
+		});
+		await writerHoldsRows;
+		const pass = expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
+		await waitForLockWait();
+		release();
+		await writer;
+
+		expect(await pass).toEqual({deleted: 1, expiring: 1, complete: true});
+		for (const job of [cronDone, adminDone]) {
+			const byId = (await jobRows(job.jobId)).find((row) => row.table_name === 'jobs_by_id');
+			expect(byId).toBeDefined();
+			const remainingSeconds = (byId!.expires_at!.getTime() - Date.now()) / 1000;
+			expect(remainingSeconds).toBeGreaterThan(3000);
+			expect(remainingSeconds).toBeLessThanOrEqual(3660);
+		}
+		expect((await new JobLedgerRepository().getJob(adminDone.jobId))?.status).toBe('succeeded');
+	});
+
+	it('pages through more legacy rows than one page holds and stops at its deadline', async () => {
+		const batch = new BatchBuilder();
+		for (let index = 0; index < 2300; index += 1) {
+			const jobId = jobIdAgedDays(30);
+			const createdAt = new Date(Date.now() - 30 * DAY_MS);
+			batch.addPrepared(
+				JobsByDayBucket.insert({
+					bucket_day: createdAt.toISOString().slice(0, 10),
+					created_at: createdAt,
+					job_id: jobId,
+					task_type: 'flushUserActivityBuffer',
+					status: 'queued',
+					requested_by_user_id: null,
+				}),
+			);
+		}
+		await batch.executeChunked(500, false);
+		await forgetLedgerExpiry();
+
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() - 1)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: false,
+		});
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 2300,
+			expiring: 0,
+			complete: true,
+		});
+		const left = await raw.query(`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = 'jobs_by_day_bucket'`);
+		expect(left.rows[0]).toEqual({n: 0});
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
+	});
+	describeListJobsPaging();
+});

fluxer_api/src/api/jobs/PostgresJobLedgerExpiry.ts

@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {postgresKvPassIsFresh, recordPostgresKvCleanPass} from '@app/api/database/PostgresKvQueryExecutor';
+import {EXPIRED_JOB_ERROR, JOB_LEDGER_TTL_SECONDS, JOB_STALE_AFTER_MS} from '@app/api/jobs/JobLedgerRepository';
+import {FLUXER_EPOCH} from '@fluxer/constants/src/Core';
+import {TIMESTAMP_SHIFT} from '@fluxer/snowflake/src/Snowflake';
+import {type IPostgresClient, quoteIdentifier} from '@pkgs/postgres/src/Client';
+import {ms} from 'itty-time';
+
+const LEGACY_JOB_LEDGER_MARKER = 'job_ledger_expiry_v1';
+const PAGE_SIZE = 2000;
+const CLEAN_PASS_INTERVAL_MS = ms('1 day');
+const BIGINT_KEY_PREFIX = '{"__fluxer_type":"bigint","value":"';
+const BIGINT_KEY_SUFFIX = '"}';
+
+function bigintKeyExpr(key: string): string {
+	const prefixLength = BIGINT_KEY_PREFIX.length;
+	const affixLength = prefixLength + BIGINT_KEY_SUFFIX.length;
+	return `(CASE WHEN left(${key}, ${prefixLength}) = '${BIGINT_KEY_PREFIX}' AND right(${key}, ${BIGINT_KEY_SUFFIX.length}) = '${BIGINT_KEY_SUFFIX}' THEN substr(${key}, ${prefixLength + 1}, length(${key}) - ${affixLength})::numeric END)`;
+}
+
+interface LedgerTable {
+	name: string;
+	jobIdExpr: string;
+	deleteStale: boolean;
+	markStaleDeadletter: boolean;
+}
+
+const LEDGER_TABLES: ReadonlyArray<LedgerTable> = [
+	{
+		name: 'jobs_active',
+		jobIdExpr: bigintKeyExpr('kv.row_key'),
+		deleteStale: true,
+		markStaleDeadletter: false,
+	},
+	{
+		name: 'jobs_by_id',
+		jobIdExpr: bigintKeyExpr('kv.row_key'),
+		deleteStale: false,
+		markStaleDeadletter: true,
+	},
+	{
+		name: 'jobs_by_day_bucket',
+		jobIdExpr: bigintKeyExpr('split_part(kv.row_key, chr(31), 3)'),
+		deleteStale: false,
+		markStaleDeadletter: false,
+	},
+];
+
+export interface LegacyJobLedgerExpiryResult {
+	deleted: number;
+	expiring: number;
+	complete: boolean;
+}
+
+function pageSql(table: string, target: LedgerTable): string {
+	const staleRemovable = target.deleteStale ? 'c.created_at < $7' : '(c.created_at < $7 AND c.system_job)';
+	const removable = `c.job_id IS NULL OR c.created_at < $6 OR ${staleRemovable}`;
+	const rowData = target.markStaleDeadletter
+		? `CASE WHEN c.unfinished THEN kv.row_data || jsonb_build_object('status', 'deadletter', 'error_message', $9::text, 'completed_at', jsonb_build_object('__fluxer_type', 'date', 'value', $10::text)) ELSE kv.row_data END`
+		: 'kv.row_data';
+	return `
+WITH page AS (
+	SELECT kv.row_key, kv.row_data, ${target.jobIdExpr} AS job_id
+	FROM ${table} kv
+	WHERE kv.table_name = $1 AND kv.expires_at IS NULL AND kv.row_key > $2
+	ORDER BY kv.row_key
+	LIMIT $3
+), classified AS (
+	SELECT
+		page.row_key,
+		page.job_id,
+		to_timestamp(((div(page.job_id, $4::numeric) + $5::numeric) / 1000)::double precision) AS created_at,
+		COALESCE(page.row_data -> 'requested_by_user_id', 'null'::jsonb) = 'null'::jsonb AS system_job,
+		COALESCE(page.row_data ->> 'status' IN ('queued', 'running'), false) AS unfinished
+	FROM page
+), removed AS (
+	DELETE FROM ${table} kv
+	USING classified c
+	WHERE kv.table_name = $1 AND kv.row_key = c.row_key AND kv.expires_at IS NULL AND (${removable})
+	RETURNING kv.row_key
+), expiring AS (
+	UPDATE ${table} kv
+	SET expires_at = c.created_at + make_interval(secs => $8::double precision), updated_at = now(), row_data = ${rowData}
+	FROM classified c
+	WHERE kv.table_name = $1 AND kv.row_key = c.row_key AND kv.expires_at IS NULL AND c.created_at < $7 AND NOT (${removable})
+	RETURNING kv.row_key
+)
+SELECT
+	(SELECT max(row_key) FROM page) AS last_row_key,
+	(SELECT count(*) FROM page) AS scanned,
+	(SELECT count(*) FROM removed) AS deleted,
+	(SELECT count(*) FROM expiring) AS expiring`;
+}
+
+export async function expireLegacyJobLedgerRows(
+	client: IPostgresClient,
+	deadlineMs: number,
+): Promise<LegacyJobLedgerExpiryResult | null> {
+	const table = quoteIdentifier(client.kvTable());
+	if (await postgresKvPassIsFresh(client, LEGACY_JOB_LEDGER_MARKER, CLEAN_PASS_INTERVAL_MS)) {
+		return null;
+	}
+	const now = Date.now();
+	const retentionCutoff = new Date(now - JOB_LEDGER_TTL_SECONDS * 1000);
+	const staleCutoff = new Date(now - JOB_STALE_AFTER_MS);
+	const completedAt = new Date(now).toISOString();
+	let scanned = 0;
+	let deleted = 0;
+	let expiring = 0;
+	for (const target of LEDGER_TABLES) {
+		const sql = pageSql(table, target);
+		const deadletterValues = target.markStaleDeadletter ? [EXPIRED_JOB_ERROR, completedAt] : [];
+		let cursor = '';
+		for (;;) {
+			if (Date.now() >= deadlineMs) {
+				return {deleted, expiring, complete: false};
+			}
+			const result = await client.query<{
+				last_row_key: string | null;
+				scanned: string;
+				deleted: string;
+				expiring: string;
+			}>(sql, [
+				target.name,
+				cursor,
+				PAGE_SIZE,
+				(1n << TIMESTAMP_SHIFT).toString(),
+				FLUXER_EPOCH.toString(),
+				retentionCutoff,
+				staleCutoff,
+				JOB_LEDGER_TTL_SECONDS,
+				...deadletterValues,
+			]);
+			const page = result.rows[0];
+			if (!page || page.last_row_key === null) {
+				break;
+			}
+			scanned += Number(page.scanned);
+			deleted += Number(page.deleted);
+			expiring += Number(page.expiring);
+			cursor = page.last_row_key;
+		}
+	}
+	if (scanned === 0) {
+		await recordPostgresKvCleanPass(client, LEGACY_JOB_LEDGER_MARKER);
+	}
+	return {deleted, expiring, complete: true};
+}

fluxer_api/src/api/middleware/FileShaCache.ts

@@ -1,7 +1,8 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {AdminRepository} from '@app/api/admin/AdminRepository';
-import {BANNED_FILE_SHAS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
+import {Config} from '@app/api/Config';
+import {BANNED_FILE_SHAS_REFRESH_CHANNEL, isBlocklistFeedFileSha} from '@app/api/constants/ContentModeration';
 import {Logger} from '@app/api/Logger';
 import {RefreshSubscription} from '@app/api/utils/RefreshSubscription';
 import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
@@ -38,8 +39,11 @@ class FileShaCache {
 	async refresh(): Promise<void> {
 		const rows = await this.adminRepository.loadAllBannedFileShas();
 		const next = new Set<string>();
+		const includeFeedRows = Config.blocklistFeeds.enabled;
 		for (const row of rows) {
-			if (row.sha256_hex) next.add(row.sha256_hex.toLowerCase());
+			if (!row.sha256_hex) continue;
+			if (!includeFeedRows && isBlocklistFeedFileSha(row)) continue;
+			next.add(row.sha256_hex.toLowerCase());
 		}
 		this.banned = next;
 		this.consecutiveFailures = 0;

fluxer_api/src/api/middleware/UrlBlocklistCache.ts

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {AdminRepository} from '@app/api/admin/AdminRepository';
+import {Config} from '@app/api/Config';
 import {BANNED_URL_DOMAINS_REFRESH_CHANNEL, BANNED_URLS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
 import type {IStorageService} from '@app/api/infrastructure/IStorageService';
 import {Logger} from '@app/api/Logger';
@@ -68,7 +69,7 @@ class UrlBlocklistCache {
 	}
 
 	private async loadFeedUrls(): Promise<Set<string>> {
-		if (!this.storageService) return new Set();
+		if (!this.storageService || !Config.blocklistFeeds.enabled) return new Set();
 		const lines = await readLinesFromS3(this.storageService, RISK_S3_KEYS.feedUrls);
 		return new Set(lines);
 	}

fluxer_api/src/api/risk/RiskBlocklistS3.ts

@@ -47,3 +47,7 @@ export async function readLinesFromS3(storage: IStorageService, key: string): Pr
 		return [];
 	}
 }
+
+export async function deleteRiskS3Object(storage: IStorageService, key: string): Promise<void> {
+	await storage.deleteObject(RISK_S3_BUCKET, key);
+}

fluxer_api/src/api/risk/__tests__/PostgresIpInfoKv.test.ts

@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {IpInfoRequestAuditEvent} from '@pkgs/geoip/src/IpInfoService';
+import {
+	createPostgresIpInfoCache,
+	createPostgresIpInfoRequestAuditLogger,
+	IPINFO_CACHE_TTL_SECONDS,
+	IPINFO_REQUEST_AUDIT_TTL_SECONDS,
+} from '@pkgs/geoip/src/PostgresIpInfoKv';
+import type {IPostgresClient} from '@pkgs/postgres/src/Client';
+import {describe, expect, it} from 'vitest';
+
+function recordingClient(writes: Array<Array<unknown>>): IPostgresClient {
+	return {
+		async query(_text: string, values?: Array<unknown>) {
+			writes.push(values ?? []);
+			return {rows: [], rowCount: 1};
+		},
+		kvTable() {
+			return 'kv';
+		},
+	} as never;
+}
+
+function expectExpiresIn(values: Array<unknown> | undefined, ttlSeconds: number): void {
+	const expiresAt = values?.[4];
+	expect(expiresAt).toBeInstanceOf(Date);
+	const remainingSeconds = ((expiresAt as Date).getTime() - Date.now()) / 1000;
+	expect(remainingSeconds).toBeGreaterThan(ttlSeconds - 10);
+	expect(remainingSeconds).toBeLessThanOrEqual(ttlSeconds);
+}
+
+const EVENT: IpInfoRequestAuditEvent = {
+	requestedAt: new Date('2026-09-21T12:00:00.000Z'),
+	ip: '192.0.2.1',
+	cacheKey: 'ip:192.0.2.1',
+	source: 'test',
+	reason: null,
+	outcome: 'http_success',
+	httpStatus: 200,
+	available: true,
+	riskNote: 'none',
+	latencyMs: 12,
+	requestUrl: 'https://ipinfo.test/192.0.2.1',
+	responseIp: '192.0.2.1',
+	countryCode: 'SE',
+	asnNumber: 64500,
+	isAnonymous: false,
+	isTor: false,
+	isVpn: false,
+	isProxy: false,
+	isResidentialProxy: false,
+};
+
+describe('Postgres ipinfo KV expiry', () => {
+	it('expires request audit rows after 90 days', async () => {
+		const writes: Array<Array<unknown>> = [];
+		await createPostgresIpInfoRequestAuditLogger({client: recordingClient(writes)}).record(EVENT);
+		expect(writes).toHaveLength(1);
+		expect(writes[0]?.[0]).toBe('ipinfo_requests_by_hour');
+		expectExpiresIn(writes[0], IPINFO_REQUEST_AUDIT_TTL_SECONDS);
+	});
+
+	it('falls back to the 14-day cache default', async () => {
+		const writes: Array<Array<unknown>> = [];
+		const cache = createPostgresIpInfoCache({client: recordingClient(writes)});
+		await cache.set('fallback', {ok: true});
+		await cache.set('zero', {ok: true}, 0);
+		await cache.set('short', {ok: true}, 60);
+		expect(writes.map((values) => values[0])).toEqual(['ipinfo_cache', 'ipinfo_cache', 'ipinfo_cache']);
+		expectExpiresIn(writes[0], IPINFO_CACHE_TTL_SECONDS);
+		expectExpiresIn(writes[1], IPINFO_CACHE_TTL_SECONDS);
+		expectExpiresIn(writes[2], 60);
+	});
+});

fluxer_api/src/api/test/InMemoryCassandraQueryExecutor.ts

@@ -352,7 +352,16 @@ export class InMemoryCassandraQueryExecutor implements CassandraQueryExecutorFor
 		let rows = [...this.table(meta).values()].filter((row) => matchesWhere(row, meta.where, params));
 		if (meta.orderBy) {
 			const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
-			rows = rows.sort((a, b) => compareValues(a[meta.orderBy!.col], b[meta.orderBy!.col]) * direction);
+			const column = meta.orderBy.col as string;
+			const primaryKey = meta.table.primaryKey as ReadonlyArray<string>;
+			const columns = [column, ...primaryKey.slice(primaryKey.indexOf(column) + 1)];
+			rows = rows.sort((a, b) => {
+				for (const c of columns) {
+					const cmp = compareValues(a[c], b[c]);
+					if (cmp !== 0) return cmp * direction;
+				}
+				return 0;
+			});
 		}
 		if (typeof meta.limit === 'number') {
 			rows = rows.slice(0, meta.limit);

fluxer_api/src/api/worker/BlocklistFeedStartup.ts

@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {Logger} from '@app/api/Logger';
+import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
+import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError';
+import type {WorkerService} from '@app/api/worker/WorkerService';
+import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
+import {ms} from 'itty-time';
+
+const INITIAL_SYNC_KEY = 'sync:email_domains:initialized';
+const PURGE_KEY = 'sync:blocklist_feeds:purged';
+const CLAIM_TTL_SECONDS = ms('6 hours') / 1000;
+const FEED_TASKS = [
+	'syncDisposableEmailDomains',
+	'syncUrlBlocklists',
+	'syncFileShaBlocklists',
+] as const satisfies ReadonlyArray<WorkerTaskName>;
+
+export async function queueBlocklistFeedStartupJobs(
+	kvClient: Pick<IKVProvider, 'setnx' | 'del'>,
+	workerService: Pick<WorkerService, 'addJob'>,
+	enabled: boolean,
+): Promise<void> {
+	if (enabled) {
+		if (await kvClient.setnx(INITIAL_SYNC_KEY, '1', CLAIM_TTL_SECONDS)) {
+			Logger.info('Triggering initial disposable email domain sync');
+			await queueJobs(workerService, ['syncDisposableEmailDomains']);
+		}
+		return;
+	}
+	const wasEnabled = (await kvClient.del(INITIAL_SYNC_KEY)) > 0;
+	const claimed = await kvClient.setnx(PURGE_KEY, '1', CLAIM_TTL_SECONDS);
+	if (!wasEnabled && !claimed) return;
+	Logger.info('Removing blocklist feed data, blocklist feeds are disabled');
+	await queueJobs(workerService, FEED_TASKS);
+}
+
+async function queueJobs(
+	workerService: Pick<WorkerService, 'addJob'>,
+	tasks: ReadonlyArray<WorkerTaskName>,
+): Promise<void> {
+	for (const task of tasks) {
+		try {
+			await workerService.addJob(task, {});
+		} catch (error) {
+			if (!(error instanceof WorkerQueueOverflowError)) {
+				throw error;
+			}
+			Logger.warn({task}, 'Dropped blocklist feed job, jobs stream is at its limit');
+		}
+	}
+}

fluxer_api/src/api/worker/JetStreamWorkerQueue.ts

@@ -17,13 +17,13 @@ import {
 	StorageType,
 	type StreamConfig,
 } from '@nats-io/jetstream';
-import {nanos} from '@nats-io/transport-node';
+import {millis, nanos} from '@nats-io/transport-node';
 import type {JetStreamConnectionManager} from '@pkgs/nats/src/JetStreamConnectionManager';
 import type {WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes';
 
 const STREAM_NAME = 'JOBS';
 const SUBJECT_PREFIX = 'jobs.';
-const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+export const JOBS_STREAM_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
 const LEGACY_CONSUMER_NAME = 'workers';
 const DLQ_STREAM_NAME = 'JOBS_DLQ';
 const DLQ_SUBJECT_PREFIX = 'dlq.';
@@ -59,7 +59,7 @@ const JOBS_STREAM: WorkerStreamDefinition = {
 	name: STREAM_NAME,
 	subject: `${SUBJECT_PREFIX}>`,
 	retention: RetentionPolicy.Workqueue,
-	maxAgeMs: MAX_AGE_MS,
+	maxAgeMs: JOBS_STREAM_MAX_AGE_MS,
 	minBytes: STREAM_MIN_BYTES,
 	maxMessages: STREAM_MAX_MSGS,
 	maxMessagesPerSubject: STREAM_MAX_MSGS_PER_SUBJECT,
@@ -106,6 +106,7 @@ export class JetStreamWorkerQueue {
 	private consumersReady = false;
 	private streamSetup: Promise<void> | null = null;
 	private dlqStreamSetup: Promise<void> | null = null;
+	private jobsStreamMaxAgeMs = JOBS_STREAM_MAX_AGE_MS;
 
 	constructor(connectionManager: JetStreamConnectionManager) {
 		this.connectionManager = connectionManager;
@@ -125,11 +126,16 @@ export class JetStreamWorkerQueue {
 		if (existingConfig === null) {
 			await this.addStream(jsm);
 		} else {
+			this.jobsStreamMaxAgeMs = millis(existingConfig.max_age);
 			await this.applyStreamLimits(jsm, existingConfig);
 		}
 		this.streamReady = true;
 	}
 
+	getJobsStreamMaxAgeMs(): number {
+		return this.jobsStreamMaxAgeMs;
+	}
+
 	private async oversizedSubjects(jsm: JetStreamManager): Promise<Array<[string, number]> | null> {
 		try {
 			const info = await jsm.streams.info(STREAM_NAME, {subjects_filter: JOBS_STREAM.subject});
@@ -462,7 +468,7 @@ export class JetStreamWorkerQueue {
 			priority?: number;
 			jobKey?: string;
 		},
-	): Promise<string> {
+	): Promise<{seq: string; duplicate: boolean}> {
 		const js = this.connectionManager.getJetStreamClient();
 		const subject = `${SUBJECT_PREFIX}${taskType}`;
 		const body = JSON.stringify({
@@ -477,8 +483,7 @@ export class JetStreamWorkerQueue {
 			const ack = await js.publish(subject, body, {
 				msgID,
 			});
-			const jobId = `${ack.seq}`;
-			return jobId;
+			return {seq: `${ack.seq}`, duplicate: ack.duplicate === true};
 		} catch (error) {
 			const rejection = describeStreamRejection(error);
 			if (rejection === null) {

fluxer_api/src/api/worker/WorkerLaneConfig.ts

@@ -67,6 +67,7 @@ const LANE_CONFIG = {
 		consumerName: 'workers_batch',
 		tasks: [
 			'expireAttachments',
+			'expireStaleJobs',
 			'indexChannelMessages',
 			'indexGuildMembers',
 			'processAssetDeletionQueue',

fluxer_api/src/api/worker/WorkerMain.ts

@@ -23,8 +23,9 @@ import {
 } from '@app/api/middleware/ServiceSingletons';
 import {initializeSearch, shutdownSearch} from '@app/api/SearchFactory';
 import {awaitAll} from '@app/api/utils/ConcurrencyUtils';
+import {queueBlocklistFeedStartupJobs} from '@app/api/worker/BlocklistFeedStartup';
 import {CronScheduler} from '@app/api/worker/CronScheduler';
-import {JetStreamWorkerQueue} from '@app/api/worker/JetStreamWorkerQueue';
+import {JetStreamWorkerQueue, JOBS_STREAM_MAX_AGE_MS} from '@app/api/worker/JetStreamWorkerQueue';
 import {clearWorkerDependencies, setWorkerDependencies} from '@app/api/worker/WorkerContext';
 import {initializeWorkerDependencies, type WorkerDependencies} from '@app/api/worker/WorkerDependencies';
 import {WorkerHeartbeat} from '@app/api/worker/WorkerHeartbeat';
@@ -34,7 +35,6 @@ import {
 	validateLaneCompleteness,
 } from '@app/api/worker/WorkerLaneConfig';
 import {createWorkerProcessErrorHandler} from '@app/api/worker/WorkerProcessErrorHandler';
-import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError';
 import {WorkerRunner} from '@app/api/worker/WorkerRunner';
 import {WorkerService} from '@app/api/worker/WorkerService';
 import {workerTasks} from '@app/api/worker/WorkerTaskRegistry';
@@ -43,9 +43,8 @@ import {BACKGROUND_READ_TIMEOUT_MS, initCassandra, shutdownCassandra} from '@pkg
 import {JetStreamConnectionManager} from '@pkgs/nats/src/JetStreamConnectionManager';
 import {getDefaultPostgresClient, initPostgres, shutdownPostgres} from '@pkgs/postgres/src/Client';
 import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
-import {ms} from 'itty-time';
 
-function registerCronJobs(cron: CronScheduler): void {
+function registerCronJobs(cron: CronScheduler, jobsStreamMaxAgeMs: number): void {
 	cron.upsert('processAssetDeletionQueue', 'processAssetDeletionQueue', {}, '0 */5 * * * *', {ledger: false});
 	if (Config.cachePurge.adapter !== 'none') {
 		cron.upsert('processCachePurgeQueue', 'processCachePurgeQueue', {}, '*/10 * * * * *', {ledger: false});
@@ -62,6 +61,14 @@ function registerCronJobs(cron: CronScheduler): void {
 	}
 	cron.upsert('processInactivityDeletions', 'processInactivityDeletions', {}, '0 0 */6 * * *', {ledger: false});
 	cron.upsert('expireAttachments', 'expireAttachments', {}, '0 0 */12 * * *', {ledger: false});
+	if (jobsStreamMaxAgeMs > 0 && jobsStreamMaxAgeMs <= JOBS_STREAM_MAX_AGE_MS) {
+		cron.upsert('expireStaleJobs', 'expireStaleJobs', {}, '0 45 3 * * *', {ledger: false});
+	} else {
+		Logger.warn(
+			{jobsStreamMaxAgeMs},
+			'Jobs stream keeps jobs past 7 days, stale jobs stay active until their ledger rows expire',
+		);
+	}
 	cron.upsert('prunePostgresKvTtl', 'prunePostgresKvTtl', {}, '0 */5 * * * *', {ledger: false});
 	cron.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *', {ledger: false});
 	if (Config.blocklistFeeds.enabled) {
@@ -240,26 +247,9 @@ export async function startWorkerMain(): Promise<void> {
 		}
 		dependencies = await initializeWorkerDependencies(snowflakeService);
 		setWorkerDependencies(dependencies);
-		if (Config.blocklistFeeds.enabled) {
-			const didClaimEmailSync = await dependencies.kvClient.setnx(
-				'sync:email_domains:initialized',
-				'1',
-				ms('6 hours') / 1000,
-			);
-			if (didClaimEmailSync) {
-				Logger.info('Triggering initial disposable email domain sync');
-				try {
-					await workerService.addJob('syncDisposableEmailDomains', {});
-				} catch (error) {
-					if (!(error instanceof WorkerQueueOverflowError)) {
-						throw error;
-					}
-					Logger.warn('Dropped initial disposable email domain sync, jobs stream is at its limit');
-				}
-			}
-		}
+		await queueBlocklistFeedStartupJobs(dependencies.kvClient, workerService, Config.blocklistFeeds.enabled);
 		cron = new CronScheduler(workerService, Logger, dependencies.kvClient, heartbeat);
-		registerCronJobs(cron);
+		registerCronJobs(cron, queue.getJobsStreamMaxAgeMs());
 		for (const lane of activeWorkerLanes) {
 			const laneTasks: Record<string, WorkerTaskHandler> = {};
 			for (const taskType of lane.taskTypes) {

fluxer_api/src/api/worker/WorkerService.ts

@@ -29,10 +29,10 @@ export class WorkerService implements IWorkerService<WorkerTaskName> {
 		const skipLedger = options?.skipLedger === true;
 		const requireLedger = options?.requireLedger === true;
 		const payloadRecord = payload as Record<string, unknown>;
-		let ledgerWritten = false;
+		let ledgerCreatedAt: Date | null = null;
 		if (!skipLedger) {
 			try {
-				await this.ledger.createJob({
+				ledgerCreatedAt = await this.ledger.createJob({
 					jobId,
 					taskType,
 					payload: payloadRecord,
@@ -43,29 +43,32 @@ export class WorkerService implements IWorkerService<WorkerTaskName> {
 					jetStreamLane: findLaneForTask(taskType),
 					jetStreamSeq: null,
 				});
-				ledgerWritten = true;
 			} catch (ledgerErr) {
 				Logger.error({err: ledgerErr, jobId: jobId.toString(), taskType}, 'Failed to write ledger row for job');
 				if (requireLedger) throw ledgerErr;
 			}
 		}
-		const enrichedPayload = ledgerWritten ? {...payloadRecord, __jobId: jobId.toString()} : payloadRecord;
+		const enrichedPayload = ledgerCreatedAt !== null ? {...payloadRecord, __jobId: jobId.toString()} : payloadRecord;
 		try {
-			const seq = await this.queue.enqueue(taskType, enrichedPayload, {
+			const {seq, duplicate} = await this.queue.enqueue(taskType, enrichedPayload, {
 				...(options?.runAt !== undefined && {runAt: options.runAt}),
 				...(options?.maxAttempts !== undefined && {maxAttempts: options.maxAttempts}),
 				...(options?.priority !== undefined && {priority: options.priority}),
 				...(options?.jobKey !== undefined && {jobKey: options.jobKey}),
 			});
-			if (ledgerWritten) {
+			if (ledgerCreatedAt !== null && duplicate) {
+				await this.ledger
+					.discardJob(jobId, ledgerCreatedAt)
+					.catch((err) => Logger.warn({err, jobId: jobId.toString()}, 'Ledger discardJob failed'));
+			} else if (ledgerCreatedAt !== null) {
 				await this.ledger
 					.setJetStreamSeq(jobId, seq)
 					.catch((err) => Logger.warn({err, jobId: jobId.toString()}, 'Ledger setJetStreamSeq failed'));
 			}
-			Logger.debug({taskType, jobId: jobId.toString(), seq}, 'Job queued successfully');
+			Logger.debug({taskType, jobId: jobId.toString(), seq, duplicate}, 'Job queued successfully');
 			return jobId;
 		} catch (error) {
-			if (ledgerWritten) {
+			if (ledgerCreatedAt !== null) {
 				await this.ledger
 					.markDeadletter(jobId, error instanceof Error ? error.message : String(error))
 					.catch((err) => Logger.warn({err, jobId: jobId.toString()}, 'Ledger markDeadletter failed'));

fluxer_api/src/api/worker/WorkerTaskRegistry.ts

@@ -14,6 +14,7 @@ import bulkDeleteUserMessages from '@app/api/worker/tasks/BulkDeleteUserMessages
 import bulkDeleteUserMessagesScoped from '@app/api/worker/tasks/BulkDeleteUserMessagesScoped';
 import deleteUserMessagesInGuildByTime from '@app/api/worker/tasks/DeleteUserMessagesInGuildByTime';
 import expireAttachments from '@app/api/worker/tasks/ExpireAttachments';
+import expireStaleJobs from '@app/api/worker/tasks/ExpireStaleJobs';
 import extractEmbeds from '@app/api/worker/tasks/ExtractEmbeds';
 import finalizeNcmecAttachmentReport from '@app/api/worker/tasks/FinalizeNcmecAttachmentReport';
 import flushUserActivityBuffer from '@app/api/worker/tasks/FlushUserActivityBuffer';
@@ -59,6 +60,7 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
 	bulkUpdateUserFlags: bulkUpdateUserFlags,
 	deleteUserMessagesInGuildByTime,
 	expireAttachments,
+	expireStaleJobs,
 	extractEmbeds,
 	finalizeNcmecAttachmentReport,
 	handleMentions,

fluxer_api/src/api/worker/tasks/ExpireStaleJobs.ts

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {JOB_STALE_AFTER_MS, JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
+import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+
+const PAGE_SIZE = 500;
+const MAX_CLEARED_PER_RUN = 1000;
+
+const expireStaleJobs: WorkerTaskHandler = async (_payload, helpers) => {
+	const result = await new JobLedgerRepository().expireStaleActiveJobs({
+		staleBeforeMs: Date.now() - JOB_STALE_AFTER_MS,
+		pageSize: PAGE_SIZE,
+		maxCleared: MAX_CLEARED_PER_RUN,
+	});
+	if (!result.complete) {
+		helpers.logger.warn({...result}, 'Stale job sweep reached its per-run cap');
+	} else if (result.cleared > 0) {
+		helpers.logger.info({...result}, 'Expired stale jobs');
+	}
+};
+
+export default expireStaleJobs;

fluxer_api/src/api/worker/tasks/PrunePostgresKvTtl.ts

@@ -1,18 +1,34 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {Config} from '@app/api/Config';
+import {expireLegacyDefaultTtlRows} from '@app/api/database/PostgresKvDefaultTtlExpiry';
 import {pruneExpiredPostgresKvRows} from '@app/api/database/PostgresKvQueryExecutor';
+import {expireLegacyJobLedgerRows} from '@app/api/jobs/PostgresJobLedgerExpiry';
 import {getDefaultPostgresClient} from '@pkgs/postgres/src/Client';
 import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+import {ms} from 'itty-time';
 
 const PRUNE_BATCH_SIZE = 5000;
 const MAX_PRUNE_BATCHES_PER_RUN = 20;
+const LEGACY_EXPIRY_BUDGET_MS = ms('2 minutes');
 
 const prunePostgresKvTtl: WorkerTaskHandler = async (_payload, helpers) => {
 	if (Config.database.backend !== 'postgres') {
 		return;
 	}
 	const client = getDefaultPostgresClient();
+	const deadlineMs = Date.now() + LEGACY_EXPIRY_BUDGET_MS;
+	const legacyJobs = await expireLegacyJobLedgerRows(client, deadlineMs);
+	if (legacyJobs !== null && (legacyJobs.deleted > 0 || legacyJobs.expiring > 0 || !legacyJobs.complete)) {
+		helpers.logger.info({...legacyJobs}, 'Expired legacy job ledger rows');
+	}
+	const legacyDefaults = await expireLegacyDefaultTtlRows(client, deadlineMs);
+	if (
+		legacyDefaults !== null &&
+		(legacyDefaults.deleted > 0 || legacyDefaults.expiring > 0 || !legacyDefaults.complete)
+	) {
+		helpers.logger.info({...legacyDefaults}, 'Expired rows written without their table default TTL');
+	}
 	let deleted = 0;
 	for (let batch = 0; batch < MAX_PRUNE_BATCHES_PER_RUN; batch += 1) {
 		const batchDeleted = await pruneExpiredPostgresKvRows(client, PRUNE_BATCH_SIZE);

fluxer_api/src/api/worker/tasks/SyncDisposableEmailDomains.ts

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {domainToASCII} from 'node:url';
+import {Config} from '@app/api/Config';
 import {isAccountPolicyContactDomainReputationExempt} from '@app/api/risk/AccountPolicyService';
 import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits';
 import * as FetchUtils from '@app/api/utils/FetchUtils';
@@ -131,10 +132,7 @@ async function throwIfCancelled(helpers: WorkerTaskHelpers): Promise<void> {
 	}
 }
 
-const syncDisposableEmailDomains: WorkerTaskHandler = async (_payload, helpers) => {
-	helpers.logger.info('Starting disposable email domain sync');
-	await helpers.setContextLink('/suspicious-email-domains');
-	const {adminRepository} = getWorkerDependencies();
+async function fetchFeedDomains(helpers: WorkerTaskHelpers): Promise<Set<string>> {
 	const freshSet = new Set<string>();
 	const perSourceCounts: Record<string, number> = {};
 	const perSourceRawCounts: Record<string, number> = {};
@@ -164,6 +162,14 @@ const syncDisposableEmailDomains: WorkerTaskHandler = async (_payload, helpers)
 		},
 		'Fetched disposable email domains from all sources',
 	);
+	return freshSet;
+}
+
+const syncDisposableEmailDomains: WorkerTaskHandler = async (_payload, helpers) => {
+	helpers.logger.info('Starting disposable email domain sync');
+	await helpers.setContextLink('/suspicious-email-domains');
+	const {adminRepository} = getWorkerDependencies();
+	const freshSet = Config.blocklistFeeds.enabled ? await fetchFeedDomains(helpers) : new Set<string>();
 	const currentSet = await loadCurrentDisposableEmailDomains();
 	let addCount = 0;
 	for (const domain of freshSet) {

fluxer_api/src/api/worker/tasks/SyncFileShaBlocklists.ts

@@ -1,16 +1,39 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import {BANNED_FILE_SHAS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
+import {Config} from '@app/api/Config';
+import {
+	BANNED_FILE_SHAS_REFRESH_CHANNEL,
+	ContentBlocklistCategory,
+	isBlocklistFeedFileSha,
+} from '@app/api/constants/ContentModeration';
 import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits';
 import * as FetchUtils from '@app/api/utils/FetchUtils';
 import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
-import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+import type {WorkerTaskHandler, WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
 
 const MALWARE_BAZAAR_SHA256_URL = 'https://bazaar.abuse.ch/export/txt/sha256/recent/';
 const SHA256_RE = /^[0-9a-fA-F]{64}$/;
+
+async function removeFeedFileShas(helpers: WorkerTaskHelpers): Promise<void> {
+	const {adminRepository, kvClient} = getWorkerDependencies();
+	let removed = 0;
+	for (const row of await adminRepository.loadAllBannedFileShas()) {
+		if (!isBlocklistFeedFileSha(row)) continue;
+		if (await adminRepository.unbanFeedFileSha(row.sha256_hex)) removed++;
+	}
+	if (removed > 0) {
+		await kvClient.publish(BANNED_FILE_SHAS_REFRESH_CHANNEL, 'refresh');
+	}
+	helpers.logger.info({removed}, 'Removed file-SHA blocklist feed rows');
+}
+
 const syncFileShaBlocklists: WorkerTaskHandler = async (_payload, helpers) => {
 	helpers.logger.info('Starting file-SHA blocklist sync');
 	await helpers.setContextLink('/file-sha-bans');
+	if (!Config.blocklistFeeds.enabled) {
+		await removeFeedFileShas(helpers);
+		return;
+	}
 	const {adminRepository, kvClient} = getWorkerDependencies();
 	let added = 0;
 	try {
@@ -38,7 +61,7 @@ const syncFileShaBlocklists: WorkerTaskHandler = async (_payload, helpers) => {
 			if (existingSet.has(sha)) continue;
 			await adminRepository.banFileSha({
 				sha256_hex: sha,
-				category: 'malware_bazaar',
+				category: ContentBlocklistCategory.MALWARE_BAZAAR,
 				severity: 2,
 				content_type: null,
 				source_url: MALWARE_BAZAAR_SHA256_URL,

fluxer_api/src/api/worker/tasks/SyncUrlBlocklists.ts

@@ -1,7 +1,8 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import {Config} from '@app/api/Config';
 import {BANNED_URLS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
-import {RISK_S3_KEYS, writeLinesToS3} from '@app/api/risk/RiskBlocklistS3';
+import {deleteRiskS3Object, RISK_S3_KEYS, writeLinesToS3} from '@app/api/risk/RiskBlocklistS3';
 import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits';
 import * as FetchUtils from '@app/api/utils/FetchUtils';
 import {canonicalizeUrl} from '@app/api/utils/UrlNormalizer';
@@ -62,10 +63,23 @@ async function fetchFeed(source: FeedSource): Promise<Array<string>> {
 	return source.parse(text);
 }
 
+const MISSING_FEED_FILE_ERRORS = new Set(['NoSuchBucket', 'NoSuchKey', 'NotFound']);
+
 const syncUrlBlocklists: WorkerTaskHandler = async (_payload, helpers) => {
 	helpers.logger.info('Starting URL blocklist sync');
 	await helpers.setContextLink('/url-domain-bans');
 	const {storageService, kvClient} = getWorkerDependencies();
+	if (!Config.blocklistFeeds.enabled) {
+		try {
+			await deleteRiskS3Object(storageService, RISK_S3_KEYS.feedUrls);
+		} catch (error) {
+			if (!(error instanceof Error && MISSING_FEED_FILE_ERRORS.has(error.name))) {
+				helpers.logger.warn({error}, 'Failed to delete the URL blocklist feed file');
+			}
+		}
+		await kvClient.publish(BANNED_URLS_REFRESH_CHANNEL, 'refresh');
+		return;
+	}
 	const results = await Promise.allSettled(
 		FEED_SOURCES.map(async (source) => ({source, rawUrls: await fetchFeed(source)})),
 	);

fluxer_api/src/api/worker/tests/BlocklistFeedStartup.test.ts

@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {MockKVProvider} from '@app/api/test/mocks/MockKVProvider';
+import {queueBlocklistFeedStartupJobs} from '@app/api/worker/BlocklistFeedStartup';
+import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
+import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError';
+import type {WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes';
+import {describe, expect, it, vi} from 'vitest';
+
+const INITIAL_SYNC_KEY = 'sync:email_domains:initialized';
+const FEED_TASKS = ['syncDisposableEmailDomains', 'syncUrlBlocklists', 'syncFileShaBlocklists'];
+
+function createWorkerService() {
+	return {addJob: vi.fn(async (_task: WorkerTaskName, _payload: WorkerJobPayload) => 1n)};
+}
+
+function queuedTasks(workerService: ReturnType<typeof createWorkerService>): Array<string> {
+	return workerService.addJob.mock.calls.map(([task]) => task);
+}
+
+describe('queueBlocklistFeedStartupJobs', () => {
+	it('with feeds on, a fresh start queues only the disposable sync and claims it for six hours', async () => {
+		const kv = new MockKVProvider();
+		const workerService = createWorkerService();
+
+		await queueBlocklistFeedStartupJobs(kv, workerService, true);
+
+		expect(workerService.addJob.mock.calls).toEqual([['syncDisposableEmailDomains', {}]]);
+		expect(kv.setnxSpy.mock.calls).toEqual([[INITIAL_SYNC_KEY, '1', 21600]]);
+		expect(kv.delSpy).not.toHaveBeenCalled();
+	});
+
+	it('with feeds on, a start inside the claim queues nothing', async () => {
+		const kv = new MockKVProvider();
+		const workerService = createWorkerService();
+
+		await queueBlocklistFeedStartupJobs(kv, workerService, true);
+		await queueBlocklistFeedStartupJobs(kv, workerService, true);
+
+		expect(queuedTasks(workerService)).toEqual(['syncDisposableEmailDomains']);
+	});
+
+	it('with feeds off, the first start queues all three feed tasks and a second start queues none', async () => {
+		const kv = new MockKVProvider();
+		const workerService = createWorkerService();
+
+		await queueBlocklistFeedStartupJobs(kv, workerService, false);
+		expect(workerService.addJob.mock.calls).toEqual(FEED_TASKS.map((task) => [task, {}]));
+
+		await queueBlocklistFeedStartupJobs(kv, workerService, false);
+		expect(queuedTasks(workerService)).toEqual(FEED_TASKS);
+	});
+
+	it('with feeds off, a start after a feeds-on start cleans up again inside the claim', async () => {
+		const kv = new MockKVProvider();
+		const workerService = createWorkerService();
+
+		await queueBlocklistFeedStartupJobs(kv, workerService, false);
+		await queueBlocklistFeedStartupJobs(kv, workerService, true);
+		await queueBlocklistFeedStartupJobs(kv, workerService, false);
+
+		expect(queuedTasks(workerService)).toEqual([...FEED_TASKS, 'syncDisposableEmailDomains', ...FEED_TASKS]);
+		expect(await kv.exists(INITIAL_SYNC_KEY)).toBe(0);
+	});
+
+	it('a legacy initial sync key without expiry is cleared, so re-enabling runs the initial sync', async () => {
+		const kv = new MockKVProvider();
+		await kv.set(INITIAL_SYNC_KEY, '1');
+		expect(await kv.ttl(INITIAL_SYNC_KEY)).toBe(-1);
+
+		await queueBlocklistFeedStartupJobs(kv, createWorkerService(), false);
+		expect(await kv.exists(INITIAL_SYNC_KEY)).toBe(0);
+
+		const workerService = createWorkerService();
+		await queueBlocklistFeedStartupJobs(kv, workerService, true);
+		expect(queuedTasks(workerService)).toEqual(['syncDisposableEmailDomains']);
+	});
+
+	it('a full jobs stream drops the job without failing startup', async () => {
+		const overflowing = createWorkerService();
+		overflowing.addJob.mockImplementation(async (task) => {
+			throw new WorkerQueueOverflowError(task, 'maximum messages exceeded');
+		});
+		await expect(queueBlocklistFeedStartupJobs(new MockKVProvider(), overflowing, false)).resolves.toBeUndefined();
+		expect(queuedTasks(overflowing)).toEqual(FEED_TASKS);
+
+		const failing = createWorkerService();
+		failing.addJob.mockRejectedValue(new Error('jetstream unavailable'));
+		await expect(queueBlocklistFeedStartupJobs(new MockKVProvider(), failing, false)).rejects.toThrow(
+			'jetstream unavailable',
+		);
+	});
+});

fluxer_api/src/api/worker/tests/BlocklistFeedsDisabled.test.ts

@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {Config} from '@app/api/Config';
+import {
+	BANNED_FILE_SHAS_REFRESH_CHANNEL,
+	BANNED_URLS_REFRESH_CHANNEL,
+	ContentBlocklistCategory,
+} from '@app/api/constants/ContentModeration';
+import type {BannedFileShaRow} from '@app/api/database/types/AdminArchiveTypes';
+import {fileShaCache} from '@app/api/middleware/FileShaCache';
+import {getAdminRepository} from '@app/api/middleware/ServiceSingletons';
+import {urlBlocklistCache} from '@app/api/middleware/UrlBlocklistCache';
+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
+import type {MockKVProvider} from '@app/api/test/mocks/MockKVProvider';
+import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
+import {canonicalizeUrl} from '@app/api/utils/UrlNormalizer';
+import syncDisposableEmailDomains from '@app/api/worker/tasks/SyncDisposableEmailDomains';
+import syncFileShaBlocklists from '@app/api/worker/tasks/SyncFileShaBlocklists';
+import syncUrlBlocklists from '@app/api/worker/tasks/SyncUrlBlocklists';
+import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext';
+import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
+import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, type MockInstance, vi} from 'vitest';
+
+const FEED_BUCKET = 'fluxer-geoip';
+const FEED_KEY = 'blocklists/feed-urls.txt';
+const FEED_SHA = 'a1'.repeat(32);
+const ADMIN_BAZAAR_SHA = 'b2'.repeat(32);
+const ADMIN_MANUAL_SHA = 'c3'.repeat(32);
+const UNOWNED_NCMEC_SHA = 'd4'.repeat(32);
+const ADMIN_USER_ID = 42n;
+
+function createHelpers(overrides: Partial<WorkerTaskHelpers> = {}): WorkerTaskHelpers {
+	return {
+		logger: new NoopLogger(),
+		jobId: 4242n,
+		addJob: async () => 0n,
+		reportProgress: async () => {},
+		shouldCancel: async () => false,
+		setContextLink: async () => {},
+		...overrides,
+	};
+}
+
+function fileShaRow(sha256Hex: string, category: string, addedBy: bigint | null): BannedFileShaRow {
+	return {
+		sha256_hex: sha256Hex,
+		category,
+		severity: 2,
+		content_type: null,
+		source_url: null,
+		added_at: new Date(),
+		added_by: addedBy,
+		notes: null,
+	};
+}
+
+async function seedFileShas(): Promise<void> {
+	const adminRepository = getAdminRepository();
+	await adminRepository.banFileSha(fileShaRow(FEED_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, null));
+	await adminRepository.banFileSha(
+		fileShaRow(ADMIN_BAZAAR_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, ADMIN_USER_ID),
+	);
+	await adminRepository.banFileSha(fileShaRow(ADMIN_MANUAL_SHA, ContentBlocklistCategory.MANUAL, ADMIN_USER_ID));
+	await adminRepository.banFileSha(fileShaRow(UNOWNED_NCMEC_SHA, ContentBlocklistCategory.NCMEC, null));
+}
+
+describe('blocklist feeds turned off', () => {
+	let harness: ApiTestHarness;
+	let previousFeedsEnabled: boolean;
+	let fetchSpy: MockInstance<typeof fetch>;
+
+	beforeAll(async () => {
+		harness = await createApiTestHarness();
+	});
+
+	beforeEach(async () => {
+		previousFeedsEnabled = Config.blocklistFeeds.enabled;
+		await harness.reset();
+		harness.storageService.reset();
+		setWorkerDependenciesForTest({
+			adminRepository: getAdminRepository(),
+			kvClient: harness.kvProvider,
+			storageService: harness.storageService,
+		});
+		fetchSpy = vi.spyOn(globalThis, 'fetch');
+	});
+
+	afterEach(() => {
+		Config.blocklistFeeds.enabled = previousFeedsEnabled;
+		fetchSpy.mockRestore();
+	});
+
+	afterAll(async () => {
+		clearWorkerDependencies();
+		await harness?.shutdown();
+	});
+
+	function publishCalls(): Array<Array<string>> {
+		return (harness.kvProvider as MockKVProvider).publishSpy.mock.calls;
+	}
+
+	it('removes every stored disposable domain without fetching a feed', async () => {
+		const adminRepository = getAdminRepository();
+		for (const domain of ['mailinator.com', 'guerrillamail.com', 'tempmail.dev']) {
+			await adminRepository.addDisposableEmailDomain(domain);
+		}
+		Config.blocklistFeeds.enabled = false;
+		const reportProgress = vi.fn(async () => {});
+
+		await syncDisposableEmailDomains({}, createHelpers({reportProgress}));
+
+		expect(await adminRepository.listDisposableEmailDomains()).toEqual([]);
+		expect(fetchSpy).not.toHaveBeenCalled();
+		expect(reportProgress).toHaveBeenLastCalledWith(3, 3, '+0 added, -3 removed');
+	});
+
+	it('reports a stored domain as disposable only while feeds are on', async () => {
+		const adminRepository = getAdminRepository();
+		await adminRepository.addDisposableEmailDomain('mailinator.com');
+
+		Config.blocklistFeeds.enabled = true;
+		expect(await adminRepository.isEmailDomainDisposable('mailinator.com')).toBe(true);
+		Config.blocklistFeeds.enabled = false;
+		expect(await adminRepository.isEmailDomainDisposable('mailinator.com')).toBe(false);
+	});
+
+	it('removes MalwareBazaar feed rows and keeps every other file-SHA ban', async () => {
+		await seedFileShas();
+		Config.blocklistFeeds.enabled = false;
+
+		await syncFileShaBlocklists({}, createHelpers());
+
+		const remaining = (await getAdminRepository().loadAllBannedFileShas()).map((row) => row.sha256_hex).sort();
+		expect(remaining).toEqual([ADMIN_BAZAAR_SHA, ADMIN_MANUAL_SHA, UNOWNED_NCMEC_SHA]);
+		expect(publishCalls()).toEqual([[BANNED_FILE_SHAS_REFRESH_CHANNEL, 'refresh']]);
+		expect(fetchSpy).not.toHaveBeenCalled();
+	});
+
+	it('keeps a file-SHA ban an Admin took over after the purge read the table', async () => {
+		const adminRepository = getAdminRepository();
+		await adminRepository.banFileSha(fileShaRow(FEED_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, null));
+		const staleRows = await adminRepository.loadAllBannedFileShas();
+		await adminRepository.banFileSha(fileShaRow(FEED_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, ADMIN_USER_ID));
+		const staleRead = vi.spyOn(adminRepository, 'loadAllBannedFileShas').mockResolvedValueOnce(staleRows);
+		Config.blocklistFeeds.enabled = false;
+
+		await syncFileShaBlocklists({}, createHelpers());
+
+		staleRead.mockRestore();
+		expect(await adminRepository.isFileShaBanned(FEED_SHA)).toBe(true);
+	});
+
+	it('file-SHA cache skips feed rows only while feeds are off', async () => {
+		await seedFileShas();
+
+		Config.blocklistFeeds.enabled = false;
+		await fileShaCache.refresh();
+		expect(fileShaCache.isBanned(FEED_SHA)).toBe(false);
+		expect(fileShaCache.isBanned(ADMIN_BAZAAR_SHA)).toBe(true);
+		expect(fileShaCache.isBanned(ADMIN_MANUAL_SHA)).toBe(true);
+		expect(fileShaCache.isBanned(UNOWNED_NCMEC_SHA)).toBe(true);
+
+		Config.blocklistFeeds.enabled = true;
+		await fileShaCache.refresh();
+		expect(fileShaCache.isBanned(FEED_SHA)).toBe(true);
+	});
+
+	it('deletes the URL feed file and never fetches while feeds are off', async () => {
+		await harness.storageService.uploadObject({
+			bucket: FEED_BUCKET,
+			key: FEED_KEY,
+			body: Buffer.from('https://feed.example/x\n'),
+		});
+		Config.blocklistFeeds.enabled = false;
+
+		await syncUrlBlocklists({}, createHelpers());
+
+		expect(harness.storageService.deleteObjectSpy).toHaveBeenCalledWith(FEED_BUCKET, FEED_KEY);
+		expect(harness.storageService.hasObject(FEED_BUCKET, FEED_KEY)).toBe(false);
+		expect(publishCalls()).toEqual([[BANNED_URLS_REFRESH_CHANNEL, 'refresh']]);
+		expect(fetchSpy).not.toHaveBeenCalled();
+
+		const quietLogger = new NoopLogger();
+		const quietWarn = vi.spyOn(quietLogger, 'warn');
+		vi.spyOn(harness.storageService, 'deleteObject').mockRejectedValueOnce(
+			Object.assign(new Error('The specified bucket does not exist'), {name: 'NoSuchBucket'}),
+		);
+		await expect(syncUrlBlocklists({}, createHelpers({logger: quietLogger}))).resolves.toBeUndefined();
+		expect(quietWarn).not.toHaveBeenCalled();
+
+		const loudLogger = new NoopLogger();
+		const loudWarn = vi.spyOn(loudLogger, 'warn');
+		harness.storageService.configure({shouldFailDelete: true});
+		await expect(syncUrlBlocklists({}, createHelpers({logger: loudLogger}))).resolves.toBeUndefined();
+		expect(loudWarn).toHaveBeenCalledTimes(1);
+	});
+
+	it('URL cache ignores the feed file only while feeds are off', async () => {
+		const feedUrl = canonicalizeUrl('https://feed.example/x');
+		const adminUrl = canonicalizeUrl('https://admin.example/y');
+		expect(feedUrl).not.toBeNull();
+		expect(adminUrl).not.toBeNull();
+		await harness.storageService.uploadObject({
+			bucket: FEED_BUCKET,
+			key: FEED_KEY,
+			body: Buffer.from(`${feedUrl}\n`),
+		});
+		await getAdminRepository().banUrl({
+			url_canonical: adminUrl!,
+			category: ContentBlocklistCategory.MANUAL,
+			severity: 2,
+			source_url: null,
+			added_at: new Date(),
+			added_by: ADMIN_USER_ID,
+			notes: null,
+		});
+		urlBlocklistCache.setStorageService(harness.storageService);
+
+		Config.blocklistFeeds.enabled = false;
+		await urlBlocklistCache.refresh();
+		expect(urlBlocklistCache.isUrlBanned('https://feed.example/x')).toBe(false);
+		expect(urlBlocklistCache.isUrlBanned('https://admin.example/y')).toBe(true);
+
+		Config.blocklistFeeds.enabled = true;
+		await urlBlocklistCache.refresh();
+		expect(urlBlocklistCache.isUrlBanned('https://feed.example/x')).toBe(true);
+	});
+});

fluxer_api/src/api/worker/tests/BlocklistFeedsDisabledPostgres.test.ts

@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {AdminRepository} from '@app/api/admin/AdminRepository';
+import {Config} from '@app/api/Config';
+import {ContentBlocklistCategory} from '@app/api/constants/ContentModeration';
+import {setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
+import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';
+import type {BannedFileShaRow} from '@app/api/database/types/AdminArchiveTypes';
+import {startDockerContainer} from '@app/api/test/DockerTestContainer';
+import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
+import {MockKVProvider} from '@app/api/test/mocks/MockKVProvider';
+import {MockStorageService} from '@app/api/test/mocks/MockStorageService';
+import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
+import syncDisposableEmailDomains from '@app/api/worker/tasks/SyncDisposableEmailDomains';
+import syncFileShaBlocklists from '@app/api/worker/tasks/SyncFileShaBlocklists';
+import {
+	clearWorkerDependencies,
+	getWorkerDependencies,
+	setWorkerDependenciesForTest,
+} from '@app/api/worker/WorkerContext';
+import {
+	getDefaultPostgresClient,
+	type IPostgresClient,
+	initPostgres,
+	shutdownPostgres,
+} from '@pkgs/postgres/src/Client';
+import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
+import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';
+
+const KV_TABLE = 'kv_blocklist_feeds_disabled';
+const CONTAINER = `fluxer-feeds-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+const FEED_SHA = 'a1'.repeat(32);
+const ADMIN_BAZAAR_SHA = 'b2'.repeat(32);
+const ADMIN_MANUAL_SHA = 'c3'.repeat(32);
+const ADMIN_USER_ID = 1_234_567_890_123n;
+
+async function sleep(ms: number): Promise<void> {
+	await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function freePort(): Promise<number> {
+	return new Promise((resolve, reject) => {
+		const server = createServer();
+		server.on('error', reject);
+		server.listen(0, '127.0.0.1', () => {
+			const address = server.address();
+			if (typeof address === 'string' || address === null) {
+				reject(new Error('no port'));
+				return;
+			}
+			const port = address.port;
+			server.close(() => resolve(port));
+		});
+	});
+}
+
+function createHelpers(): WorkerTaskHelpers {
+	return {
+		logger: new NoopLogger(),
+		jobId: 4242n,
+		addJob: async () => 0n,
+		reportProgress: async () => {},
+		shouldCancel: async () => false,
+		setContextLink: async () => {},
+	};
+}
+
+function fileShaRow(sha256Hex: string, category: string, addedBy: bigint | null): BannedFileShaRow {
+	return {
+		sha256_hex: sha256Hex,
+		category,
+		severity: 2,
+		content_type: null,
+		source_url: null,
+		added_at: new Date(),
+		added_by: addedBy,
+		notes: null,
+	};
+}
+
+describe.skipIf(!dockerAvailable)('blocklist feeds turned off against postgres', () => {
+	let raw: IPostgresClient;
+	let executor: PostgresKvQueryExecutor;
+	let previousFeedsEnabled: boolean;
+
+	beforeAll(async () => {
+		const port = await freePort();
+		startDockerContainer([
+			'run',
+			'-d',
+			'--name',
+			CONTAINER,
+			'-e',
+			'POSTGRES_USER=fluxer',
+			'-e',
+			'POSTGRES_PASSWORD=fluxer',
+			'-e',
+			'POSTGRES_DB=fluxer',
+			'-p',
+			`127.0.0.1:${port}:5432`,
+			'postgres:16-alpine',
+			'-c',
+			'fsync=off',
+		]);
+		let ready = false;
+		for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+			await sleep(500);
+			const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
+				stdio: 'ignore',
+			});
+			if (probe.status !== 0) continue;
+			try {
+				await initPostgres({
+					url: `postgres://fluxer:[email protected]:${port}/fluxer`,
+					maxConnections: 4,
+					kvTable: KV_TABLE,
+				});
+				await getDefaultPostgresClient().query('SELECT 1');
+				ready = true;
+			} catch {
+				await shutdownPostgres().catch(() => {});
+			}
+		}
+		if (!ready) throw new Error('postgres never came up');
+		raw = getDefaultPostgresClient();
+		await ensurePostgresKvSchema(raw);
+		executor = new PostgresKvQueryExecutor(raw);
+	}, 900_000);
+
+	beforeEach(async () => {
+		previousFeedsEnabled = Config.blocklistFeeds.enabled;
+		await raw.query(`DELETE FROM ${KV_TABLE}`);
+		setCassandraQueryExecutorForTesting(executor);
+		setWorkerDependenciesForTest({
+			adminRepository: new AdminRepository(),
+			kvClient: new MockKVProvider(),
+			storageService: new MockStorageService(),
+		});
+		Config.blocklistFeeds.enabled = false;
+	});
+
+	afterEach(() => {
+		Config.blocklistFeeds.enabled = previousFeedsEnabled;
+	});
+
+	afterAll(async () => {
+		clearWorkerDependencies();
+		setCassandraQueryExecutorForTesting(new InMemoryCassandraQueryExecutor());
+		await shutdownPostgres().catch(() => {});
+		spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+	});
+
+	it('removes every disposable_email_domains row from the KV table', async () => {
+		const repository = new AdminRepository();
+		for (let i = 0; i < 1200; i++) {
+			await repository.addDisposableEmailDomain(`disposable-${i}.example`);
+		}
+		const seeded = await raw.query<{count: number}>(
+			`SELECT count(*)::int AS count FROM ${KV_TABLE} WHERE table_name = 'disposable_email_domains'`,
+		);
+		expect(seeded.rows[0]?.count).toBe(1200);
+
+		await syncDisposableEmailDomains({}, createHelpers());
+
+		const remaining = await raw.query<{count: number}>(
+			`SELECT count(*)::int AS count FROM ${KV_TABLE} WHERE table_name = 'disposable_email_domains'`,
+		);
+		expect(remaining.rows[0]?.count).toBe(0);
+	});
+
+	it('keeps file-SHA rows with added_by set and removes feed rows stored with a JSON null added_by', async () => {
+		const repository = new AdminRepository();
+		await repository.banFileSha(fileShaRow(FEED_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, null));
+		await repository.banFileSha(fileShaRow(ADMIN_BAZAAR_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, ADMIN_USER_ID));
+		await repository.banFileSha(fileShaRow(ADMIN_MANUAL_SHA, ContentBlocklistCategory.MANUAL, ADMIN_USER_ID));
+		const feedRow = await raw.query<{added_by_type: string}>(
+			`SELECT jsonb_typeof(row_data->'added_by') AS added_by_type FROM ${KV_TABLE}
+WHERE table_name = 'banned_file_shas' AND row_data->>'sha256_hex' = $1`,
+			[FEED_SHA],
+		);
+		expect(feedRow.rows).toEqual([{added_by_type: 'null'}]);
+
+		await syncFileShaBlocklists({}, createHelpers());
+
+		const remaining = await raw.query<{sha: string}>(
+			`SELECT row_data->>'sha256_hex' AS sha FROM ${KV_TABLE} WHERE table_name = 'banned_file_shas' ORDER BY 1`,
+		);
+		expect(remaining.rows.map((row) => row.sha)).toEqual([ADMIN_BAZAAR_SHA, ADMIN_MANUAL_SHA]);
+	});
+
+	it('keeps a file-SHA ban an Admin took over after the purge read the table', async () => {
+		const {adminRepository} = getWorkerDependencies();
+		await adminRepository.banFileSha(fileShaRow(FEED_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, null));
+		const staleRows = await adminRepository.loadAllBannedFileShas();
+		await adminRepository.banFileSha(fileShaRow(FEED_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, ADMIN_USER_ID));
+		const staleRead = vi.spyOn(adminRepository, 'loadAllBannedFileShas').mockResolvedValueOnce(staleRows);
+
+		await syncFileShaBlocklists({}, createHelpers());
+
+		staleRead.mockRestore();
+		expect(await adminRepository.isFileShaBanned(FEED_SHA)).toBe(true);
+	});
+});

fluxer_api/src/api/worker/tests/JetStreamWorkerQueue.test.ts

@@ -94,7 +94,7 @@ function createQueue(params: {
 	dlqExists?: boolean;
 	reject?: (config: Partial<StreamConfig>) => Error | null;
 	updateError?: Error;
-	publish?: (subject: string) => {seq: number};
+	publish?: (subject: string, body: string, options: {msgID: string}) => {seq: number; duplicate?: boolean};
 	subjectCounts?: Record<string, number>;
 	subjectCountsError?: Error;
 }): {
@@ -171,9 +171,9 @@ function createQueue(params: {
 				},
 			}),
 		getJetStreamClient: () => ({
-			publish: (subject: string) => {
+			publish: (subject: string, body: string, options: {msgID: string}) => {
 				const publish = params.publish ?? (() => ({seq: 1}));
-				return Promise.resolve(publish(subject));
+				return Promise.resolve(publish(subject, body, options));
 			},
 		}),
 	} as unknown as JetStreamConnectionManager;
@@ -281,6 +281,27 @@ describe('jobs stream limits', () => {
 	});
 });
 
+describe('jobs stream max age', () => {
+	it('reports the max age of a jobs stream it creates', async () => {
+		const {queue, added} = createQueue({existing: null});
+		await queue.ensureStream();
+		expect(queue.getJobsStreamMaxAgeMs()).toBe(7 * 24 * 60 * 60 * 1000);
+		expect(added[0]?.max_age).toBe(queue.getJobsStreamMaxAgeMs() * 1_000_000);
+	});
+
+	it('reports the max age an existing jobs stream really has and never changes it', async () => {
+		for (const [maxAgeNanos, expectedMs] of [
+			[30 * 24 * 60 * 60 * 1_000_000_000, 30 * 24 * 60 * 60 * 1000],
+			[0, 0],
+		] as const) {
+			const {queue, updated} = createQueue({existing: {...LEGACY_CONFIG, max_age: maxAgeNanos} as StreamConfig});
+			await queue.ensureStream();
+			expect(queue.getJobsStreamMaxAgeMs()).toBe(expectedMs);
+			expect(updated.some((config) => 'max_age' in config)).toBe(false);
+		}
+	});
+});
+
 describe('dead-letter stream', () => {
 	it('keeps startup alive when the dead-letter stream does not fit', async () => {
 		const {queue, dlqAdded} = createQueue({dlqExists: false, reject: () => noStorageError()});
@@ -298,16 +319,16 @@ describe('dead-letter stream', () => {
 describe('jobs stream enqueue shedding', () => {
 	it('rejects enqueues once the stream is at its cap', async () => {
 		const {queue} = createQueue({existing: LEGACY_CONFIG, publish: boundedPublisher(2)});
-		await expect(queue.enqueue('extractEmbeds', {})).resolves.toBe('1');
-		await expect(queue.enqueue('extractEmbeds', {})).resolves.toBe('2');
+		await expect(queue.enqueue('extractEmbeds', {})).resolves.toEqual({seq: '1', duplicate: false});
+		await expect(queue.enqueue('extractEmbeds', {})).resolves.toEqual({seq: '2', duplicate: false});
 		await expect(queue.enqueue('extractEmbeds', {})).rejects.toBeInstanceOf(WorkerQueueOverflowError);
 	});
 
 	it('caps each task type independently', async () => {
 		const {queue} = createQueue({existing: LEGACY_CONFIG, publish: boundedPublisher(1)});
-		await expect(queue.enqueue('extractEmbeds', {})).resolves.toBe('1');
+		await expect(queue.enqueue('extractEmbeds', {})).resolves.toEqual({seq: '1', duplicate: false});
 		await expect(queue.enqueue('extractEmbeds', {})).rejects.toBeInstanceOf(WorkerQueueOverflowError);
-		await expect(queue.enqueue('handleMentions', {})).resolves.toBe('2');
+		await expect(queue.enqueue('handleMentions', {})).resolves.toEqual({seq: '2', duplicate: false});
 	});
 
 	it('sheds enqueues the server refuses for lack of resources', async () => {
@@ -320,6 +341,29 @@ describe('jobs stream enqueue shedding', () => {
 		await expect(queue.enqueue('handleMentions', {})).rejects.toBeInstanceOf(WorkerQueueOverflowError);
 	});
 
+	it('reports a publish the stream deduplicated under the same job key', async () => {
+		const seen = new Map<string, number>();
+		const {queue} = createQueue({
+			existing: LEGACY_CONFIG,
+			publish: (_subject, _body, options) => {
+				const existing = seen.get(options.msgID);
+				if (existing !== undefined) return {seq: existing, duplicate: true};
+				seen.set(options.msgID, seen.size + 1);
+				return {seq: seen.size, duplicate: false};
+			},
+		});
+		const options = {jobKey: 'batch-audit-log-message-deletes:1'};
+		await expect(queue.enqueue('batchGuildAuditLogMessageDeletes', {}, options)).resolves.toEqual({
+			seq: '1',
+			duplicate: false,
+		});
+		await expect(queue.enqueue('batchGuildAuditLogMessageDeletes', {}, options)).resolves.toEqual({
+			seq: '1',
+			duplicate: true,
+		});
+		await expect(queue.enqueue('batchGuildAuditLogMessageDeletes', {})).resolves.toEqual({seq: '2', duplicate: false});
+	});
+
 	it('rethrows publish failures that are not stream limits', async () => {
 		const failure = new Error('no responders');
 		const {queue} = createQueue({

fluxer_api/src/api/worker/tests/WorkerServiceLedger.test.ts

@@ -7,6 +7,7 @@ import {WorkerService} from '@app/api/worker/WorkerService';
 import {describe, expect, test} from 'vitest';
 
 const JOB_ID = 4242n;
+const CREATED_AT = new Date('2026-09-21T12:00:00.000Z');
 
 function createSnowflake(): ISnowflakeService {
 	return {
@@ -14,17 +15,24 @@ function createSnowflake(): ISnowflakeService {
 	} as unknown as ISnowflakeService;
 }
 
-function createHarness(options?: {createJobError?: Error; enqueueError?: Error}) {
+function createHarness(options?: {
+	createJobError?: Error;
+	enqueueError?: Error;
+	duplicate?: boolean;
+	discardError?: Error;
+}) {
 	const calls: Array<string> = [];
 	const createdJobs: Array<CreateJobInput> = [];
 	const enqueued: Array<{taskType: string; payload: Record<string, unknown>}> = [];
 	const seqUpdates: Array<{jobId: bigint; seq: string}> = [];
 	const deadletters: Array<{jobId: bigint; errorMessage: string}> = [];
+	const discarded: Array<{jobId: bigint; createdAt: Date}> = [];
 	const ledger = {
 		createJob: async (input: CreateJobInput) => {
 			calls.push('createJob');
 			if (options?.createJobError) throw options.createJobError;
 			createdJobs.push(input);
+			return CREATED_AT;
 		},
 		setJetStreamSeq: async (jobId: bigint, seq: string) => {
 			calls.push('setJetStreamSeq');
@@ -34,17 +42,22 @@ function createHarness(options?: {createJobError?: Error; enqueueError?: Error})
 			calls.push('markDeadletter');
 			deadletters.push({jobId, errorMessage});
 		},
+		discardJob: async (jobId: bigint, createdAt: Date) => {
+			calls.push('discardJob');
+			if (options?.discardError) throw options.discardError;
+			discarded.push({jobId, createdAt});
+		},
 	} as unknown as IJobLedgerRepository;
 	const queue = {
 		enqueue: async (taskType: string, payload: Record<string, unknown>) => {
 			calls.push('enqueue');
 			if (options?.enqueueError) throw options.enqueueError;
 			enqueued.push({taskType, payload});
-			return 'seq-9';
+			return {seq: 'seq-9', duplicate: options?.duplicate === true};
 		},
 	} as unknown as JetStreamWorkerQueue;
 	const service = new WorkerService(queue, createSnowflake(), ledger);
-	return {service, calls, createdJobs, enqueued, seqUpdates, deadletters};
+	return {service, calls, createdJobs, enqueued, seqUpdates, deadletters, discarded};
 }
 
 describe('WorkerService ledger ordering', () => {
@@ -88,6 +101,42 @@ describe('WorkerService ledger ordering', () => {
 		expect(harness.deadletters).toEqual([{jobId: JOB_ID, errorMessage: 'stream unreachable'}]);
 	});
 
+	test('discards the ledger row when the stream already holds a job under the same key', async () => {
+		const harness = createHarness({duplicate: true});
+
+		const jobId = await harness.service.addJob(
+			'batchGuildAuditLogMessageDeletes',
+			{guildId: '1'},
+			{jobKey: 'batch-audit-log-message-deletes:1'},
+		);
+
+		expect(jobId).toBe(JOB_ID);
+		expect(harness.calls).toEqual(['createJob', 'enqueue', 'discardJob']);
+		expect(harness.discarded).toEqual([{jobId: JOB_ID, createdAt: CREATED_AT}]);
+		expect(harness.seqUpdates).toEqual([]);
+	});
+
+	test('still returns the job id when discarding the duplicate ledger row fails', async () => {
+		const harness = createHarness({duplicate: true, discardError: new Error('write timeout')});
+
+		const jobId = await harness.service.addJob(
+			'batchGuildAuditLogMessageDeletes',
+			{guildId: '1'},
+			{jobKey: 'batch-audit-log-message-deletes:1'},
+		);
+
+		expect(jobId).toBe(JOB_ID);
+		expect(harness.calls).toEqual(['createJob', 'enqueue', 'discardJob']);
+	});
+
+	test('does not touch the ledger for a duplicate the caller never ledgered', async () => {
+		const harness = createHarness({duplicate: true});
+
+		await harness.service.addJob('handleMentions', {}, {skipLedger: true, jobKey: 'mentions:1'});
+
+		expect(harness.calls).toEqual(['enqueue']);
+	});
+
 	test('never touches the ledger when the caller skips it', async () => {
 		const harness = createHarness();
 

fluxer_docs/src/content/docs/admin-api/blocklists.mdx

@@ -13,7 +13,7 @@ Each list has its own [Admin ACLs](/admin-api/#acl-registry). A read needs the s
 Fluxer builds each permission name from `ban:`, the list name with each hyphen written as an underscore, and the verb, so the `url-domain` list uses `ban:url_domain:check`, `ban:url_domain:add`, and `ban:url_domain:remove`. The `email-domain-suspicious` list is the one exception and uses `suspicious_email_domain:check`, `suspicious_email_domain:add`, and `suspicious_email_domain:remove`.
 
 :::note[The Admin API exposes no other blocklist]
-Fluxer synchronises disposable email domains from external feeds every six hours. No operation on this page reads or writes them.
+Fluxer synchronises disposable email domains from external feeds every six hours when `FLUXER_BLOCKLIST_FEEDS_ENABLED` is on. No operation on this page reads or writes them.
 :::
 
 ## Blocklist types
@@ -67,6 +67,8 @@ Every `url`, `url-domain`, `file-sha`, and `avatar-hash` row has a category nami
 | gifct | Imported from a GIFCT hash set |
 | stop_ncii | Imported from a StopNCII hash set |
 
+With blocklist feeds off, Fluxer ignores and removes every `file-sha` row in the `malware_bazaar` category that no Admin added.
+
 The request field is a free string of 1 through 64 characters, so a value outside this registry is accepted and stored verbatim. Tolerate a stored category outside the registry.
 
 ## Content blocklist severities

fluxer_docs/src/content/docs/admin-api/jobs.mdx

@@ -11,7 +11,7 @@ These routes list recorded background jobs and request cancellation. Create jobs
 Reads require `jobs:view` and cancellation requires `jobs:cancel`. Every operation on this page shares the `admin:jobs:view` bucket. The three reads record no [Admin audit entry](/admin-api/#admin-audit-entry-object), and cancellation records one.
 
 :::note[Job updates vary by task]
-Mention processing, link previews, and every scheduled task except `syncDisposableEmailDomains`, `syncUrlBlocklists` and `syncFileShaBlocklists` run with no job record and never appear here. A job also runs with no record when Fluxer fails to write that record. When a later write of status, progress or attempts fails, Fluxer logs the failure and the job continues, so the stored values can be behind the real run. Use the [archive routes](/admin-api/archives/) to track archive progress and failures.
+Mention processing, link previews, message delete audit log batching, and every scheduled task except `syncDisposableEmailDomains`, `syncUrlBlocklists` and `syncFileShaBlocklists` run with no job record and never appear here. Fluxer deletes a job record about 90 days after the job was queued. A job also runs with no record when Fluxer fails to write that record. When a later write of status, progress or attempts fails, Fluxer logs the failure and the job continues, so the stored values can be behind the real run. Use the [archive routes](/admin-api/archives/) to track archive progress and failures.
 :::
 
 ## Admin job object
@@ -108,6 +108,8 @@ Pass all three fields from `next_cursor` back to [List jobs](#list-jobs) using t
 
 `queued` and `running` accept cancellation requests. The other statuses are terminal. A cancellation request does not guarantee that the task will stop.
 
+The queue drops a job 7 days after it was queued. A job still recorded as `queued` or `running` after 8 days is set to `deadletter` with the error `Expired from the job queue`.
+
 ## Processing lanes
 
 `jet_stream_lane` identifies the job's processing group.
@@ -139,12 +141,12 @@ Returns a page of [Admin job](#admin-job-object) objects, newest first, within t
 | cursor_bucket_day? | string | `next_cursor.bucket_day` as a `YYYY-MM-DD` UTC date |
 | cursor_created_at? | string | `next_cursor.created_at` as an ISO 8601 timestamp (1-64 characters) |
 | cursor_job_id? | snowflake | The job identifier from `next_cursor.job_id` |
-| max_lookback_days? | integer | The number of days before the cursor date or today to include (1-60, default 14) |
+| max_lookback_days? | integer | The number of days before today to include (1-60, default 14) |
 | status? | string | Filter by recorded [job status](#job-statuses) |
 | task_type? | string | Filter by [task type](#background-job-task-types) (1-128 characters) |
 | requested_by_user_id? | snowflake | Filter by recorded requester |
 
-Supply all three cursor parameters together or omit all three. An incomplete or malformed cursor returns 400 `INVALID_FORM_BODY`. An unknown task type returns an empty page.
+Supply all three cursor parameters together or omit all three. An incomplete or malformed cursor returns 400 `INVALID_FORM_BODY`. An unknown task type returns an empty page. A full page always returns `next_cursor`, so the next request can return an empty page.
 
 ### Response body
 

fluxer_docs/src/content/docs/operator/configuration.mdx

@@ -989,7 +989,9 @@ Default `flag_spammer`. What happens when it fires. `flag_spammer` or `suppress_
 
 #### `FLUXER_BLOCKLIST_FEEDS_ENABLED`
 
-Defaults to the inverse of `FLUXER_SELF_HOSTED`. External blocklist feeds. Off by default on a self-hosted instance.
+Defaults to the inverse of `FLUXER_SELF_HOSTED`. Off by default on a self-hosted instance. With feeds on, the worker downloads disposable email domains and the URLhaus and PhishTank URL lists every six hours, and MalwareBazaar file hashes every twelve hours. Fluxer checks registrations and email changes against the domains, posted links against the URLs, and uploads against the hashes.
+
+With feeds off, Fluxer checks none of this data. The first worker start with feeds off removes every disposable email domain, the URL feed file, and every `malware_bazaar` file-SHA ban that no Admin added. File-SHA bans added through the Admin API stay. Turning feeds back on downloads the disposable domains at the next worker start, the URLs within six hours, and the hashes within twelve.
 
 #### `FLUXER_TOR_EXIT_LIST_ENABLED`
 
@@ -1776,7 +1778,7 @@ Default `256mb`. The ceiling for `edge`.
 
 #### `FLUXER_POSTGRES_MEMORY_LIMIT`
 
-Default `5gb`. The ceiling for `postgres`. Must be at or above `FLUXER_POSTGRES_MEMORY_RESERVATION` or the container fails to create. The 256mb `shm_size` is charged against it.
+Default `5gb`. The ceiling for `postgres`. Must be at or above `FLUXER_POSTGRES_MEMORY_RESERVATION` or the container fails to create. Shared memory in use under `FLUXER_POSTGRES_SHM_SIZE` is charged against it.
 
 #### `FLUXER_VALKEY_MEMORY_LIMIT`
 
@@ -1932,7 +1934,11 @@ Default `256MB`. The budget for one VACUUM, CREATE INDEX, or ALTER TABLE. One su
 
 Default `128MB`. The budget for each autovacuum worker. Postgres runs three workers by default, so budget three times this value.
 
-The remaining Postgres settings are fixed on the command line, with no variable of their own: `min_wal_size=512MB`, `max_wal_size=2GB`, `wal_buffers=16MB`, and `shm_size: 256mb` on the service itself.
+#### `FLUXER_POSTGRES_SHM_SIZE`
+
+Default `1gb`. The shared memory of the `postgres` container, used by parallel queries and parallel maintenance. Keep it well above `FLUXER_POSTGRES_MAINTENANCE_WORK_MEM`. A manual VACUUM sizes its shared memory from that budget, and fails with `could not resize shared memory segment` when it does not fit.
+
+The WAL settings `min_wal_size=512MB`, `max_wal_size=2GB` and `wal_buffers=16MB` are fixed on the command line, with no variable of their own.
 
 The bundled Valkey uses persistent storage. These settings control its memory limit and behaviour when full.
 

fluxer_docs/src/content/docs/operator/upgrading.mdx

@@ -182,6 +182,18 @@ When `api` starts, it writes the new address to the existing `default-server-1`
 
 Voice media is unaffected. It never went through the edge, and `7881/tcp` and `7882/udp` still reach the host directly.
 
+## Short-lived data expires on Postgres
+
+On the Postgres backend, data the schema keeps for a limited time expires the way it does on Cassandra, counted from its last write. After the upgrade, these are the differences most people notice.
+
+- The mentions inbox lists the last 7 days of mentions.
+- A device that has not opened Fluxer in 90 days gets no push notifications until Fluxer is opened on it again.
+- An account's ended sessions in the admin dashboard go back 30 days.
+
+Within minutes of the upgrade, `worker` starts deleting existing data that is past that lifetime. A rollback does not bring it back. [Restore a backup](#restore-a-backup) does. `worker` checks again once a day, so data written while a rollback was in place gets the same treatment after the next upgrade.
+
+Instances on Cassandra already behave this way.
+
 ## What the backup covers
 
 Each upgrade writes one record directory, named `record-` and a UTC stamp, under `backups` or wherever `--backup-dir` points. It holds: