fix(api): honour default TTLs and expire stale job ledger rows (#2887)

fluxer_api/pkgs/geoip/src/PostgresIpInfoKv.ts

@@ -11,6 +11,8 @@ interface PostgresIpInfoOptions {
 }
 
 const VALUE_SEPARATOR = '\u001f';
+export const IPINFO_CACHE_TTL_SECONDS = 14 * 24 * 60 * 60;
+export const IPINFO_REQUEST_AUDIT_TTL_SECONDS = 90 * 24 * 60 * 60;
 
 function getClient(options: PostgresIpInfoOptions): IPostgresClient | null {
 	return options.client ?? options.getClient?.() ?? null;
@@ -34,12 +36,9 @@ async function upsertKvRow(
 	partitionKey: string,
 	key: string,
 	row: Record<string, unknown>,
-	ttlSeconds?: number,
+	ttlSeconds: number,
 ): Promise<void> {
-	const expiresAt =
-		ttlSeconds != null && Number.isFinite(ttlSeconds) && ttlSeconds > 0
-			? new Date(Date.now() + ttlSeconds * 1000)
-			: null;
+	const expiresAt = new Date(Date.now() + ttlSeconds * 1000);
 	await client.query(
 		`INSERT INTO ${table(client)} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
 VALUES ($1, $2, $3, $4::jsonb, $5, now())
@@ -77,7 +76,14 @@ export function createPostgresIpInfoCache(options: PostgresIpInfoOptions): IpInf
 			try {
 				const client = getClient(options);
 				if (!client) return;
-				await upsertKvRow(client, 'ipinfo_cache', rowKey([key]), rowKey([key]), {cache_key: key, payload}, ttlSeconds);
+				await upsertKvRow(
+					client,
+					'ipinfo_cache',
+					rowKey([key]),
+					rowKey([key]),
+					{cache_key: key, payload},
+					ttlSeconds != null && Number.isFinite(ttlSeconds) && ttlSeconds > 0 ? ttlSeconds : IPINFO_CACHE_TTL_SECONDS,
+				);
 			} catch (error) {
 				options.onError?.(error, 'ipinfo_cache_set');
 			}
@@ -124,6 +130,7 @@ export function createPostgresIpInfoRequestAuditLogger(options: PostgresIpInfoOp
 						is_residential_proxy: event.isResidentialProxy,
 						metadata_json: serializeMetadata(event.metadata),
 					},
+					IPINFO_REQUEST_AUDIT_TTL_SECONDS,
 				);
 			} catch (error) {
 				options.onError?.(error, 'ipinfo_request_audit_record');

fluxer_api/pkgs/postgres/src/Client.ts

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import assert from 'node:assert/strict';
-import type {Pool, PoolClient, QueryResult, QueryResultRow} from 'pg';
+import type {Pool, PoolClient, PoolConfig, QueryResult, QueryResultRow} from 'pg';
 import pg from 'pg';
 
 const MAX_DIAGNOSTIC_FIELD_LENGTH = 128;
@@ -131,7 +131,7 @@ class PostgresClient implements IPostgresClient {
 	}
 
 	private async openPool(): Promise<void> {
-		const pool = new pg.Pool({
+		const poolConfig: PoolConfig & {scramMaxIterations: number} = {
 			connectionString: this.config.url || undefined,
 			host: this.config.url ? undefined : (this.config.host ?? '127.0.0.1'),
 			port: this.config.url ? undefined : (this.config.port ?? 5432),
@@ -140,7 +140,9 @@ class PostgresClient implements IPostgresClient {
 			password: this.config.url ? undefined : (this.config.password ?? 'fluxer'),
 			ssl: this.config.ssl ? {rejectUnauthorized: true, ca: normalizePem(this.config.sslCa)} : undefined,
 			max: this.config.maxConnections ?? 20,
-		});
+			scramMaxIterations: 0,
+		};
+		const pool = new pg.Pool(poolConfig);
 		this.observePoolConnections(pool);
 		try {
 			const client = await pool.connect();

fluxer_api/src/api/Tables.ts

@@ -341,6 +341,7 @@ import {
 	type UsersPendingDeletionRow,
 } from '@app/api/database/types/UserTypes';
 import {ATTACHMENT_DECAY_COLUMNS, type AttachmentDecayRow} from '@app/api/types/AttachmentDecayTypes';
+import {seconds} from 'itty-time';
 
 export const Users = defineTable<UserRow, 'user_id'>({
 	name: 'users',
@@ -499,16 +500,19 @@ export const GuildAuditLogs = defineTable<GuildAuditLogRow, 'guild_id' | 'log_id
 	name: 'guild_audit_logs_v2',
 	columns: GUILD_AUDIT_LOG_COLUMNS,
 	primaryKey: ['guild_id', 'log_id'],
+	defaultTtlSeconds: seconds('45 days'),
 });
 export const GuildAuditLogsByUser = defineTable<GuildAuditLogRow, 'guild_id' | 'user_id' | 'log_id'>({
 	name: 'guild_audit_logs_v2_by_user',
 	columns: GUILD_AUDIT_LOG_COLUMNS,
 	primaryKey: ['guild_id', 'user_id', 'log_id'],
+	defaultTtlSeconds: seconds('45 days'),
 });
 export const GuildAuditLogsByAction = defineTable<GuildAuditLogRow, 'guild_id' | 'action_type' | 'log_id'>({
 	name: 'guild_audit_logs_v2_by_action',
 	columns: GUILD_AUDIT_LOG_COLUMNS,
 	primaryKey: ['guild_id', 'action_type', 'log_id'],
+	defaultTtlSeconds: seconds('45 days'),
 });
 export const GuildAuditLogsByUserAction = defineTable<
 	GuildAuditLogRow,
@@ -517,6 +521,7 @@ export const GuildAuditLogsByUserAction = defineTable<
 	name: 'guild_audit_logs_v2_by_user_action',
 	columns: GUILD_AUDIT_LOG_COLUMNS,
 	primaryKey: ['guild_id', 'user_id', 'action_type', 'log_id'],
+	defaultTtlSeconds: seconds('45 days'),
 });
 export const GuildMembershipMetadata = defineTable<GuildMembershipMetadataRow, 'guild_id' | 'user_id'>({
 	name: 'guild_membership_metadata',
@@ -655,6 +660,7 @@ export const RecentMentions = defineTable<RecentMentionRow, 'user_id' | 'message
 	name: 'recent_mentions',
 	columns: RECENT_MENTION_COLUMNS,
 	primaryKey: ['user_id', 'message_id'],
+	defaultTtlSeconds: seconds('7 days'),
 });
 
 interface RecentMentionsByGuildRow {
@@ -678,6 +684,7 @@ export const RecentMentionsByGuild = defineTable<RecentMentionsByGuildRow, 'user
 	name: 'recent_mentions_by_guild',
 	columns: RECENT_MENTIONS_BY_GUILD_COLUMNS,
 	primaryKey: ['user_id', 'guild_id', 'message_id'],
+	defaultTtlSeconds: seconds('7 days'),
 });
 export const SavedMessages = defineTable<SavedMessageRow, 'user_id' | 'message_id'>({
 	name: 'saved_messages',
@@ -688,6 +695,7 @@ export const PushSubscriptions = defineTable<PushSubscriptionRow, 'user_id' | 's
 	name: 'push_subscriptions',
 	columns: PUSH_SUBSCRIPTION_COLUMNS,
 	primaryKey: ['user_id', 'subscription_id'],
+	defaultTtlSeconds: seconds('90 days'),
 });
 export const Payments = defineTable<PaymentRow, 'checkout_session_id'>({
 	name: 'payments',
@@ -854,11 +862,13 @@ export const EmailVerificationTokens = defineTable<EmailVerificationTokenRow, 't
 	name: 'email_verification_tokens',
 	columns: EMAIL_VERIFICATION_TOKEN_COLUMNS,
 	primaryKey: ['token_', 'user_id'],
+	defaultTtlSeconds: seconds('24 hours'),
 });
 export const PasswordResetTokens = defineTable<PasswordResetTokenRow, 'token_' | 'user_id'>({
 	name: 'password_reset_tokens',
 	columns: PASSWORD_RESET_TOKEN_COLUMNS,
 	primaryKey: ['token_', 'user_id'],
+	defaultTtlSeconds: seconds('24 hours'),
 });
 export const PasswordResetTokensByUserId = defineTable<
 	{
@@ -870,16 +880,19 @@ export const PasswordResetTokensByUserId = defineTable<
 	name: 'password_reset_tokens_by_user_id',
 	columns: ['user_id', 'token_'],
 	primaryKey: ['user_id', 'token_'],
+	defaultTtlSeconds: seconds('24 hours'),
 });
 export const EmailRevertTokens = defineTable<EmailRevertTokenRow, 'token_' | 'user_id'>({
 	name: 'email_revert_tokens',
 	columns: EMAIL_REVERT_TOKEN_COLUMNS,
 	primaryKey: ['token_', 'user_id'],
+	defaultTtlSeconds: seconds('48 hours'),
 });
 export const PhoneTokens = defineTable<PhoneTokenRow, 'token_'>({
 	name: 'phone_tokens',
 	columns: PHONE_TOKEN_COLUMNS,
 	primaryKey: ['token_'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const AuthSessions = defineTable<AuthSessionRow, 'session_id_hash'>({
 	name: 'auth_sessions',
@@ -901,11 +914,13 @@ export const AuthSessionTombstones = defineTable<AuthSessionTombstoneRow, 'user_
 	name: 'auth_session_tombstones',
 	columns: AUTH_SESSION_TOMBSTONE_COLUMNS,
 	primaryKey: ['user_id', 'session_id_hash'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const UserCountryHistory = defineTable<UserCountryHistoryRow, 'user_id' | 'country'>({
 	name: 'user_country_history',
 	columns: USER_COUNTRY_HISTORY_COLUMNS,
 	primaryKey: ['user_id', 'country'],
+	defaultTtlSeconds: seconds('365 days'),
 });
 export const MfaBackupCodes = defineTable<MfaBackupCodeRow, 'user_id' | 'code'>({
 	name: 'mfa_backup_codes',
@@ -932,6 +947,7 @@ export const IpAuthorizationTokens = defineTable<IpAuthorizationTokenRow, 'token
 	name: 'ip_authorization_tokens',
 	columns: IP_AUTHORIZATION_TOKEN_COLUMNS,
 	primaryKey: ['token_', 'user_id'],
+	defaultTtlSeconds: seconds('30 minutes'),
 });
 export const AuthorizedIps = defineTable<AuthorizedIpRow, 'user_id' | 'ip'>({
 	name: 'authorized_ips_v2',
@@ -1057,26 +1073,31 @@ export const OAuth2AuthorizationCodes = defineTable<OAuth2AuthorizationCodeRow,
 	name: 'oauth2_authorization_codes',
 	columns: OAUTH2_AUTHORIZATION_CODE_COLUMNS,
 	primaryKey: ['code'],
+	defaultTtlSeconds: seconds('10 minutes'),
 });
 export const OAuth2AccessTokens = defineTable<OAuth2AccessTokenRow, 'token_'>({
 	name: 'oauth2_access_tokens',
 	columns: OAUTH2_ACCESS_TOKEN_COLUMNS,
 	primaryKey: ['token_'],
+	defaultTtlSeconds: seconds('7 days'),
 });
 export const OAuth2AccessTokensByUser = defineTable<OAuth2AccessTokenByUserRow, 'user_id' | 'token_'>({
 	name: 'oauth2_access_tokens_by_user',
 	columns: OAUTH2_ACCESS_TOKENS_BY_USER_COLUMNS,
 	primaryKey: ['user_id', 'token_'],
+	defaultTtlSeconds: seconds('7 days'),
 });
 export const OAuth2RefreshTokens = defineTable<OAuth2RefreshTokenRow, 'token_'>({
 	name: 'oauth2_refresh_tokens',
 	columns: OAUTH2_REFRESH_TOKEN_COLUMNS,
 	primaryKey: ['token_'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const OAuth2RefreshTokensByUser = defineTable<OAuth2RefreshTokenByUserRow, 'user_id' | 'token_'>({
 	name: 'oauth2_refresh_tokens_by_user',
 	columns: OAUTH2_REFRESH_TOKENS_BY_USER_COLUMNS,
 	primaryKey: ['user_id', 'token_'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 
 interface WebhooksByChannelRow {
@@ -1117,12 +1138,14 @@ export const JobsById = defineTable<JobByIdRow, 'job_id'>({
 	name: 'jobs_by_id',
 	columns: JOB_BY_ID_COLUMNS,
 	primaryKey: ['job_id'],
+	defaultTtlSeconds: seconds('90 days'),
 });
 export const JobsByDayBucket = defineTable<JobByDayBucketRow, 'bucket_day' | 'created_at' | 'job_id'>({
 	name: 'jobs_by_day_bucket',
 	columns: JOB_BY_DAY_BUCKET_COLUMNS,
 	primaryKey: ['bucket_day', 'created_at', 'job_id'],
 	partitionKey: ['bucket_day'],
+	defaultTtlSeconds: seconds('90 days'),
 });
 export const JobsActive = defineTable<JobActiveRow, 'job_id'>({
 	name: 'jobs_active',
@@ -1133,11 +1156,13 @@ export const AttachmentUploadTracesByKey = defineTable<AttachmentUploadTraceByKe
 	name: 'attachment_upload_traces_by_key',
 	columns: ATTACHMENT_UPLOAD_TRACE_BY_KEY_COLUMNS,
 	primaryKey: ['upload_key'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const AttachmentUploadTracesByAttachment = defineTable<AttachmentUploadTraceByAttachmentRow, 'attachment_id'>({
 	name: 'attachment_upload_traces_by_attachment',
 	columns: ATTACHMENT_UPLOAD_TRACE_BY_ATTACHMENT_COLUMNS,
 	primaryKey: ['attachment_id'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const NcmecAttachmentSubmissions = defineTable<NcmecAttachmentSubmissionRow, 'attachment_id'>({
 	name: 'ncmec_attachment_submissions',
@@ -1154,6 +1179,7 @@ export const RegistrationEventsByIp = defineTable<RegistrationEventByIpRow, 'ip'
 	columns: REGISTRATION_EVENT_BY_IP_COLUMNS,
 	primaryKey: ['ip', 'created_at', 'user_id'],
 	partitionKey: ['ip'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const RegistrationEventsBySubnet = defineTable<
 	RegistrationEventBySubnetRow,
@@ -1164,6 +1190,7 @@ export const RegistrationEventsBySubnet = defineTable<
 	columns: REGISTRATION_EVENT_BY_SUBNET_COLUMNS,
 	primaryKey: ['subnet', 'created_at', 'user_id'],
 	partitionKey: ['subnet'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const RegistrationEventsByEmailDomain = defineTable<
 	RegistrationEventByEmailDomainRow,
@@ -1174,6 +1201,7 @@ export const RegistrationEventsByEmailDomain = defineTable<
 	columns: REGISTRATION_EVENT_BY_EMAIL_DOMAIN_COLUMNS,
 	primaryKey: ['email_domain', 'created_at', 'user_id'],
 	partitionKey: ['email_domain'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const RegistrationEventsByPlusAddressBase = defineTable<
 	RegistrationEventByPlusAddressBaseRow,
@@ -1184,6 +1212,7 @@ export const RegistrationEventsByPlusAddressBase = defineTable<
 	columns: REGISTRATION_EVENT_BY_PLUS_ADDRESS_BASE_COLUMNS,
 	primaryKey: ['plus_address_base', 'created_at', 'user_id'],
 	partitionKey: ['plus_address_base'],
+	defaultTtlSeconds: seconds('30 days'),
 });
 export const LatestRiskContextByUser = defineTable<LatestRiskContextByUserRow, 'user_id'>({
 	name: 'latest_risk_context_by_user',
@@ -1194,6 +1223,7 @@ export const SuspiciousIps = defineTable<SuspiciousIpRow, 'ip'>({
 	name: 'suspicious_ips',
 	columns: SUSPICIOUS_IP_COLUMNS,
 	primaryKey: ['ip'],
+	defaultTtlSeconds: seconds('180 days'),
 });
 export const RiskOutcomesByIp = defineTable<RiskOutcomeByIpRow, 'ip' | 'created_at' | 'user_id' | 'outcome_code', 'ip'>(
 	{
@@ -1201,6 +1231,7 @@ export const RiskOutcomesByIp = defineTable<RiskOutcomeByIpRow, 'ip' | 'created_
 		columns: RISK_OUTCOME_BY_IP_COLUMNS,
 		primaryKey: ['ip', 'created_at', 'user_id', 'outcome_code'],
 		partitionKey: ['ip'],
+		defaultTtlSeconds: seconds('180 days'),
 	},
 );
 export const RiskOutcomesBySubnet = defineTable<
@@ -1212,6 +1243,7 @@ export const RiskOutcomesBySubnet = defineTable<
 	columns: RISK_OUTCOME_BY_SUBNET_COLUMNS,
 	primaryKey: ['subnet', 'created_at', 'user_id', 'outcome_code'],
 	partitionKey: ['subnet'],
+	defaultTtlSeconds: seconds('180 days'),
 });
 export const RiskOutcomesByEmailDomain = defineTable<
 	RiskOutcomeByEmailDomainRow,
@@ -1222,6 +1254,7 @@ export const RiskOutcomesByEmailDomain = defineTable<
 	columns: RISK_OUTCOME_BY_EMAIL_DOMAIN_COLUMNS,
 	primaryKey: ['email_domain', 'created_at', 'user_id', 'outcome_code'],
 	partitionKey: ['email_domain'],
+	defaultTtlSeconds: seconds('180 days'),
 });
 export const RiskOutcomesByAsn = defineTable<
 	RiskOutcomeByAsnRow,
@@ -1232,6 +1265,7 @@ export const RiskOutcomesByAsn = defineTable<
 	columns: RISK_OUTCOME_BY_ASN_COLUMNS,
 	primaryKey: ['asn', 'created_at', 'user_id', 'outcome_code'],
 	partitionKey: ['asn'],
+	defaultTtlSeconds: seconds('180 days'),
 });
 export const RiskAssessments = defineTable<RiskAssessmentRow, 'assessment_id'>({
 	name: 'risk_assessments',
@@ -1248,6 +1282,7 @@ export const InboundSmsChallenges = defineTable<InboundSmsChallengeRow, 'challen
 	name: 'inbound_sms_challenges',
 	columns: INBOUND_SMS_CHALLENGE_COLUMNS,
 	primaryKey: ['challenge_code'],
+	defaultTtlSeconds: seconds('15 minutes'),
 });
 export const InboundSmsChallengesByUser = defineTable<
 	InboundSmsChallengeByUserRow,
@@ -1258,16 +1293,19 @@ export const InboundSmsChallengesByUser = defineTable<
 	columns: INBOUND_SMS_CHALLENGE_BY_USER_COLUMNS,
 	primaryKey: ['user_id', 'created_at'],
 	partitionKey: ['user_id'],
+	defaultTtlSeconds: seconds('15 minutes'),
 });
 export const PhoneLookupCache = defineTable<PhoneLookupCacheRow, 'phone'>({
 	name: 'phone_lookup_cache',
 	columns: PHONE_LOOKUP_CACHE_COLUMNS,
 	primaryKey: ['phone'],
+	defaultTtlSeconds: seconds('7 days'),
 });
 export const PhoneVerificationAttempts = defineTable<PhoneVerificationAttemptRow, 'attempt_id'>({
 	name: 'phone_verification_attempts',
 	columns: PHONE_VERIFICATION_ATTEMPT_COLUMNS,
 	primaryKey: ['attempt_id'],
+	defaultTtlSeconds: seconds('90 days'),
 });
 export const BillingCustomers = defineTable<BillingCustomerRow, 'provider_id'>({
 	name: 'billing_customers',

fluxer_api/src/api/admin/AdminRepository.ts

@@ -2,7 +2,15 @@
 
 import type {AdminAuditLog, BannedIpEntry, BannedIpKind, IAdminRepository} from '@app/api/admin/IAdminRepository';
 import {createUserID} from '@app/api/BrandedTypes';
-import {deleteOneOrMany, fetchMany, fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
+import {Config} from '@app/api/Config';
+import {ContentBlocklistCategory} from '@app/api/constants/ContentModeration';
+import {
+	deleteOneOrMany,
+	executeConditional,
+	fetchMany,
+	fetchOne,
+	upsertOne,
+} from '@app/api/database/CassandraQueryExecution';
 import type {
 	AdminAuditLogRow,
 	BannedAvatarHashRow,
@@ -282,6 +290,7 @@ export class AdminRepository implements IAdminRepository {
 	}
 
 	async isEmailDomainDisposable(domain: string): Promise<boolean> {
+		if (!Config.blocklistFeeds.enabled) return false;
 		const domainLower = domain.toLowerCase();
 		if (isAccountPolicyContactDomainReputationExempt(domainLower)) return false;
 		const result = await fetchOne<{
@@ -395,6 +404,15 @@ export class AdminRepository implements IAdminRepository {
 		await deleteOneOrMany(BannedFileShas.deleteByPk({sha256_hex: sha256Hex.toLowerCase()}));
 	}
 
+	async unbanFeedFileSha(sha256Hex: string): Promise<boolean> {
+		return executeConditional(
+			BannedFileShas.conditionalDeleteByPk(
+				{sha256_hex: sha256Hex.toLowerCase()},
+				{added_by: null, category: ContentBlocklistCategory.MALWARE_BAZAAR},
+			),
+		);
+	}
+
 	async loadAllBannedFileShas(): Promise<Array<BannedFileShaRow>> {
 		return fetchMany<BannedFileShaRow>(LOAD_ALL_BANNED_FILE_SHAS_QUERY.bind({}));
 	}

fluxer_api/src/api/admin/IAdminRepository.ts

@@ -109,6 +109,8 @@ export abstract class IAdminRepository {
 
 	abstract unbanFileSha(sha256Hex: string): Promise<void>;
 
+	abstract unbanFeedFileSha(sha256Hex: string): Promise<boolean>;
+
 	abstract loadAllBannedFileShas(): Promise<Array<BannedFileShaRow>>;
 
 	abstract isAvatarHashBanned(hashShort: string): Promise<boolean>;

fluxer_api/src/api/constants/ContentModeration.ts

@@ -1,5 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import type {BannedFileShaRow} from '@app/api/database/types/AdminArchiveTypes';
+
 export const BANNED_URLS_REFRESH_CHANNEL = 'banned_urls_refresh';
 export const BANNED_URL_DOMAINS_REFRESH_CHANNEL = 'banned_url_domains_refresh';
 export const BANNED_FILE_SHAS_REFRESH_CHANNEL = 'banned_file_shas_refresh';
@@ -23,3 +25,7 @@ export const ContentBlocklistCategory = {
 	GIFCT: 'gifct',
 	STOP_NCII: 'stop_ncii',
 } as const;
+
+export function isBlocklistFeedFileSha(row: Pick<BannedFileShaRow, 'category' | 'added_by'>): boolean {
+	return row.added_by == null && row.category === ContentBlocklistCategory.MALWARE_BAZAAR;
+}

fluxer_api/src/api/database/CassandraDefaultTtlParity.test.ts

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import fs from 'node:fs';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {DEFAULT_TTL_TABLES} from '@app/api/database/PostgresKvDefaultTtlExpiry';
+import * as DonationTables from '@app/api/donation/DonationTables';
+import * as Tables from '@app/api/Tables';
+import {IPINFO_CACHE_TTL_SECONDS, IPINFO_REQUEST_AUDIT_TTL_SECONDS} from '@pkgs/geoip/src/PostgresIpInfoKv';
+import {describe, expect, it} from 'vitest';
+
+const THIS_DIR = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(THIS_DIR, '../../../..');
+
+interface SchemaTable {
+	name: string;
+	options: string;
+}
+
+const SCHEMA = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'tools/dev/cassandra_target_schema.json'), 'utf8')) as {
+	tables: Array<SchemaTable>;
+};
+
+const SCHEMA_DEFAULTS = new Map<string, number>(
+	SCHEMA.tables.flatMap((table): Array<[string, number]> => {
+		const match = /default_time_to_live = (\d+)/.exec(table.options);
+		return match ? [[table.name, Number(match[1])]] : [];
+	}),
+);
+
+const DSL_TABLES = [...Object.values(Tables), ...Object.values(DonationTables)];
+const DSL_NAMES = new Set<string>(DSL_TABLES.map((table) => table.name));
+
+const NON_DSL_DEFAULTS: Record<string, number | null> = {
+	ipinfo_cache: IPINFO_CACHE_TTL_SECONDS,
+	ipinfo_requests_by_hour: IPINFO_REQUEST_AUDIT_TTL_SECONDS,
+	billing_webhook_events: null,
+	forensic_identifier_by_key_day: null,
+	forensic_identifier_by_request: null,
+	forensic_request_meta_by_actor_day: null,
+	forensic_request_meta_by_id: null,
+	forensic_request_meta_by_route_day_shard: null,
+	forensic_resource_exposure_by_request: null,
+	forensic_resource_exposure_by_route_day_shard: null,
+	forensic_resource_exposure_by_subject_day: null,
+};
+
+const OWN_EXPIRY_PASS = new Set(['jobs_by_id', 'jobs_by_day_bucket']);
+
+function schemaDefault(name: string): number {
+	return SCHEMA_DEFAULTS.get(name) ?? 0;
+}
+
+function byName(left: {name: string}, right: {name: string}): number {
+	return left.name.localeCompare(right.name);
+}
+
+describe('Cassandra default TTL parity', () => {
+	it('declares every Cassandra default TTL on the matching table', () => {
+		const mismatches = DSL_TABLES.flatMap((table) => {
+			const declared = table.defaultTtlSeconds ?? 0;
+			return declared === schemaDefault(table.name)
+				? []
+				: [{table: table.name, declared, schema: schemaDefault(table.name)}];
+		});
+		expect(mismatches).toEqual([]);
+	});
+
+	it('declares a writer or no writer for every other table with a default', () => {
+		const undeclared = [...SCHEMA_DEFAULTS]
+			.filter(([name, ttl]) => ttl > 0 && !DSL_NAMES.has(name) && !Object.hasOwn(NON_DSL_DEFAULTS, name))
+			.map(([name]) => name);
+		expect(undeclared).toEqual([]);
+		const stale = Object.keys(NON_DSL_DEFAULTS).filter((name) => schemaDefault(name) === 0 || DSL_NAMES.has(name));
+		expect(stale).toEqual([]);
+		const mismatched = Object.entries(NON_DSL_DEFAULTS)
+			.filter(([name, ttl]) => ttl !== null && ttl !== schemaDefault(name))
+			.map(([name]) => name);
+		expect(mismatched).toEqual([]);
+	});
+
+	it('the Postgres expiry pass covers every table with a default except the job ledger', () => {
+		const expected = [...SCHEMA_DEFAULTS]
+			.filter(([name, ttl]) => ttl > 0 && NON_DSL_DEFAULTS[name] !== null && !OWN_EXPIRY_PASS.has(name))
+			.map(([name, ttl]) => ({name, defaultTtlSeconds: ttl}))
+			.sort(byName);
+		expect([...DEFAULT_TTL_TABLES].sort(byName)).toEqual(expected);
+	});
+});

fluxer_api/src/api/database/CassandraMetaRegistry.ts

@@ -12,6 +12,7 @@ interface TableMetadata {
 	columns: ReadonlyArray<string>;
 	primaryKey: ReadonlyArray<string>;
 	partitionKey: ReadonlyArray<string>;
+	defaultTtlSeconds?: number;
 }
 
 const kvMetaRegistry = new Map<string, KvQueryMeta<Record<string, unknown>>>();
@@ -24,6 +25,7 @@ export function registerTableSpec<Row extends object>(tableSpec: KvTableSpec<Row
 		columns: tableSpec.columns as ReadonlyArray<string>,
 		primaryKey: tableSpec.primaryKey as ReadonlyArray<string>,
 		partitionKey: tableSpec.partitionKey as ReadonlyArray<string>,
+		defaultTtlSeconds: tableSpec.defaultTtlSeconds,
 	};
 	tableRegistry.set(tableSpec.name, metadata);
 }

fluxer_api/src/api/database/CassandraTableDsl.test.ts

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import {getTableMetadata} from '@app/api/database/CassandraMetaRegistry';
 import {defineTable} from '@app/api/database/CassandraTableDsl';
 import {Db, type PreparedQuery} from '@app/api/database/CassandraTypes';
 import {describe, expect, it} from 'vitest';
@@ -76,3 +77,41 @@ describe('CassandraTableDsl select templates', () => {
 		expect(longQuery.cql).not.toContain('LIMIT 20');
 	});
 });
+
+describe('CassandraTableDsl default TTL', () => {
+	it('keeps the CQL of a table with a default TTL free of USING TTL', () => {
+		const DefaultTtlRows = defineTable<TtlHelperTestRow, 'id'>({
+			name: 'default_ttl_dsl_rows',
+			columns: ['id', 'value'],
+			primaryKey: ['id'],
+			defaultTtlSeconds: 600,
+		});
+		expect(DefaultTtlRows.defaultTtlSeconds).toBe(600);
+		const queries = [
+			DefaultTtlRows.insert({id: 'insert', value: 'a'}),
+			DefaultTtlRows.upsertAll({id: 'upsert', value: 'b'}),
+			DefaultTtlRows.patchByPk({id: 'patch'}, {value: Db.set('c')}),
+		];
+		for (const query of queries) {
+			expect(query.cql).not.toContain('USING TTL');
+			expect(query.kvMeta?.table.defaultTtlSeconds).toBe(600);
+		}
+		expect(getTableMetadata('default_ttl_dsl_rows')?.defaultTtlSeconds).toBe(600);
+		expect(TtlHelperTestRows.defaultTtlSeconds).toBeUndefined();
+		expect(getTableMetadata('ttl_helper_test_rows')?.defaultTtlSeconds).toBeUndefined();
+	});
+
+	it('rejects a default TTL of zero, a fraction or past the maximum', () => {
+		for (const defaultTtlSeconds of [0, 1.5, 630_720_001]) {
+			expect(() =>
+				defineTable<TtlHelperTestRow, 'id'>({
+					name: 'default_ttl_dsl_rejected_rows',
+					columns: ['id', 'value'],
+					primaryKey: ['id'],
+					defaultTtlSeconds,
+				}),
+			).toThrow();
+		}
+		expect(getTableMetadata('default_ttl_dsl_rejected_rows')).toBeUndefined();
+	});
+});

fluxer_api/src/api/database/CassandraTableDsl.ts

@@ -83,6 +83,7 @@ export function defineTable<Row extends object, PK extends ColumnName<Row>, Part
 	columns: ReadonlyArray<ColumnName<Row>>;
 	primaryKey: ReadonlyArray<PK>;
 	partitionKey?: ReadonlyArray<PartKey>;
+	defaultTtlSeconds?: number;
 }): Table<Row, PK, PartKey> {
 	const columns = [...def.columns];
 	const pk = [...def.primaryKey];
@@ -91,11 +92,15 @@ export function defineTable<Row extends object, PK extends ColumnName<Row>, Part
 	for (const c of columns) assertCqlIdentifier(c as string);
 	for (const k of pk) assertCqlIdentifier(k as string);
 	for (const k of partitionKey) assertCqlIdentifier(k as string);
+	if (def.defaultTtlSeconds !== undefined && validateTtlSeconds(def.defaultTtlSeconds) === 0) {
+		throw new Error(`Table "${def.name}" needs a positive default TTL`);
+	}
 	const tableSpec: KvTableSpec<Row> = {
 		name: def.name,
 		columns,
 		primaryKey: pk as ReadonlyArray<ColumnName<Row>>,
 		partitionKey: partitionKey as ReadonlyArray<ColumnName<Row>>,
+		defaultTtlSeconds: def.defaultTtlSeconds,
 	};
 	registerTableSpec(tableSpec);
 	const nonPkColumns = columns.filter((c) => !pk.includes(c as PK)) as Array<Exclude<ColumnName<Row>, PK>>;
@@ -685,6 +690,7 @@ WHERE ${pk.map((k) => `${k} = :${k}`).join(' AND ')};
 		columns: def.columns,
 		primaryKey: def.primaryKey,
 		partitionKey: partitionKey,
+		defaultTtlSeconds: def.defaultTtlSeconds,
 		selectCql,
 		select,
 		updateAllCql() {

fluxer_api/src/api/database/CassandraTypes.ts

@@ -56,6 +56,7 @@ export interface KvTableSpec<Row extends object = Record<string, unknown>> {
 	columns: ReadonlyArray<ColumnName<Row>>;
 	primaryKey: ReadonlyArray<ColumnName<Row>>;
 	partitionKey: ReadonlyArray<ColumnName<Row>>;
+	defaultTtlSeconds?: number;
 }
 
 export interface KvColumnParam<Row extends object = Record<string, unknown>> {
@@ -190,6 +191,7 @@ export interface Table<Row extends object, PK extends ColumnName<Row>, PartKey e
 	columns: ReadonlyArray<ColumnName<Row>>;
 	primaryKey: ReadonlyArray<PK>;
 	partitionKey: ReadonlyArray<PartKey>;
+	defaultTtlSeconds: number | undefined;
 	selectCql(opts?: {
 		columns?: ReadonlyArray<ColumnName<Row>>;
 		where?: WhereExpr<Row> | ReadonlyArray<WhereExpr<Row>>;

fluxer_api/src/api/database/PostgresKvDefaultTtl.test.ts

@@ -0,0 +1,490 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {defineTable} from '@app/api/database/CassandraTableDsl';
+import {Db} from '@app/api/database/CassandraTypes';
+import {
+	DEFAULT_TTL_EXPIRY_RESUME,
+	DEFAULT_TTL_TABLES,
+	expireLegacyDefaultTtlRows,
+} from '@app/api/database/PostgresKvDefaultTtlExpiry';
+import {
+	ensurePostgresKvSchema,
+	PostgresKvQueryExecutor,
+	pruneExpiredPostgresKvRows,
+} from '@app/api/database/PostgresKvQueryExecutor';
+import {startDockerContainer} from '@app/api/test/DockerTestContainer';
+import {
+	getDefaultPostgresClient,
+	type IPostgresClient,
+	initPostgres,
+	shutdownPostgres,
+} from '@pkgs/postgres/src/Client';
+import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
+
+const KV_TABLE = 'kv_default_ttl';
+const CONTAINER = `fluxer-kvttl-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+const DEFAULT_TTL_SECONDS = 600;
+
+interface ProbeRow {
+	id: string;
+	value: string | null;
+	note: string | null;
+}
+
+interface OwnedProbeRow {
+	owner: string;
+	id: string;
+	value: string | null;
+}
+
+const DefaultTtlProbe = defineTable<ProbeRow, 'id'>({
+	name: 'default_ttl_probe',
+	columns: ['id', 'value', 'note'],
+	primaryKey: ['id'],
+	defaultTtlSeconds: DEFAULT_TTL_SECONDS,
+});
+
+const DefaultTtlProbeRows = defineTable<OwnedProbeRow, 'owner' | 'id', 'owner'>({
+	name: 'default_ttl_probe_rows',
+	columns: ['owner', 'id', 'value'],
+	primaryKey: ['owner', 'id'],
+	partitionKey: ['owner'],
+	defaultTtlSeconds: DEFAULT_TTL_SECONDS,
+});
+
+const NoTtlProbe = defineTable<ProbeRow, 'id'>({
+	name: 'no_ttl_probe',
+	columns: ['id', 'value', 'note'],
+	primaryKey: ['id'],
+});
+
+async function sleep(ms: number): Promise<void> {
+	await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function freePort(): Promise<number> {
+	return new Promise((resolve, reject) => {
+		const server = createServer();
+		server.on('error', reject);
+		server.listen(0, '127.0.0.1', () => {
+			const address = server.address();
+			if (typeof address === 'string' || address === null) {
+				reject(new Error('no port'));
+				return;
+			}
+			const port = address.port;
+			server.close(() => resolve(port));
+		});
+	});
+}
+
+function expectExpiresIn(value: Date | number | null, ttlSeconds: number): void {
+	expect(value).toBeInstanceOf(Date);
+	const remainingSeconds = ((value as Date).getTime() - Date.now()) / 1000;
+	expect(remainingSeconds).toBeGreaterThan(ttlSeconds - 60);
+	expect(remainingSeconds).toBeLessThanOrEqual(ttlSeconds);
+}
+
+describe.skipIf(!dockerAvailable)('Postgres KV default TTL', () => {
+	let raw: IPostgresClient;
+	let executor: PostgresKvQueryExecutor;
+
+	async function stored(table: string, id: string): Promise<{expires_at: Date | number | null; row_data: object}> {
+		const result = await raw.query<{expires_at: Date | number | null; row_data: object}>(
+			`SELECT expires_at, row_data FROM ${KV_TABLE} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
+			[table, id],
+		);
+		expect(result.rows).toHaveLength(1);
+		return result.rows[0]!;
+	}
+
+	async function expiresAt(table: string, id: string): Promise<Date | number | null> {
+		return (await stored(table, id)).expires_at;
+	}
+
+	async function neverExpires(table: string, id: string): Promise<boolean> {
+		const result = await raw.query<{forever: boolean}>(
+			`SELECT expires_at = 'infinity'::timestamptz AS forever FROM ${KV_TABLE} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
+			[table, id],
+		);
+		return result.rows[0]?.forever === true;
+	}
+
+	async function setExpiry(table: string, id: string, expression: string): Promise<void> {
+		await raw.query(
+			`UPDATE ${KV_TABLE} SET expires_at = ${expression} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
+			[table, id],
+		);
+	}
+
+	async function seed(table: string, key: string, age: string, expires: Date | string | null = null): Promise<string> {
+		const result = await raw.query<{updated_at: string}>(
+			`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
+VALUES ($1, $2, $2, '{}'::jsonb, $3::timestamptz, now() - $4::interval)
+RETURNING updated_at::text`,
+			[table, key, expires, age],
+		);
+		return result.rows[0]!.updated_at;
+	}
+
+	async function remaining(): Promise<Array<{table_name: string; row_key: string}>> {
+		const result = await raw.query<{table_name: string; row_key: string}>(
+			`SELECT table_name, row_key FROM ${KV_TABLE} WHERE table_name <> '__fluxer_schema_migrations' ORDER BY table_name, row_key`,
+		);
+		return result.rows;
+	}
+
+	async function ageMarker(): Promise<void> {
+		await raw.query(
+			`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('applied_at', now() - interval '2 days') WHERE table_name = '__fluxer_schema_migrations' AND row_key = 'default_ttl_expiry_v1'`,
+		);
+	}
+
+	async function resumePoint(): Promise<object | null> {
+		const result = await raw.query<{row_data: object}>(
+			`SELECT row_data FROM ${KV_TABLE} WHERE table_name = '__fluxer_schema_migrations' AND row_key = $1`,
+			[DEFAULT_TTL_EXPIRY_RESUME],
+		);
+		return result.rows[0]?.row_data ?? null;
+	}
+
+	async function markerCount(): Promise<number> {
+		const result = await raw.query<{n: number}>(
+			`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = '__fluxer_schema_migrations' AND row_key = 'default_ttl_expiry_v1'`,
+		);
+		return result.rows[0]!.n;
+	}
+
+	beforeAll(async () => {
+		const port = await freePort();
+		startDockerContainer([
+			'run',
+			'-d',
+			'--name',
+			CONTAINER,
+			'-e',
+			'POSTGRES_USER=fluxer',
+			'-e',
+			'POSTGRES_PASSWORD=fluxer',
+			'-e',
+			'POSTGRES_DB=fluxer',
+			'-p',
+			`127.0.0.1:${port}:5432`,
+			'postgres:16-alpine',
+			'-c',
+			'fsync=off',
+		]);
+		let ready = false;
+		for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+			await sleep(500);
+			const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
+				stdio: 'ignore',
+			});
+			if (probe.status !== 0) continue;
+			try {
+				await initPostgres({
+					url: `postgres://fluxer:[email protected]:${port}/fluxer`,
+					maxConnections: 4,
+					kvTable: KV_TABLE,
+				});
+				await getDefaultPostgresClient().query('SELECT 1');
+				ready = true;
+			} catch {
+				await shutdownPostgres().catch(() => {});
+			}
+		}
+		if (!ready) throw new Error('postgres never came up');
+		raw = getDefaultPostgresClient();
+		await ensurePostgresKvSchema(raw);
+		executor = new PostgresKvQueryExecutor(raw);
+	}, 900_000);
+
+	beforeEach(async () => {
+		await raw.query(`DELETE FROM ${KV_TABLE}`);
+	});
+
+	afterAll(async () => {
+		await shutdownPostgres().catch(() => {});
+		spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+	});
+
+	it('gives every full-row write without a TTL the table default', async () => {
+		await executor.executeQuery(DefaultTtlProbe.insert({id: 'insert', value: 'a', note: null}));
+		await executor.executeQuery(DefaultTtlProbe.upsertAll({id: 'upsert', value: 'b', note: 'n'}));
+		expect(
+			await executor.executeQuery(DefaultTtlProbe.insertIfNotExists({id: 'claimed', value: 'c', note: null})),
+		).toEqual([{'[applied]': true}]);
+		expect(
+			await executor.executeQuery(
+				DefaultTtlProbeRows.conditionalBatch([{action: 'insert', row: {owner: 'o', id: 'batched', value: 'd'}}]),
+			),
+		).toEqual([{'[applied]': true}]);
+
+		for (const id of ['insert', 'upsert', 'claimed']) {
+			expectExpiresIn(await expiresAt('default_ttl_probe', id), DEFAULT_TTL_SECONDS);
+		}
+		expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'batched'), DEFAULT_TTL_SECONDS);
+	});
+
+	it('keeps an explicit TTL ahead of the default', async () => {
+		await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'short', value: 'a', note: null}, 60));
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'short'), 60);
+
+		await executor.executeQuery(DefaultTtlProbe.insert({id: 'patched', value: 'a', note: null}));
+		await executor.executeQuery(DefaultTtlProbe.patchByPkWithTtl({id: 'patched'}, {value: Db.set('b')}, 60));
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'patched'), 60);
+	});
+
+	it('keeps an explicit TTL of zero as no expiry', async () => {
+		await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'forever', value: 'a', note: null}, 0));
+		expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
+		expect(
+			await executor.executeQuery(
+				DefaultTtlProbe.select({where: DefaultTtlProbe.where.eq('id')}).bind({id: 'forever'}),
+			),
+		).toEqual([{id: 'forever', value: 'a', note: null}]);
+
+		await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'forever'}, {note: Db.set('patched')}));
+		expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
+
+		await pruneExpiredPostgresKvRows(raw);
+		expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
+	});
+
+	it('raises a patched row to the default but never lowers it', async () => {
+		await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'longer', value: 'a', note: null}, 3600));
+		await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'longer'}, {note: Db.set('patched')}));
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'longer'), 3600);
+
+		await executor.executeQuery(DefaultTtlProbe.insert({id: 'soon', value: 'a', note: null}));
+		await setExpiry('default_ttl_probe', 'soon', "now() + interval '5 seconds'");
+		await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'soon'}, {note: Db.set('patched')}));
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'soon'), DEFAULT_TTL_SECONDS);
+
+		await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'missing'}, {note: Db.set('created')}));
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'missing'), DEFAULT_TTL_SECONDS);
+
+		await executor.executeQuery(DefaultTtlProbe.insert({id: 'unset', value: 'a', note: null}));
+		await setExpiry('default_ttl_probe', 'unset', 'NULL');
+		await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'unset'}, {note: Db.set('patched')}));
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'unset'), DEFAULT_TTL_SECONDS);
+
+		await executor.executeQuery(DefaultTtlProbe.insert({id: 'expired', value: 'a', note: null}));
+		await setExpiry('default_ttl_probe', 'expired', "now() - interval '1 second'");
+		await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'expired'}, {note: Db.set('patched')}));
+		const revived = await stored('default_ttl_probe', 'expired');
+		expect(revived.row_data).toEqual({id: 'expired', note: 'patched'});
+		expectExpiresIn(revived.expires_at, DEFAULT_TTL_SECONDS);
+	});
+
+	it('raises conditional patches the same way', async () => {
+		await executor.executeQuery(DefaultTtlProbe.insert({id: 'soon', value: 'a', note: null}));
+		await setExpiry('default_ttl_probe', 'soon', "now() + interval '5 seconds'");
+		expect(
+			await executor.executeQuery(
+				DefaultTtlProbe.conditionalPatchByPk({id: 'soon'}, {note: Db.set('patched')}, {value: 'a'}),
+			),
+		).toEqual([{'[applied]': true}]);
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'soon'), DEFAULT_TTL_SECONDS);
+
+		await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'longer', value: 'a', note: null}, 3600));
+		expect(
+			await executor.executeQuery(
+				DefaultTtlProbe.conditionalPatchByPk({id: 'longer'}, {note: Db.set('patched')}, {value: 'a'}),
+			),
+		).toEqual([{'[applied]': true}]);
+		expectExpiresIn(await expiresAt('default_ttl_probe', 'longer'), 3600);
+
+		await executor.executeQuery(DefaultTtlProbeRows.insert({owner: 'o', id: 'existing', value: 'old'}));
+		await setExpiry('default_ttl_probe_rows', 'existing', 'NULL');
+		expect(
+			await executor.executeQuery(
+				DefaultTtlProbeRows.conditionalBatch([
+					{action: 'insert', row: {owner: 'o', id: 'added', value: 'new'}},
+					{
+						action: 'patch',
+						pk: {owner: 'o', id: 'existing'},
+						patch: {value: Db.set('updated')},
+						expected: {value: 'old'},
+					},
+				]),
+			),
+		).toEqual([{'[applied]': true}]);
+		expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'added'), DEFAULT_TTL_SECONDS);
+		expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'existing'), DEFAULT_TTL_SECONDS);
+	});
+
+	it('leaves tables without a default untouched', async () => {
+		await executor.executeQuery(NoTtlProbe.insert({id: 'plain', value: 'a', note: null}));
+		expect(await expiresAt('no_ttl_probe', 'plain')).toBeNull();
+		await executor.executeQuery(NoTtlProbe.patchByPk({id: 'plain'}, {note: Db.set('patched')}));
+		expect(await expiresAt('no_ttl_probe', 'plain')).toBeNull();
+		await executor.executeQuery(NoTtlProbe.insertWithTtl({id: 'zero', value: 'a', note: null}, 0));
+		expect(await expiresAt('no_ttl_probe', 'zero')).toBeNull();
+	});
+
+	it('gives rows an older image wrote the expiry of their last write and deletes the ones past it', async () => {
+		const mentionWrittenAt = await seed('recent_mentions', 'rm-day', '1 day');
+		await seed('recent_mentions', 'rm-week', '8 days');
+		await seed('attachment_upload_traces_by_key', 'at-31', '31 days');
+		await seed('attachment_upload_traces_by_key', 'at-29', '29 days');
+		await seed('phone_lookup_cache', 'pl-8', '8 days');
+		await seed('donor_magic_link_tokens', 'dm-hour', '1 hour');
+		await seed('ipinfo_requests_by_hour', 'ip-day', '1 day');
+		await seed('jobs_by_id', 'job', '100 days');
+		await seed('users', 'user', '100 days');
+		await seed('recent_mentions', 'rm-forever', '1 day', 'infinity');
+		await seed('recent_mentions', 'rm-hour', '30 days', new Date(Date.now() + 3_600_000));
+
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 4,
+			expiring: 3,
+			complete: true,
+		});
+		expect(await remaining()).toEqual([
+			{table_name: 'attachment_upload_traces_by_key', row_key: 'at-29'},
+			{table_name: 'ipinfo_requests_by_hour', row_key: 'ip-day'},
+			{table_name: 'jobs_by_id', row_key: 'job'},
+			{table_name: 'recent_mentions', row_key: 'rm-day'},
+			{table_name: 'recent_mentions', row_key: 'rm-forever'},
+			{table_name: 'recent_mentions', row_key: 'rm-hour'},
+			{table_name: 'users', row_key: 'user'},
+		]);
+
+		const exact = await raw.query<{row_key: string; exact: boolean; unchanged: boolean | null}>(
+			`SELECT row_key,
+	expires_at = updated_at + CASE table_name WHEN 'recent_mentions' THEN interval '7 days' WHEN 'attachment_upload_traces_by_key' THEN interval '30 days' ELSE interval '90 days' END AS exact,
+	CASE WHEN row_key = 'rm-day' THEN updated_at = $1::timestamptz END AS unchanged
+FROM ${KV_TABLE}
+WHERE row_key IN ('rm-day', 'at-29', 'ip-day')
+ORDER BY row_key`,
+			[mentionWrittenAt],
+		);
+		expect(exact.rows).toEqual([
+			{row_key: 'at-29', exact: true, unchanged: null},
+			{row_key: 'ip-day', exact: true, unchanged: null},
+			{row_key: 'rm-day', exact: true, unchanged: true},
+		]);
+		const untouched = await raw.query<{row_key: string; state: string}>(
+			`SELECT row_key, CASE WHEN expires_at IS NULL THEN 'unset' WHEN expires_at = 'infinity' THEN 'forever' ELSE 'set' END AS state
+FROM ${KV_TABLE}
+WHERE row_key IN ('job', 'user', 'rm-forever', 'rm-hour')
+ORDER BY row_key`,
+		);
+		expect(untouched.rows).toEqual([
+			{row_key: 'job', state: 'unset'},
+			{row_key: 'rm-forever', state: 'forever'},
+			{row_key: 'rm-hour', state: 'set'},
+			{row_key: 'user', state: 'unset'},
+		]);
+
+		expect(await markerCount()).toBe(0);
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await markerCount()).toBe(1);
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
+	});
+
+	it('checks again a day after a clean pass', async () => {
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
+
+		await seed('recent_mentions', 'rm-rolled-back', '1 day');
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
+		const before = await raw.query(`SELECT expires_at FROM ${KV_TABLE} WHERE row_key = 'rm-rolled-back'`);
+		expect(before.rows).toEqual([{expires_at: null}]);
+
+		await ageMarker();
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 1,
+			complete: true,
+		});
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
+	});
+
+	it('pages through more rows than one page holds and stops at its deadline', async () => {
+		await raw.query(
+			`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data, updated_at)
+SELECT 'recent_mentions', 'rm-' || lpad(g::text, 5, '0'), 'rm-' || lpad(g::text, 5, '0'), '{}'::jsonb, now() - interval '1 day'
+FROM generate_series(1, 2300) g`,
+		);
+
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() - 1)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: false,
+		});
+		expect(await markerCount()).toBe(0);
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 2300,
+			complete: true,
+		});
+		const unset = await raw.query<{n: number}>(
+			`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = 'recent_mentions' AND expires_at IS NULL`,
+		);
+		expect(unset.rows[0]).toEqual({n: 0});
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
+	});
+
+	it('saves where a run stopped and starts the next run there', async () => {
+		const first = DEFAULT_TTL_TABLES[0]!.name;
+		const last = DEFAULT_TTL_TABLES.at(-1)!.name;
+		await seed(first, 'a', '1 hour');
+		await seed(first, 'z', '1 hour');
+		await seed(last, 'k', '1 hour');
+
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() - 1)).toEqual({deleted: 0, expiring: 0, complete: false});
+		expect(await resumePoint()).toEqual({table: first, row_key: '', unset: 0});
+
+		await raw.query(
+			`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('table', $1::text, 'row_key', 'm', 'unset', 0) WHERE table_name = '__fluxer_schema_migrations' AND row_key = $2`,
+			[first, DEFAULT_TTL_EXPIRY_RESUME],
+		);
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 2,
+			complete: true,
+		});
+		const untouched = await raw.query<{expires_at: Date | null}>(
+			`SELECT expires_at FROM ${KV_TABLE} WHERE table_name = $1 AND row_key = 'a'`,
+			[first],
+		);
+		expect(untouched.rows).toEqual([{expires_at: null}]);
+		expect(await resumePoint()).toBeNull();
+		expect(await markerCount()).toBe(0);
+
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 1,
+			complete: true,
+		});
+		expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await markerCount()).toBe(1);
+	});
+});

fluxer_api/src/api/database/PostgresKvDefaultTtlExpiry.ts

@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {
+	POSTGRES_KV_MIGRATION_TABLE,
+	postgresKvPassIsFresh,
+	recordPostgresKvCleanPass,
+} from '@app/api/database/PostgresKvQueryExecutor';
+import * as DonationTables from '@app/api/donation/DonationTables';
+import * as Tables from '@app/api/Tables';
+import {IPINFO_CACHE_TTL_SECONDS, IPINFO_REQUEST_AUDIT_TTL_SECONDS} from '@pkgs/geoip/src/PostgresIpInfoKv';
+import {type IPostgresClient, quoteIdentifier} from '@pkgs/postgres/src/Client';
+import {ms} from 'itty-time';
+
+const DEFAULT_TTL_EXPIRY_MARKER = 'default_ttl_expiry_v1';
+export const DEFAULT_TTL_EXPIRY_RESUME = 'default_ttl_expiry_v1_resume';
+const PAGE_SIZE = 2000;
+const CLEAN_PASS_INTERVAL_MS = ms('1 day');
+const OWN_EXPIRY_PASS = new Set<string>([Tables.JobsById.name, Tables.JobsByDayBucket.name]);
+
+export const DEFAULT_TTL_TABLES: ReadonlyArray<{name: string; defaultTtlSeconds: number}> = [
+	...[...Object.values(Tables), ...Object.values(DonationTables)].flatMap((table) =>
+		table.defaultTtlSeconds === undefined || OWN_EXPIRY_PASS.has(table.name)
+			? []
+			: [{name: table.name, defaultTtlSeconds: table.defaultTtlSeconds}],
+	),
+	{name: 'ipinfo_cache', defaultTtlSeconds: IPINFO_CACHE_TTL_SECONDS},
+	{name: 'ipinfo_requests_by_hour', defaultTtlSeconds: IPINFO_REQUEST_AUDIT_TTL_SECONDS},
+];
+
+export interface LegacyDefaultTtlExpiryResult {
+	deleted: number;
+	expiring: number;
+	complete: boolean;
+}
+
+interface ResumePoint {
+	table: string;
+	rowKey: string;
+	unset: number;
+}
+
+async function readResumePoint(client: IPostgresClient, kvTable: string): Promise<ResumePoint | null> {
+	const result = await client.query<{row_data: Record<string, unknown>}>(
+		`SELECT row_data FROM ${kvTable} WHERE table_name = $1 AND row_key = $2`,
+		[POSTGRES_KV_MIGRATION_TABLE, DEFAULT_TTL_EXPIRY_RESUME],
+	);
+	const data = result.rows[0]?.row_data;
+	if (typeof data?.table !== 'string' || typeof data.row_key !== 'string' || typeof data.unset !== 'number') {
+		return null;
+	}
+	return {table: data.table, rowKey: data.row_key, unset: data.unset};
+}
+
+async function writeResumePoint(client: IPostgresClient, kvTable: string, point: ResumePoint | null): Promise<void> {
+	if (point === null) {
+		await client.query(`DELETE FROM ${kvTable} WHERE table_name = $1 AND row_key = $2`, [
+			POSTGRES_KV_MIGRATION_TABLE,
+			DEFAULT_TTL_EXPIRY_RESUME,
+		]);
+		return;
+	}
+	await client.query(
+		`INSERT INTO ${kvTable} (table_name, partition_key, row_key, row_data)
+VALUES ($1, $2, $2, jsonb_build_object('table', $3::text, 'row_key', $4::text, 'unset', $5::bigint))
+ON CONFLICT (table_name, row_key) DO UPDATE SET row_data = EXCLUDED.row_data, updated_at = now()`,
+		[POSTGRES_KV_MIGRATION_TABLE, DEFAULT_TTL_EXPIRY_RESUME, point.table, point.rowKey, point.unset],
+	);
+}
+
+function pageSql(table: string): string {
+	return `
+WITH page AS (
+	SELECT kv.row_key, kv.expires_at IS NULL AS unset
+	FROM ${table} kv
+	WHERE kv.table_name = $1 AND kv.row_key > $2
+	ORDER BY kv.row_key
+	LIMIT $3
+), removed AS (
+	DELETE FROM ${table} kv
+	USING page
+	WHERE kv.table_name = $1 AND kv.row_key = page.row_key AND kv.expires_at IS NULL
+		AND kv.updated_at + make_interval(secs => $4::double precision) <= now()
+	RETURNING 1
+), expiring AS (
+	UPDATE ${table} kv
+	SET expires_at = kv.updated_at + make_interval(secs => $4::double precision)
+	FROM page
+	WHERE kv.table_name = $1 AND kv.row_key = page.row_key AND kv.expires_at IS NULL
+		AND kv.updated_at + make_interval(secs => $4::double precision) > now()
+	RETURNING 1
+)
+SELECT
+	(SELECT max(row_key) FROM page) AS last_row_key,
+	(SELECT count(*) FROM page WHERE unset) AS unset,
+	(SELECT count(*) FROM removed) AS deleted,
+	(SELECT count(*) FROM expiring) AS expiring`;
+}
+
+export async function expireLegacyDefaultTtlRows(
+	client: IPostgresClient,
+	deadlineMs: number,
+): Promise<LegacyDefaultTtlExpiryResult | null> {
+	if (await postgresKvPassIsFresh(client, DEFAULT_TTL_EXPIRY_MARKER, CLEAN_PASS_INTERVAL_MS)) {
+		return null;
+	}
+	const kvTable = quoteIdentifier(client.kvTable());
+	const sql = pageSql(kvTable);
+	const resume = await readResumePoint(client, kvTable);
+	const resumeIndex = resume === null ? -1 : DEFAULT_TTL_TABLES.findIndex((target) => target.name === resume.table);
+	let unset = resumeIndex < 0 ? 0 : resume!.unset;
+	let deleted = 0;
+	let expiring = 0;
+	for (let index = Math.max(resumeIndex, 0); index < DEFAULT_TTL_TABLES.length; index += 1) {
+		const target = DEFAULT_TTL_TABLES[index]!;
+		let cursor = index === resumeIndex ? resume!.rowKey : '';
+		for (;;) {
+			if (Date.now() >= deadlineMs) {
+				await writeResumePoint(client, kvTable, {table: target.name, rowKey: cursor, unset});
+				return {deleted, expiring, complete: false};
+			}
+			const result = await client.query<{
+				last_row_key: string | null;
+				unset: string;
+				deleted: string;
+				expiring: string;
+			}>(sql, [target.name, cursor, PAGE_SIZE, target.defaultTtlSeconds]);
+			const page = result.rows[0];
+			if (!page || page.last_row_key === null) {
+				break;
+			}
+			unset += Number(page.unset);
+			deleted += Number(page.deleted);
+			expiring += Number(page.expiring);
+			cursor = page.last_row_key;
+		}
+	}
+	await writeResumePoint(client, kvTable, null);
+	if (unset === 0) {
+		await recordPostgresKvCleanPass(client, DEFAULT_TTL_EXPIRY_MARKER);
+	}
+	return {deleted, expiring, complete: true};
+}

fluxer_api/src/api/database/PostgresKvQueryExecutor.ts

@@ -89,6 +89,28 @@ const NUMERIC_ROW_KEY_NUMBER_PATTERN = '^(-?[0-9]+(?:\\.[0-9]+)?(?:[eE][-+]?[0-9
 const EXPIRED_STORED_ROW = 'kv.expires_at IS NOT NULL AND kv.expires_at <= now()';
 const MERGED_ROW_DATA = `CASE WHEN ${EXPIRED_STORED_ROW} THEN EXCLUDED.row_data ELSE kv.row_data || EXCLUDED.row_data END`;
 const KEPT_EXPIRES_AT = `CASE WHEN ${EXPIRED_STORED_ROW} THEN NULL ELSE kv.expires_at END`;
+const NO_EXPIRY = 'infinity';
+
+export async function postgresKvPassIsFresh(
+	client: IPostgresClient,
+	marker: string,
+	maxAgeMs: number,
+): Promise<boolean> {
+	const result = await client.query(
+		`SELECT 1 FROM ${quoteIdentifier(client.kvTable())} WHERE table_name = $1 AND row_key = $2 AND (row_data ->> 'applied_at')::timestamptz > now() - make_interval(secs => $3::double precision)`,
+		[POSTGRES_KV_MIGRATION_TABLE, marker, maxAgeMs / 1000],
+	);
+	return result.rows.length > 0;
+}
+
+export async function recordPostgresKvCleanPass(client: IPostgresClient, marker: string): Promise<void> {
+	await client.query(
+		`INSERT INTO ${quoteIdentifier(client.kvTable())} (table_name, partition_key, row_key, row_data)
+VALUES ($1, $2, $2, jsonb_build_object('applied_at', now()))
+ON CONFLICT (table_name, row_key) DO UPDATE SET row_data = EXCLUDED.row_data, updated_at = now()`,
+		[POSTGRES_KV_MIGRATION_TABLE, marker],
+	);
+}
 
 function numericRowKeyExpr(column: string): string {
 	return `(COALESCE(substring(${column} from '${NUMERIC_ROW_KEY_BIGINT_PATTERN}'), substring(${column} from '${NUMERIC_ROW_KEY_NUMBER_PATTERN}'))::numeric)`;
@@ -333,20 +355,21 @@ function projectRow(row: Row, columns: ReadonlyArray<string> | undefined): Row {
 	return projected;
 }
 
-function rowComparator(meta: KvQueryMeta): (left: Row, right: Row) => number {
-	if (meta.orderBy) {
-		const column = meta.orderBy.col as string;
-		const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
-		return (left, right) => compareValues(left[column], right[column]) * direction;
+function compareColumns(columns: ReadonlyArray<string>, left: Row, right: Row): number {
+	for (const column of columns) {
+		const cmp = compareValues(left[column], right[column]);
+		if (cmp !== 0) return cmp;
 	}
-	const columns = meta.table.primaryKey as ReadonlyArray<string>;
-	return (left, right) => {
-		for (const column of columns) {
-			const cmp = compareValues(left[column], right[column]);
-			if (cmp !== 0) return cmp;
-		}
-		return 0;
-	};
+	return 0;
+}
+
+function rowComparator(meta: KvQueryMeta): (left: Row, right: Row) => number {
+	const primaryKey = meta.table.primaryKey as ReadonlyArray<string>;
+	if (!meta.orderBy) return (left, right) => compareColumns(primaryKey, left, right);
+	const column = meta.orderBy.col as string;
+	const columns = [column, ...primaryKey.slice(primaryKey.indexOf(column) + 1)];
+	const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
+	return (left, right) => compareColumns(columns, left, right) * direction;
 }
 
 function sortRows(meta: KvQueryMeta, rows: Array<Row>): Array<Row> {
@@ -679,7 +702,7 @@ function logFullScan(meta: KvQueryMeta): void {
 	logWarn({table: meta.table.name, action: meta.action, where: shape.summary || 'none'}, 'Postgres KV full table scan');
 }
 
-function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null | undefined {
+function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | typeof NO_EXPIRY | null | undefined {
 	const ttlParam = meta.ttlParamName;
 	if (!ttlParam) return undefined;
 	const ttlRaw = params[ttlParam];
@@ -687,7 +710,13 @@ function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null |
 		throw new Error(`TTL parameter ${ttlParam} must be a number`);
 	}
 	const ttlSeconds = validateTtlSeconds(ttlRaw);
-	return ttlSeconds === 0 ? null : new Date(Date.now() + ttlSeconds * 1000);
+	if (ttlSeconds === 0) return meta.table.defaultTtlSeconds === undefined ? null : NO_EXPIRY;
+	return new Date(Date.now() + ttlSeconds * 1000);
+}
+
+function defaultExpiresAt(meta: KvQueryMeta): Date | undefined {
+	const ttlSeconds = meta.table.defaultTtlSeconds;
+	return ttlSeconds === undefined ? undefined : new Date(Date.now() + ttlSeconds * 1000);
 }
 
 function encodePageState(pageState: PageState): string {
@@ -1191,7 +1220,8 @@ export class PostgresKvQueryExecutor {
 				'kv_del_expired',
 			);
 		}
-		const expiresAt = ttlExpiresAt(meta, params) ?? null;
+		const explicit = ttlExpiresAt(meta, params);
+		const expiresAt = explicit === undefined ? (defaultExpiresAt(meta) ?? null) : explicit;
 		const result = await db.query(
 			`INSERT INTO ${this.table} AS kv (table_name, partition_key, row_key, row_data, expires_at, updated_at)
 VALUES ($1, $2, $3, $4::jsonb, $5, now())
@@ -1244,10 +1274,14 @@ WHERE NOT $6`,
 			}
 			bindings.push(JSON.stringify(encodeRow(paramsRow(params, meta.patchKeys))));
 			const assignments = [`row_data = kv.row_data || $${bindings.length}::jsonb`, 'updated_at = now()'];
-			const expiresAt = ttlExpiresAt(meta, params);
-			if (expiresAt !== undefined) {
-				bindings.push(expiresAt);
+			const explicit = ttlExpiresAt(meta, params);
+			const fallback = explicit === undefined ? defaultExpiresAt(meta) : undefined;
+			if (explicit !== undefined) {
+				bindings.push(explicit);
 				assignments.push(`expires_at = $${bindings.length}`);
+			} else if (fallback !== undefined) {
+				bindings.push(fallback);
+				assignments.push(`expires_at = GREATEST(kv.expires_at, $${bindings.length}::timestamptz)`);
 			}
 			sql = `UPDATE ${this.table} kv SET ${assignments.join(', ')} WHERE ${where}`;
 		}
@@ -1346,15 +1380,27 @@ WHERE NOT $6`,
 		for (const column of meta.patchKeys ?? []) {
 			incoming[column] = column in params ? params[column] : null;
 		}
-		const ttl = ttlExpiresAt(meta, params);
-		const expiresAtExpr = ttl === undefined ? KEPT_EXPIRES_AT : 'EXCLUDED.expires_at';
+		const explicit = ttlExpiresAt(meta, params);
+		const fallback = explicit === undefined ? defaultExpiresAt(meta) : undefined;
+		const [expiresAtExpr, statementName] =
+			explicit !== undefined
+				? ['EXCLUDED.expires_at', 'kv_patch_set_ttl']
+				: fallback !== undefined
+					? ['GREATEST(kv.expires_at, EXCLUDED.expires_at)', 'kv_patch_default_ttl']
+					: [KEPT_EXPIRES_AT, 'kv_patch_keep_ttl'];
 		await db.query(
 			`INSERT INTO ${this.table} AS kv (table_name, partition_key, row_key, row_data, expires_at, updated_at)
 VALUES ($1, $2, $3, $4::jsonb, $5, now())
 ON CONFLICT (table_name, row_key)
 DO UPDATE SET partition_key = EXCLUDED.partition_key, row_data = ${MERGED_ROW_DATA}, expires_at = ${expiresAtExpr}, updated_at = now()`,
-			[meta.table.name, partitionKey(meta, incoming), key, JSON.stringify(encodeRow(incoming)), ttl ?? null],
-			ttl === undefined ? 'kv_patch_keep_ttl' : 'kv_patch_set_ttl',
+			[
+				meta.table.name,
+				partitionKey(meta, incoming),
+				key,
+				JSON.stringify(encodeRow(incoming)),
+				explicit ?? fallback ?? null,
+			],
+			statementName,
 		);
 	}
 

fluxer_api/src/api/database/PostgresKvStatementNames.test.ts

@@ -52,6 +52,8 @@ const Composite: KvTableSpec<Row> = {
 	partitionKey: ['owner_id'],
 };
 
+const Expiring: KvTableSpec<Row> = {...Composite, name: 'stmt_expiring', defaultTtlSeconds: 600};
+
 const Bucketed: KvTableSpec<Row> = {
 	name: 'stmt_bucketed',
 	columns: ['bucket', 'item_id', 'payload'],
@@ -118,6 +120,7 @@ async function runShapes(): Promise<Array<Statement>> {
 			meta(Composite, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload'], ttlParamName: 'ttl_'}),
 			{...OWNER_ITEM, ttl_: 600} as CassandraParams,
 		],
+		[meta(Expiring, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload']}), OWNER_ITEM],
 	];
 	for (const [kvMeta, params] of cases) {
 		await executor.executeQuery({cql: `__stmt_${kvMeta.action}`, params, kvMeta: kvMeta as KvQueryMeta});
@@ -142,6 +145,7 @@ describe('PostgresKvQueryExecutor statement names', () => {
 			'kv_del_keys',
 			'kv_del_rowkeys',
 			'kv_get_row',
+			'kv_patch_default_ttl',
 			'kv_patch_keep_ttl',
 			'kv_patch_set_ttl',
 			'kv_sel_range',
@@ -231,6 +235,17 @@ async function exerciseKvShapes(executor: PostgresKvQueryExecutor): Promise<void
 		kvMeta: meta(Composite, 'select', [eq('owner_id'), eq('item_id')]) as KvQueryMeta,
 	});
 	expect(patched.map((row) => row.payload)).toEqual(['patched']);
+	await executor.executeQuery({
+		cql: '__stmt_patch_default_ttl',
+		params: {owner_id: 'o5', item_id: 'i5', payload: 'defaulted'} as CassandraParams,
+		kvMeta: meta(Expiring, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload']}) as KvQueryMeta,
+	});
+	const defaulted = await executor.executeQuery<Row>({
+		cql: '__stmt_point',
+		params: {owner_id: 'o5', item_id: 'i5'} as CassandraParams,
+		kvMeta: meta(Expiring, 'select', [eq('owner_id'), eq('item_id')]) as KvQueryMeta,
+	});
+	expect(defaulted.map((row) => row.payload)).toEqual(['defaulted']);
 	await executor.executeQuery({
 		cql: '__stmt_delete',
 		params: {owner_id: 'o0', item_id: 'i0'} as CassandraParams,
@@ -323,6 +338,7 @@ describe.skipIf(!dockerAvailable)('PostgresKvQueryExecutor statement names again
 			'kv_del_expired',
 			'kv_del_rowkeys',
 			'kv_get_row',
+			'kv_patch_default_ttl',
 			'kv_patch_keep_ttl',
 			'kv_patch_set_ttl',
 			'kv_sel_range',

fluxer_api/src/api/database/PostgresScramIterations.test.ts

@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {startDockerContainer} from '@app/api/test/DockerTestContainer';
+import {getDefaultPostgresClient, initPostgres, shutdownPostgres} from '@pkgs/postgres/src/Client';
+import {afterAll, beforeAll, describe, expect, it} from 'vitest';
+
+const CONTAINER = `fluxer-kvscram-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+const SCRAM_ITERATIONS = 200_000;
+
+async function sleep(ms: number): Promise<void> {
+	await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function freePort(): Promise<number> {
+	return new Promise((resolve, reject) => {
+		const server = createServer();
+		server.on('error', reject);
+		server.listen(0, '127.0.0.1', () => {
+			const address = server.address();
+			if (typeof address === 'string' || address === null) {
+				reject(new Error('no port'));
+				return;
+			}
+			const port = address.port;
+			server.close(() => resolve(port));
+		});
+	});
+}
+
+describe.skipIf(!dockerAvailable)('postgres client against a server with raised SCRAM iterations', () => {
+	let port: number;
+
+	beforeAll(async () => {
+		port = await freePort();
+		startDockerContainer([
+			'run',
+			'-d',
+			'--name',
+			CONTAINER,
+			'-e',
+			'POSTGRES_USER=fluxer',
+			'-e',
+			'POSTGRES_PASSWORD=fluxer',
+			'-e',
+			'POSTGRES_DB=fluxer',
+			'-p',
+			`127.0.0.1:${port}:5432`,
+			'postgres:16-alpine',
+			'-c',
+			'fsync=off',
+			'-c',
+			`scram_iterations=${SCRAM_ITERATIONS}`,
+		]);
+		let ready = false;
+		for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+			await sleep(500);
+			const probe = spawnSync(
+				'docker',
+				['exec', CONTAINER, 'psql', '-h', '127.0.0.1', '-U', 'fluxer', '-d', 'fluxer', '-Atc', 'SELECT 1'],
+				{stdio: 'ignore'},
+			);
+			ready = probe.status === 0;
+		}
+		if (!ready) throw new Error('postgres never came up');
+		const rehash = spawnSync(
+			'docker',
+			['exec', CONTAINER, 'psql', '-U', 'fluxer', '-d', 'fluxer', '-Atc', "ALTER ROLE fluxer PASSWORD 'fluxer'"],
+			{
+				stdio: 'ignore',
+			},
+		);
+		if (rehash.status !== 0) throw new Error('could not re-hash the role password');
+	}, 900_000);
+
+	afterAll(async () => {
+		await shutdownPostgres().catch(() => {});
+		spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+	});
+
+	it('connects when the role verifier uses more iterations than the driver default allows', async () => {
+		await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 1});
+		const verifier = await getDefaultPostgresClient().query<{rolpassword: string}>(
+			"SELECT rolpassword FROM pg_authid WHERE rolname = 'fluxer'",
+		);
+		expect(verifier.rows[0]?.rolpassword.startsWith(`SCRAM-SHA-256$${SCRAM_ITERATIONS}:`)).toBe(true);
+	});
+});

fluxer_api/src/api/donation/DonationTables.ts

@@ -13,6 +13,7 @@ import {
 	type DonorMagicLinkTokenRow,
 	type DonorRow,
 } from '@app/api/database/types/DonationTypes';
+import {seconds} from 'itty-time';
 
 export const Donors = defineTable<DonorRow, 'email'>({
 	name: 'donors',
@@ -43,9 +44,11 @@ export const DonorMagicLinkTokens = defineTable<DonorMagicLinkTokenRow, 'token_'
 	name: 'donor_magic_link_tokens',
 	columns: DONOR_MAGIC_LINK_TOKEN_COLUMNS,
 	primaryKey: ['token_'],
+	defaultTtlSeconds: seconds('15 minutes'),
 });
 export const DonorMagicLinkTokensByEmail = defineTable<DonorMagicLinkTokenByEmailRow, 'donor_email' | 'token_'>({
 	name: 'donor_magic_link_tokens_by_email',
 	columns: DONOR_MAGIC_LINK_TOKEN_BY_EMAIL_COLUMNS,
 	primaryKey: ['donor_email', 'token_'],
+	defaultTtlSeconds: seconds('15 minutes'),
 });

fluxer_api/src/api/guild/GuildAuditLogService.ts

@@ -15,6 +15,8 @@ import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType';
 import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
 import {ms} from 'itty-time';
 
+const MESSAGE_DELETE_BATCH_DELAY_MS = ms('30 seconds');
+
 interface MessageDeleteBatchGroup {
 	logs: Array<GuildAuditLog>;
 	userId: UserID;
@@ -81,14 +83,15 @@ export class GuildAuditLogService {
 	}
 
 	async scheduleMessageDeleteBatchJob(guildId: GuildID): Promise<void> {
-		const runAt = new Date(Date.now() + ms('30 seconds'));
+		const batchWindow = Math.floor(Date.now() / MESSAGE_DELETE_BATCH_DELAY_MS);
 		await this.workerService.addJob(
 			'batchGuildAuditLogMessageDeletes',
 			{guildId: guildId.toString()},
 			{
-				jobKey: `batch-audit-log-message-deletes:${guildId}`,
-				runAt,
+				jobKey: `batch-audit-log-message-deletes:${guildId}:${batchWindow}`,
+				runAt: new Date((batchWindow + 2) * MESSAGE_DELETE_BATCH_DELAY_MS),
 				maxAttempts: 3,
+				skipLedger: true,
 			},
 		);
 	}

fluxer_api/src/api/guild/tests/GuildAuditLogService.test.ts

@@ -60,7 +60,7 @@ function createService(roleNames: Map<string, string> = new Map()) {
 		{addJob} as unknown as IWorkerService<WorkerTaskName>,
 		{dispatchGuild} as unknown as IGatewayService,
 	);
-	return {service, createAuditLog, batchDeleteAndCreateAuditLogs, getRole, dispatchGuild};
+	return {service, createAuditLog, batchDeleteAndCreateAuditLogs, getRole, dispatchGuild, addJob};
 }
 
 function overwrites(
@@ -376,3 +376,26 @@ describe('GuildAuditLogService.recordPermissionOverwriteDiff', () => {
 		expect(dispatchGuild).not.toHaveBeenCalled();
 	});
 });
+
+describe('GuildAuditLogService.scheduleMessageDeleteBatchJob', () => {
+	it('gives every delete in one 30 second window a single batch job that runs after the window closes, without a ledger row', async () => {
+		vi.useFakeTimers({toFake: ['Date']});
+		try {
+			const {service, addJob} = createService();
+			for (const at of ['2026-09-21T12:00:00.000Z', '2026-09-21T12:00:29.999Z', '2026-09-21T12:00:40.000Z']) {
+				vi.setSystemTime(new Date(at));
+				await service.scheduleMessageDeleteBatchJob(GUILD_ID);
+			}
+			const options = addJob.mock.calls.map((call) => call[2] as {jobKey: string; runAt: Date; skipLedger: boolean});
+			expect(options.every((option) => option.skipLedger)).toBe(true);
+			expect(options[0]!.jobKey).toBe(options[1]!.jobKey);
+			expect(options[2]!.jobKey).not.toBe(options[1]!.jobKey);
+			expect(options[0]!.runAt.getTime()).toBeGreaterThan(new Date('2026-09-21T12:00:29.999Z').getTime());
+			expect(options[1]!.runAt).toEqual(options[0]!.runAt);
+			expect(options[2]!.runAt).toEqual(new Date('2026-09-21T12:01:30.000Z'));
+			expect(options[2]!.runAt.getTime() - options[0]!.runAt.getTime()).toBe(30_000);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+});

fluxer_api/src/api/jobs/IJobLedgerRepository.ts

@@ -32,10 +32,12 @@ export interface ListJobsResult {
 }
 
 export abstract class IJobLedgerRepository {
-	abstract createJob(input: CreateJobInput): Promise<void>;
+	abstract createJob(input: CreateJobInput): Promise<Date>;
 
 	abstract getJob(jobId: bigint): Promise<JobByIdRow | null>;
 
+	abstract discardJob(jobId: bigint, createdAt: Date): Promise<void>;
+
 	abstract markRunning(jobId: bigint, lane: string): Promise<void>;
 
 	abstract markSucceeded(jobId: bigint, result: Record<string, unknown> | null): Promise<void>;

fluxer_api/src/api/jobs/JobLedgerRepository.test.ts

@@ -1,9 +1,24 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import {setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
+import {
+	type CassandraQueryExecutorForTesting,
+	executeQuery,
+	fetchMany,
+	fetchOne,
+	setCassandraQueryExecutorForTesting,
+} from '@app/api/database/CassandraQueryExecution';
+import {Db, type PreparedQuery} from '@app/api/database/CassandraTypes';
 import type {JobStatus} from '@app/api/database/types/JobLedgerTypes';
-import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
+import {
+	EXPIRED_JOB_ERROR,
+	JOB_LEDGER_TTL_SECONDS,
+	JOB_STALE_AFTER_MS,
+	JobLedgerRepository,
+} from '@app/api/jobs/JobLedgerRepository';
+import {describeListJobsPaging} from '@app/api/jobs/ListJobsPagingSuite';
+import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
 import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
+import {createSnowflake} from '@fluxer/snowflake/src/Snowflake';
 import {afterEach, beforeEach, describe, expect, it} from 'vitest';
 
 let executor: InMemoryCassandraQueryExecutor;
@@ -22,6 +37,20 @@ async function createJob(repository: JobLedgerRepository, jobId: bigint, taskTyp
 	});
 }
 
+async function createJobAt(repository: JobLedgerRepository, jobId: bigint): Promise<Date> {
+	return repository.createJob({
+		jobId,
+		taskType: 'batchGuildAuditLogMessageDeletes',
+		payload: {},
+		requestedByUserId: null,
+		auditLogReason: null,
+		maxAttempts: 3,
+		runAt: null,
+		jetStreamLane: null,
+		jetStreamSeq: null,
+	});
+}
+
 async function listJobIdsByStatus(repository: JobLedgerRepository, status: JobStatus): Promise<Array<bigint>> {
 	const result = await repository.listJobs({limit: 50, cursor: null, filters: {status}, maxLookbackDays: 1});
 	return result.jobs.map((job) => job.job_id);
@@ -130,6 +159,197 @@ describe('JobLedgerRepository listJobs pagination', () => {
 			maxLookbackDays: 1,
 		});
 
-		expect(result.jobs.map((job) => job.job_id)).toEqual([1_000n, 1_001n, 1_002n, 1_003n, 1_004n]);
+		expect(result.jobs.map((job) => job.job_id)).toEqual([1_004n, 1_003n, 1_002n, 1_001n, 1_000n]);
+	});
+});
+
+describe('JobLedgerRepository listJobs on the in-memory executor', () => {
+	beforeEach(() => {
+		executor = new InMemoryCassandraQueryExecutor();
+		setCassandraQueryExecutorForTesting(executor);
+	});
+
+	afterEach(() => {
+		executor.reset();
+		setCassandraQueryExecutorForTesting(null);
+	});
+
+	describeListJobsPaging();
+});
+
+let staleSequence = 0;
+
+function jobIdAgedDays(days: number): bigint {
+	staleSequence += 1;
+	return createSnowflake({timestamp: Date.now() - days * 86_400_000, sequence: staleSequence % 4096, workerId: 1});
+}
+
+async function createAgedJob(repository: JobLedgerRepository, days: number): Promise<bigint> {
+	const jobId = jobIdAgedDays(days);
+	await createJob(repository, jobId, 'syncUrlBlocklists');
+	return jobId;
+}
+
+async function activeJobIds(repository: JobLedgerRepository): Promise<Array<bigint>> {
+	return (await repository.listActiveJobs()).map((job) => job.job_id).sort((a, b) => (a < b ? -1 : 1));
+}
+
+function sweep(repository: JobLedgerRepository, maxCleared = 100) {
+	return repository.expireStaleActiveJobs({
+		staleBeforeMs: Date.now() - JOB_STALE_AFTER_MS,
+		pageSize: 100,
+		maxCleared,
+	});
+}
+
+describe('JobLedgerRepository expireStaleActiveJobs', () => {
+	beforeEach(() => {
+		executor = new InMemoryCassandraQueryExecutor();
+		setCassandraQueryExecutorForTesting(executor);
+	});
+
+	afterEach(() => {
+		executor.reset();
+		setCassandraQueryExecutorForTesting(null);
+	});
+
+	it('dead-letters queued and running jobs the jobs stream has outlived and keeps younger ones active', async () => {
+		const repository = new JobLedgerRepository();
+		const staleQueued = await createAgedJob(repository, 9);
+		const staleRunning = await createAgedJob(repository, 9);
+		await repository.markRunning(staleRunning, 'batch');
+		const weekOld = await createAgedJob(repository, 7);
+		const fresh = await createAgedJob(repository, 0);
+
+		expect(await sweep(repository)).toEqual({cleared: 2, expired: 2, complete: true});
+
+		for (const jobId of [staleQueued, staleRunning]) {
+			const job = await repository.getJob(jobId);
+			expect(job?.status).toBe('deadletter');
+			expect(job?.error_message).toBe(EXPIRED_JOB_ERROR);
+			expect(job?.completed_at).toBeInstanceOf(Date);
+		}
+		expect(await activeJobIds(repository)).toEqual([weekOld, fresh].sort((a, b) => (a < b ? -1 : 1)));
+		expect((await repository.getJob(weekOld))?.status).toBe('queued');
+	});
+
+	it('drops a stale active row without touching a finished or missing job', async () => {
+		const repository = new JobLedgerRepository();
+		const finished = await createAgedJob(repository, 9);
+		await repository.markSucceeded(finished, null);
+		const orphan = jobIdAgedDays(9);
+		for (const jobId of [finished, orphan]) {
+			await executeQuery(
+				JobsActive.patchByPkWithTtl({job_id: jobId}, {status: Db.set('running')}, JOB_LEDGER_TTL_SECONDS),
+			);
+		}
+
+		expect(await sweep(repository)).toEqual({cleared: 2, expired: 0, complete: true});
+
+		const job = await repository.getJob(finished);
+		expect(job?.status).toBe('succeeded');
+		expect(job?.error_message).toBeNull();
+		expect(await fetchOne(JobsById.select({where: JobsById.where.eq('job_id')}).bind({job_id: orphan}))).toBeNull();
+		expect(await fetchMany(JobsActive.select().bind({}))).toEqual([]);
+	});
+
+	it('stops at its per-run cap and picks up the rest on the next run', async () => {
+		const repository = new JobLedgerRepository();
+		for (let index = 0; index < 3; index += 1) {
+			await createAgedJob(repository, 9);
+		}
+
+		expect(await sweep(repository, 2)).toEqual({cleared: 2, expired: 2, complete: false});
+		expect(await activeJobIds(repository)).toHaveLength(1);
+		expect(await sweep(repository, 2)).toEqual({cleared: 1, expired: 1, complete: true});
+		expect(await activeJobIds(repository)).toEqual([]);
+	});
+});
+
+describe('JobLedgerRepository getJob', () => {
+	beforeEach(() => {
+		executor = new InMemoryCassandraQueryExecutor();
+		setCassandraQueryExecutorForTesting(executor);
+	});
+
+	afterEach(() => {
+		executor.reset();
+		setCassandraQueryExecutorForTesting(null);
+	});
+
+	it('hides a job row that lost its status, creation time or task type', async () => {
+		const repository = new JobLedgerRepository();
+		await createJob(repository, 9n, 'syncUrlBlocklists');
+		expect((await repository.getJob(9n))?.status).toBe('queued');
+		for (const column of ['status', 'created_at', 'task_type'] as const) {
+			await createJob(repository, 9n, 'syncUrlBlocklists');
+			await executeQuery(JobsById.patchByPk({job_id: 9n}, {[column]: Db.clear()}));
+			expect(await repository.getJob(9n)).toBeNull();
+		}
+	});
+});
+
+describe('JobLedgerRepository discardJob', () => {
+	let inner: InMemoryCassandraQueryExecutor;
+	let log: Array<string>;
+	let failDeleteOn: string | null;
+
+	beforeEach(() => {
+		inner = new InMemoryCassandraQueryExecutor();
+		log = [];
+		failDeleteOn = null;
+		const wrapper: CassandraQueryExecutorForTesting = {
+			async executeQuery<T>(query: PreparedQuery) {
+				const meta = query.kvMeta;
+				if (meta) log.push(`${meta.action} ${meta.table.name}`);
+				if (meta?.action === 'delete' && meta.table.name === failDeleteOn) {
+					throw new Error('write timeout');
+				}
+				return inner.executeQuery<T>(query);
+			},
+			executeBatch: (queries, atomic) => inner.executeBatch(queries, atomic),
+		};
+		setCassandraQueryExecutorForTesting(wrapper);
+	});
+
+	afterEach(() => {
+		inner.reset();
+		setCassandraQueryExecutorForTesting(null);
+	});
+
+	it('removes every ledger row of a duplicate with three deletes and no read', async () => {
+		const repository = new JobLedgerRepository();
+		const createdAt = await createJobAt(repository, 7n);
+		log.length = 0;
+
+		await repository.discardJob(7n, createdAt);
+
+		expect([...log].sort()).toEqual(['delete jobs_active', 'delete jobs_by_day_bucket', 'delete jobs_by_id']);
+		expect(await repository.getJob(7n)).toBeNull();
+		expect(await repository.listActiveJobs()).toEqual([]);
+		expect(
+			await fetchMany(
+				JobsByDayBucket.select({where: JobsByDayBucket.where.eq('bucket_day')}).bind({
+					bucket_day: createdAt.toISOString().slice(0, 10),
+				}),
+			),
+		).toEqual([]);
+	});
+
+	it('keeps deleting the other ledger rows when one delete fails', async () => {
+		const repository = new JobLedgerRepository();
+		const createdAt = await createJobAt(repository, 8n);
+		failDeleteOn = 'jobs_by_id';
+
+		await expect(repository.discardJob(8n, createdAt)).rejects.toThrow('write timeout');
+
+		expect(await repository.listActiveJobs()).toEqual([]);
+		expect(
+			await fetchMany(
+				JobsByDayBucket.select({where: JobsByDayBucket.where.eq('bucket_day')}).bind({
+					bucket_day: createdAt.toISOString().slice(0, 10),
+				}),
+			),
+		).toEqual([]);
 	});
 });

fluxer_api/src/api/jobs/JobLedgerRepository.ts

@@ -1,6 +1,14 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import {BatchBuilder, deleteOneOrMany, fetchMany, fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
+import {
+	BatchBuilder,
+	deleteOneOrMany,
+	fetchMany,
+	fetchOne,
+	fetchPage,
+	type PagedQueryResult,
+	upsertOne,
+} from '@app/api/database/CassandraQueryExecution';
 import {Db} from '@app/api/database/CassandraTypes';
 import type {JobActiveRow, JobByDayBucketRow, JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
 import {
@@ -11,6 +19,17 @@ import {
 	type ListJobsResult,
 } from '@app/api/jobs/IJobLedgerRepository';
 import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
+import {awaitAll} from '@app/api/utils/ConcurrencyUtils';
+import {JOBS_STREAM_MAX_AGE_MS} from '@app/api/worker/JetStreamWorkerQueue';
+import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';
+import {ms, seconds} from 'itty-time';
+
+export const JOB_LEDGER_TTL_SECONDS = seconds('90 days');
+export const JOB_STALE_AFTER_MS = JOBS_STREAM_MAX_AGE_MS + ms('1 day');
+export const EXPIRED_JOB_ERROR = 'Expired from the job queue';
+
+const JOB_LEDGER_RETENTION_DAYS = JOB_LEDGER_TTL_SECONDS / seconds('1 day');
+const NEWEST_FIRST = {col: 'created_at', direction: 'DESC'} as const;
 
 const FETCH_JOB_BY_ID_QUERY = JobsById.select({
 	where: JobsById.where.eq('job_id'),
@@ -19,13 +38,58 @@ const FETCH_CANCEL_REQUESTED_QUERY = JobsById.select({
 	where: JobsById.where.eq('job_id'),
 });
 const ACTIVE_JOBS_QUERY = JobsActive.select();
+const ACTIVE_JOB_IDS_QUERY = JobsActive.select({columns: ['job_id']});
+const JOBS_AFTER_IN_TIE_QUERY = JobsByDayBucket.select({
+	where: [
+		JobsByDayBucket.where.eq('bucket_day'),
+		JobsByDayBucket.where.eq('created_at'),
+		JobsByDayBucket.where.lt('job_id'),
+	],
+	orderBy: NEWEST_FIRST,
+});
+
+type LedgerPosition = Pick<ListJobsCursor, 'createdAt' | 'jobId'>;
 
 function bucketDayFor(d: Date): string {
 	return d.toISOString().slice(0, 10);
 }
 
+function previousBucketDay(day: string): string {
+	const date = new Date(`${day}T00:00:00Z`);
+	date.setUTCDate(date.getUTCDate() - 1);
+	return bucketDayFor(date);
+}
+
+function dayJobsQuery(olderThanPosition: boolean, limit: number | null) {
+	return JobsByDayBucket.select({
+		where: olderThanPosition
+			? [JobsByDayBucket.where.eq('bucket_day'), JobsByDayBucket.where.lt('created_at')]
+			: JobsByDayBucket.where.eq('bucket_day'),
+		orderBy: NEWEST_FIRST,
+		...(limit === null ? {} : {limit}),
+	});
+}
+
+async function fetchDayAfter(
+	bucketDay: string,
+	after: LedgerPosition | null,
+	limit: number | null,
+): Promise<{rows: Array<JobByDayBucketRow>; exhausted: boolean}> {
+	if (after === null) {
+		const rows = await fetchMany<JobByDayBucketRow>(dayJobsQuery(false, limit).bind({bucket_day: bucketDay}));
+		return {rows, exhausted: limit === null || rows.length < limit};
+	}
+	const ties = await fetchMany<JobByDayBucketRow>(
+		JOBS_AFTER_IN_TIE_QUERY.bind({bucket_day: bucketDay, created_at: after.createdAt, job_id: after.jobId}),
+	);
+	const older = await fetchMany<JobByDayBucketRow>(
+		dayJobsQuery(true, limit).bind({bucket_day: bucketDay, created_at: after.createdAt}),
+	);
+	return {rows: [...ties, ...older], exhausted: limit === null || older.length < limit};
+}
+
 export class JobLedgerRepository extends IJobLedgerRepository {
-	async createJob(input: CreateJobInput): Promise<void> {
+	async createJob(input: CreateJobInput): Promise<Date> {
 		const now = new Date();
 		const status: JobStatus = 'queued';
 		const idRow: JobByIdRow = {
@@ -68,39 +132,62 @@ export class JobLedgerRepository extends IJobLedgerRepository {
 			started_at: null,
 		};
 		const batch = new BatchBuilder();
-		batch.addPrepared(JobsById.insert(idRow));
-		batch.addPrepared(JobsByDayBucket.insert(bucketRow));
-		batch.addPrepared(JobsActive.insert(activeRow));
+		batch.addPrepared(JobsById.insertWithTtl(idRow, JOB_LEDGER_TTL_SECONDS));
+		batch.addPrepared(JobsByDayBucket.insertWithTtl(bucketRow, JOB_LEDGER_TTL_SECONDS));
+		batch.addPrepared(JobsActive.insertWithTtl(activeRow, JOB_LEDGER_TTL_SECONDS));
 		await batch.executeChunked(10, false);
+		return now;
 	}
 
 	async getJob(jobId: bigint): Promise<JobByIdRow | null> {
-		return fetchOne<JobByIdRow>(FETCH_JOB_BY_ID_QUERY.bind({job_id: jobId}));
+		const row = await fetchOne<JobByIdRow>(FETCH_JOB_BY_ID_QUERY.bind({job_id: jobId}));
+		return row?.created_at && row.task_type && row.status ? row : null;
+	}
+
+	async discardJob(jobId: bigint, createdAt: Date): Promise<void> {
+		await awaitAll(
+			[
+				deleteOneOrMany(
+					JobsByDayBucket.deleteByPk({bucket_day: bucketDayFor(createdAt), created_at: createdAt, job_id: jobId}),
+				),
+				deleteOneOrMany(JobsById.deleteByPk({job_id: jobId})),
+				deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId})),
+			],
+			'Ledger discard left rows behind',
+		);
 	}
 
 	async markRunning(jobId: bigint, lane: string): Promise<void> {
 		const startedAt = new Date();
 		const status: JobStatus = 'running';
 		await upsertOne(
-			JobsById.patchByPk(
+			JobsById.patchByPkWithTtl(
 				{job_id: jobId},
 				{status: Db.set(status), started_at: Db.set(startedAt), jet_stream_lane: Db.set(lane)},
+				JOB_LEDGER_TTL_SECONDS,
+			),
+		);
+		await upsertOne(
+			JobsActive.patchByPkWithTtl(
+				{job_id: jobId},
+				{status: Db.set(status), started_at: Db.set(startedAt)},
+				JOB_LEDGER_TTL_SECONDS,
 			),
 		);
-		await upsertOne(JobsActive.patchByPk({job_id: jobId}, {status: Db.set(status), started_at: Db.set(startedAt)}));
 	}
 
 	async markSucceeded(jobId: bigint, result: Record<string, unknown> | null): Promise<void> {
 		const completedAt = new Date();
 		const status: JobStatus = 'succeeded';
 		await upsertOne(
-			JobsById.patchByPk(
+			JobsById.patchByPkWithTtl(
 				{job_id: jobId},
 				{
 					status: Db.set(status),
 					completed_at: Db.set(completedAt),
 					result: result === null ? Db.clear() : Db.set(JSON.stringify(result)),
 				},
+				JOB_LEDGER_TTL_SECONDS,
 			),
 		);
 		await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
@@ -109,7 +196,13 @@ export class JobLedgerRepository extends IJobLedgerRepository {
 	async markCancelled(jobId: bigint): Promise<void> {
 		const completedAt = new Date();
 		const status: JobStatus = 'cancelled';
-		await upsertOne(JobsById.patchByPk({job_id: jobId}, {status: Db.set(status), completed_at: Db.set(completedAt)}));
+		await upsertOne(
+			JobsById.patchByPkWithTtl(
+				{job_id: jobId},
+				{status: Db.set(status), completed_at: Db.set(completedAt)},
+				JOB_LEDGER_TTL_SECONDS,
+			),
+		);
 		await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
 	}
 
@@ -117,9 +210,10 @@ export class JobLedgerRepository extends IJobLedgerRepository {
 		const completedAt = new Date();
 		const status: JobStatus = 'deadletter';
 		await upsertOne(
-			JobsById.patchByPk(
+			JobsById.patchByPkWithTtl(
 				{job_id: jobId},
 				{status: Db.set(status), completed_at: Db.set(completedAt), error_message: Db.set(errorMessage)},
+				JOB_LEDGER_TTL_SECONDS,
 			),
 		);
 		await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
@@ -127,27 +221,30 @@ export class JobLedgerRepository extends IJobLedgerRepository {
 
 	async reportProgress(jobId: bigint, current: number, total: number | null, message: string | null): Promise<void> {
 		await upsertOne(
-			JobsById.patchByPk(
+			JobsById.patchByPkWithTtl(
 				{job_id: jobId},
 				{
 					progress_current: Db.set(BigInt(current)),
 					progress_total: total === null ? Db.clear() : Db.set(BigInt(total)),
 					progress_message: message === null ? Db.clear() : Db.set(message),
 				},
+				JOB_LEDGER_TTL_SECONDS,
 			),
 		);
 	}
 
 	async setContextLink(jobId: bigint, link: string): Promise<void> {
-		await upsertOne(JobsById.patchByPk({job_id: jobId}, {context_link: Db.set(link)}));
+		await upsertOne(JobsById.patchByPkWithTtl({job_id: jobId}, {context_link: Db.set(link)}, JOB_LEDGER_TTL_SECONDS));
 	}
 
 	async setJetStreamSeq(jobId: bigint, seq: string): Promise<void> {
-		await upsertOne(JobsById.patchByPk({job_id: jobId}, {jet_stream_seq: Db.set(seq)}));
+		await upsertOne(JobsById.patchByPkWithTtl({job_id: jobId}, {jet_stream_seq: Db.set(seq)}, JOB_LEDGER_TTL_SECONDS));
 	}
 
 	async requestCancel(jobId: bigint): Promise<void> {
-		await upsertOne(JobsById.patchByPk({job_id: jobId}, {cancel_requested: Db.set(true)}));
+		await upsertOne(
+			JobsById.patchByPkWithTtl({job_id: jobId}, {cancel_requested: Db.set(true)}, JOB_LEDGER_TTL_SECONDS),
+		);
 	}
 
 	async isCancelRequested(jobId: bigint): Promise<boolean> {
@@ -160,7 +257,9 @@ export class JobLedgerRepository extends IJobLedgerRepository {
 	async incrementAttempts(jobId: bigint): Promise<void> {
 		const row = await this.getJob(jobId);
 		if (!row) return;
-		await upsertOne(JobsById.patchByPk({job_id: jobId}, {attempts: Db.set(row.attempts + 1)}));
+		await upsertOne(
+			JobsById.patchByPkWithTtl({job_id: jobId}, {attempts: Db.set(row.attempts + 1)}, JOB_LEDGER_TTL_SECONDS),
+		);
 	}
 
 	async listJobs(opts: {
@@ -169,59 +268,74 @@ export class JobLedgerRepository extends IJobLedgerRepository {
 		filters: ListJobsFilters;
 		maxLookbackDays: number;
 	}): Promise<ListJobsResult> {
-		const {limit, cursor, filters, maxLookbackDays} = opts;
-		const startBucket = cursor ? new Date(`${cursor.bucketDay}T00:00:00Z`) : new Date();
-		const hasFilters = Boolean(
-			filters.status ||
-				filters.taskType ||
-				(filters.requestedByUserId !== undefined && filters.requestedByUserId !== null),
-		);
-		const collected: Array<JobByIdRow> = [];
-		let nextCursor: ListJobsCursor | null = null;
-		for (let dayOffset = 0; dayOffset <= maxLookbackDays && collected.length < limit; dayOffset++) {
-			const bucketDate = new Date(startBucket);
-			bucketDate.setUTCDate(bucketDate.getUTCDate() - dayOffset);
-			const bucketDay = bucketDayFor(bucketDate);
-			const remaining = limit - collected.length + 1;
-			const bucketLimit = hasFilters ? {} : {limit: remaining};
-			const useCursor = dayOffset === 0 && cursor !== null;
-			let bucketRows: Array<JobByDayBucketRow>;
-			if (useCursor && cursor) {
-				const query = JobsByDayBucket.select({
-					where: [JobsByDayBucket.where.eq('bucket_day'), JobsByDayBucket.where.lt('created_at')],
-					...bucketLimit,
-				});
-				bucketRows = await fetchMany<JobByDayBucketRow>(
-					query.bind({bucket_day: bucketDay, created_at: cursor.createdAt}),
-				);
-			} else {
-				const query = JobsByDayBucket.select({
-					where: JobsByDayBucket.where.eq('bucket_day'),
-					...bucketLimit,
-				});
-				bucketRows = await fetchMany<JobByDayBucketRow>(query.bind({bucket_day: bucketDay}));
+		const {limit, cursor, filters} = opts;
+		const now = Date.now();
+		const lookbackDays = Math.min(opts.maxLookbackDays, JOB_LEDGER_RETENTION_DAYS);
+		const oldestDay = bucketDayFor(new Date(now - lookbackDays * ms('1 day')));
+		const requestedBy = filters.requestedByUserId ?? null;
+		const wholeDays = Boolean(filters.status || filters.taskType || requestedBy !== null);
+		const jobs: Array<JobByIdRow> = [];
+		let lastRow: JobByDayBucketRow | null = null;
+		let day = bucketDayFor(new Date(Math.min(cursor ? cursor.createdAt.getTime() : now, now)));
+		let after: LedgerPosition | null = cursor;
+		while (jobs.length < limit && day >= oldestDay) {
+			const {rows, exhausted} = await fetchDayAfter(day, after, wholeDays ? null : limit - jobs.length);
+			for (const row of rows) {
+				after = {createdAt: row.created_at, jobId: row.job_id};
+				if (filters.taskType && row.task_type !== filters.taskType) continue;
+				if (requestedBy !== null && row.requested_by_user_id !== requestedBy) continue;
+				const job = await this.getJob(row.job_id);
+				if (!job || (filters.status && job.status !== filters.status)) continue;
+				jobs.push(job);
+				lastRow = row;
+				if (jobs.length === limit) break;
 			}
-			for (const r of bucketRows) {
-				if (filters.taskType && r.task_type !== filters.taskType) continue;
-				if (filters.requestedByUserId !== undefined && filters.requestedByUserId !== null) {
-					if (r.requested_by_user_id !== filters.requestedByUserId) continue;
-				}
-				const fullRow = await this.getJob(r.job_id);
-				if (!fullRow) continue;
-				if (filters.status && fullRow.status !== filters.status) continue;
-				if (collected.length >= limit) {
-					nextCursor = {bucketDay, createdAt: r.created_at, jobId: r.job_id};
-					break;
-				}
-				collected.push(fullRow);
+			if (exhausted) {
+				day = previousBucketDay(day);
+				after = null;
 			}
-			if (nextCursor) break;
 		}
-		if (nextCursor === null && collected.length >= limit) {
-			const last = collected[collected.length - 1];
-			nextCursor = {bucketDay: bucketDayFor(last.created_at), createdAt: last.created_at, jobId: last.job_id};
-		}
-		return {jobs: collected, nextCursor};
+		return {
+			jobs,
+			nextCursor:
+				lastRow && jobs.length === limit
+					? {bucketDay: lastRow.bucket_day, createdAt: lastRow.created_at, jobId: lastRow.job_id}
+					: null,
+		};
+	}
+
+	async expireStaleActiveJobs(opts: {
+		staleBeforeMs: number;
+		pageSize: number;
+		maxCleared: number;
+	}): Promise<{cleared: number; expired: number; complete: boolean}> {
+		let cleared = 0;
+		let expired = 0;
+		let pageState: string | null = null;
+		do {
+			const page: PagedQueryResult<Pick<JobActiveRow, 'job_id'>> = await fetchPage(
+				ACTIVE_JOB_IDS_QUERY.bind({}),
+				undefined,
+				{
+					pageSize: opts.pageSize,
+					pageState,
+				},
+			);
+			for (const {job_id: jobId} of page.rows) {
+				if (snowflakeToDate(jobId).getTime() >= opts.staleBeforeMs) continue;
+				if (cleared >= opts.maxCleared) return {cleared, expired, complete: false};
+				const job = await this.getJob(jobId);
+				if (job?.status === 'queued' || job?.status === 'running') {
+					await this.markDeadletter(jobId, EXPIRED_JOB_ERROR);
+					expired += 1;
+				} else {
+					await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
+				}
+				cleared += 1;
+			}
+			pageState = page.pageState;
+		} while (pageState !== null);
+		return {cleared, expired, complete: true};
 	}
 
 	async listActiveJobs(): Promise<Array<JobByIdRow>> {

fluxer_api/src/api/jobs/ListJobsPagingSuite.ts

@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {upsertOne} from '@app/api/database/CassandraQueryExecution';
+import type {JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
+import type {ListJobsCursor, ListJobsFilters} from '@app/api/jobs/IJobLedgerRepository';
+import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
+import {JobsByDayBucket, JobsById} from '@app/api/Tables';
+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
+
+const DAY_MS = 86_400_000;
+const HOUR_MS = 3_600_000;
+const TODAY_NOON = new Date(`${new Date().toISOString().slice(0, 10)}T12:00:00.000Z`);
+const ADMIN_USER_ID = 4_242n;
+
+interface ListedPage {
+	ids: Array<bigint>;
+	cursor: ListJobsCursor | null;
+}
+
+function at(daysAgo: number, hour: number): Date {
+	return new Date(TODAY_NOON.getTime() - daysAgo * DAY_MS + (hour - 12) * HOUR_MS);
+}
+
+async function seedJob(
+	jobId: bigint,
+	createdAt: Date,
+	opts: {taskType?: string; requestedBy?: bigint | null; status?: JobStatus; bucketOnly?: boolean} = {},
+): Promise<void> {
+	const taskType = opts.taskType ?? 'A';
+	const requestedBy = opts.requestedBy ?? null;
+	await upsertOne(
+		JobsByDayBucket.insert({
+			bucket_day: createdAt.toISOString().slice(0, 10),
+			created_at: createdAt,
+			job_id: jobId,
+			task_type: taskType,
+			status: 'queued',
+			requested_by_user_id: requestedBy,
+		}),
+	);
+	if (opts.bucketOnly) return;
+	const row: JobByIdRow = {
+		job_id: jobId,
+		task_type: taskType,
+		status: opts.status ?? 'queued',
+		progress_current: null,
+		progress_total: null,
+		progress_message: null,
+		payload: '{}',
+		result: null,
+		error_message: null,
+		created_at: createdAt,
+		started_at: null,
+		completed_at: null,
+		requested_by_user_id: requestedBy,
+		audit_log_reason: null,
+		jet_stream_seq: null,
+		jet_stream_lane: 'batch',
+		attempts: 0,
+		max_attempts: 5,
+		run_at: null,
+		cancel_requested: false,
+		context_link: null,
+	};
+	await upsertOne(JobsById.insert(row));
+}
+
+async function listPages(opts: {
+	limit: number;
+	filters?: ListJobsFilters;
+	maxLookbackDays?: number;
+	cursor?: ListJobsCursor | null;
+}): Promise<Array<ListedPage>> {
+	const repository = new JobLedgerRepository();
+	const pages: Array<ListedPage> = [];
+	let cursor = opts.cursor ?? null;
+	for (let page = 0; page < 50; page += 1) {
+		const result = await repository.listJobs({
+			limit: opts.limit,
+			cursor,
+			filters: opts.filters ?? {},
+			maxLookbackDays: opts.maxLookbackDays ?? 14,
+		});
+		pages.push({ids: result.jobs.map((job) => job.job_id), cursor: result.nextCursor});
+		if (result.nextCursor === null) return pages;
+		cursor = {
+			bucketDay: result.nextCursor.bucketDay,
+			createdAt: new Date(result.nextCursor.createdAt.toISOString()),
+			jobId: BigInt(result.nextCursor.jobId.toString()),
+		};
+	}
+	throw new Error('listJobs never stopped paging');
+}
+
+function expectPages(pages: Array<ListedPage>, limit: number, expected: Array<Array<bigint>>): void {
+	expect(pages.map((page) => page.ids)).toEqual(expected);
+	const ids = pages.flatMap((page) => page.ids);
+	expect(new Set(ids).size).toBe(ids.length);
+	for (const page of pages.slice(0, -1)) {
+		expect(page.ids).toHaveLength(limit);
+		expect(page.cursor?.jobId).toBe(page.ids.at(-1));
+	}
+	expect(pages.at(-1)?.cursor).toBeNull();
+}
+
+export function describeListJobsPaging(): void {
+	describe('listJobs paging', () => {
+		beforeEach(() => {
+			vi.useFakeTimers({toFake: ['Date']});
+			vi.setSystemTime(TODAY_NOON);
+		});
+
+		afterEach(() => {
+			vi.useRealTimers();
+		});
+
+		it('walks one day newest first through a tie group larger than the page', async () => {
+			await seedJob(20n, at(0, 11));
+			for (const jobId of [11n, 12n, 13n, 14n, 15n]) {
+				await seedJob(jobId, at(0, 10));
+			}
+			await seedJob(9n, at(0, 9));
+			await seedJob(5n, at(0, 8));
+			await seedJob(6n, at(0, 8));
+
+			const pages = await listPages({limit: 2});
+
+			expectPages(pages, 2, [[20n, 15n], [14n, 13n], [12n, 11n], [9n, 6n], [5n]]);
+			expect(pages[0]?.cursor).toEqual({
+				bucketDay: at(0, 10).toISOString().slice(0, 10),
+				createdAt: at(0, 10),
+				jobId: 15n,
+			});
+		});
+
+		it('crosses days and keeps the lookback window anchored on today', async () => {
+			await seedJob(41n, at(0, 11));
+			await seedJob(40n, at(0, 10));
+			await seedJob(32n, at(1, 11));
+			await seedJob(31n, at(1, 10));
+			await seedJob(30n, at(1, 9));
+			await seedJob(21n, at(2, 11));
+			await seedJob(20n, at(2, 10));
+			await seedJob(10n, at(3, 11));
+
+			expectPages(await listPages({limit: 2, maxLookbackDays: 2}), 2, [[41n, 40n], [32n, 31n], [30n, 21n], [20n]]);
+			expectPages(await listPages({limit: 2, maxLookbackDays: 3}), 2, [
+				[41n, 40n],
+				[32n, 31n],
+				[30n, 21n],
+				[20n, 10n],
+				[],
+			]);
+			const outside = await new JobLedgerRepository().listJobs({
+				limit: 2,
+				cursor: {bucketDay: at(3, 12).toISOString().slice(0, 10), createdAt: at(3, 12), jobId: 1n},
+				filters: {},
+				maxLookbackDays: 2,
+			});
+			expect(outside).toEqual({jobs: [], nextCursor: null});
+		});
+
+		it('starts a cursor dated in the future at today', async () => {
+			await seedJob(2n, at(0, 2));
+			await seedJob(1n, at(1, 2));
+			const farFuture = new Date('9999-12-31T00:00:00.000Z');
+
+			expectPages(await listPages({limit: 5, cursor: {bucketDay: '9999-12-31', createdAt: farFuture, jobId: 1n}}), 5, [
+				[2n, 1n],
+			]);
+		}, 2_000);
+
+		it('never lists a day past the 90-day retention', async () => {
+			await seedJob(890n, at(89, 11));
+			await seedJob(910n, at(91, 11));
+
+			expectPages(await listPages({limit: 10, maxLookbackDays: 120}), 10, [[890n]]);
+		});
+
+		it('fills a page past rows whose job record is missing instead of leaving the day', async () => {
+			for (let hour = 1; hour <= 8; hour += 1) {
+				await seedJob(BigInt(hour), at(0, hour), {bucketOnly: [3, 6, 7].includes(hour)});
+			}
+
+			expectPages(await listPages({limit: 3}), 3, [
+				[8n, 5n, 4n],
+				[2n, 1n],
+			]);
+		});
+
+		it('fills pages through task type, requester and status filters across days', async () => {
+			await seedJob(60n, at(0, 11));
+			await seedJob(59n, at(0, 10.5), {taskType: 'B'});
+			await seedJob(58n, at(0, 10));
+			await seedJob(57n, at(0, 9.5), {taskType: 'B', requestedBy: ADMIN_USER_ID});
+			await seedJob(56n, at(0, 9), {status: 'succeeded'});
+			await seedJob(55n, at(0, 8.5), {taskType: 'B'});
+			await seedJob(50n, at(1, 11));
+			await seedJob(49n, at(1, 10.5), {taskType: 'B', requestedBy: ADMIN_USER_ID});
+			await seedJob(48n, at(1, 10), {bucketOnly: true});
+			await seedJob(47n, at(1, 9.5));
+
+			expectPages(await listPages({limit: 2, filters: {taskType: 'A'}}), 2, [[60n, 58n], [56n, 50n], [47n]]);
+			expectPages(await listPages({limit: 1, filters: {status: 'succeeded'}}), 1, [[56n], []]);
+			expectPages(await listPages({limit: 5, filters: {requestedByUserId: ADMIN_USER_ID}}), 5, [[57n, 49n]]);
+		});
+
+		it('resumes after a cursor whose job was discarded between pages', async () => {
+			for (let hour = 1; hour <= 5; hour += 1) {
+				await seedJob(BigInt(hour), at(0, hour));
+			}
+			const repository = new JobLedgerRepository();
+			const first = await repository.listJobs({limit: 2, cursor: null, filters: {}, maxLookbackDays: 14});
+			expect(first.jobs.map((job) => job.job_id)).toEqual([5n, 4n]);
+
+			await repository.discardJob(4n, at(0, 4));
+
+			expectPages(await listPages({limit: 2, cursor: first.nextCursor}), 2, [[3n, 2n], [1n]]);
+		});
+
+		it('returns an empty page after a page that filled exactly', async () => {
+			await seedJob(1n, at(0, 1));
+			await seedJob(2n, at(0, 2));
+
+			expectPages(await listPages({limit: 2}), 2, [[2n, 1n], []]);
+		});
+	});
+}

fluxer_api/src/api/jobs/PostgresJobLedgerExpiry.test.ts

@@ -0,0 +1,600 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {BatchBuilder, setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
+import {Db} from '@app/api/database/CassandraTypes';
+import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';
+import type {JobActiveRow, JobByDayBucketRow, JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
+import {
+	EXPIRED_JOB_ERROR,
+	JOB_LEDGER_TTL_SECONDS,
+	JOB_STALE_AFTER_MS,
+	JobLedgerRepository,
+} from '@app/api/jobs/JobLedgerRepository';
+import {describeListJobsPaging} from '@app/api/jobs/ListJobsPagingSuite';
+import {expireLegacyJobLedgerRows} from '@app/api/jobs/PostgresJobLedgerExpiry';
+import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
+import {startDockerContainer} from '@app/api/test/DockerTestContainer';
+import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
+import {createSnowflake} from '@fluxer/snowflake/src/Snowflake';
+import {
+	getDefaultPostgresClient,
+	type IPostgresClient,
+	initPostgres,
+	shutdownPostgres,
+} from '@pkgs/postgres/src/Client';
+import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
+
+const KV_TABLE = 'kv_job_ledger_expiry';
+const CONTAINER = `fluxer-kvjobs-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+const DAY_MS = 86_400_000;
+const ADMIN_USER_ID = 1_234_567_890_123n;
+
+async function sleep(ms: number): Promise<void> {
+	await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function freePort(): Promise<number> {
+	return new Promise((resolve, reject) => {
+		const server = createServer();
+		server.on('error', reject);
+		server.listen(0, '127.0.0.1', () => {
+			const address = server.address();
+			if (typeof address === 'string' || address === null) {
+				reject(new Error('no port'));
+				return;
+			}
+			const port = address.port;
+			server.close(() => resolve(port));
+		});
+	});
+}
+
+let sequence = 0;
+
+function jobIdAgedDays(days: number): bigint {
+	sequence += 1;
+	return createSnowflake({timestamp: Date.now() - days * DAY_MS, sequence: sequence % 4096, workerId: 1});
+}
+
+interface LegacyJob {
+	jobId: bigint;
+	createdAt: Date;
+}
+
+async function seedLegacyJob(
+	executor: PostgresKvQueryExecutor,
+	opts: {ageDays: number; status: JobStatus; requestedBy: bigint | null; active: boolean},
+): Promise<LegacyJob> {
+	const jobId = jobIdAgedDays(opts.ageDays);
+	const createdAt = new Date(Date.now() - opts.ageDays * DAY_MS);
+	const idRow: JobByIdRow = {
+		job_id: jobId,
+		task_type: opts.requestedBy === null ? 'flushUserActivityBuffer' : 'refreshSearchIndex',
+		status: opts.status,
+		progress_current: null,
+		progress_total: null,
+		progress_message: null,
+		payload: '{}',
+		result: null,
+		error_message: null,
+		created_at: createdAt,
+		started_at: null,
+		completed_at: opts.status === 'succeeded' ? createdAt : null,
+		requested_by_user_id: opts.requestedBy,
+		audit_log_reason: null,
+		jet_stream_seq: '1',
+		jet_stream_lane: 'batch',
+		attempts: 0,
+		max_attempts: 5,
+		run_at: null,
+		cancel_requested: false,
+		context_link: null,
+	};
+	const bucketRow: JobByDayBucketRow = {
+		bucket_day: createdAt.toISOString().slice(0, 10),
+		created_at: createdAt,
+		job_id: jobId,
+		task_type: idRow.task_type,
+		status: 'queued',
+		requested_by_user_id: opts.requestedBy,
+	};
+	await executor.executeQuery(JobsById.insert(idRow));
+	await executor.executeQuery(JobsByDayBucket.insert(bucketRow));
+	if (opts.active) {
+		const activeRow: JobActiveRow = {
+			job_id: jobId,
+			task_type: idRow.task_type,
+			status: opts.status,
+			requested_by_user_id: opts.requestedBy,
+			created_at: createdAt,
+			started_at: null,
+		};
+		await executor.executeQuery(JobsActive.insert(activeRow));
+	}
+	return {jobId, createdAt};
+}
+
+describe.skipIf(!dockerAvailable)('job ledger expiry against postgres', () => {
+	let raw: IPostgresClient;
+	let executor: PostgresKvQueryExecutor;
+
+	async function jobRows(
+		jobId: bigint,
+	): Promise<Array<{table_name: string; expires_at: Date | null; row_data: never}>> {
+		const result = await raw.query<{table_name: string; expires_at: Date | null; row_data: never}>(
+			`SELECT table_name, expires_at, row_data FROM ${KV_TABLE}
+WHERE table_name IN ('jobs_by_id', 'jobs_active', 'jobs_by_day_bucket')
+	AND (row_key = $1 OR split_part(row_key, chr(31), 3) = $1)
+ORDER BY table_name`,
+			[JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()})],
+		);
+		return result.rows;
+	}
+
+	async function waitForLockWait(): Promise<void> {
+		for (let attempt = 0; attempt < 400; attempt += 1) {
+			const waiting = await raw.query<{n: number}>(
+				`SELECT count(*)::int AS n FROM pg_stat_activity WHERE datname = current_database() AND wait_event_type = 'Lock'`,
+			);
+			if (waiting.rows[0]!.n > 0) return;
+			await sleep(25);
+		}
+		throw new Error('the pass never waited on the writer');
+	}
+
+	async function forgetLedgerExpiry(): Promise<void> {
+		await raw.query(
+			`UPDATE ${KV_TABLE} SET expires_at = NULL WHERE table_name IN ('jobs_by_id', 'jobs_by_day_bucket')`,
+		);
+	}
+
+	async function forgetExpiryOf(...jobIds: Array<bigint>): Promise<void> {
+		const keys = jobIds.map((jobId) => JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()}));
+		await raw.query(
+			`UPDATE ${KV_TABLE} SET expires_at = NULL
+WHERE table_name IN ('jobs_by_id', 'jobs_by_day_bucket')
+	AND (row_key = ANY($1::text[]) OR split_part(row_key, chr(31), 3) = ANY($1::text[]))`,
+			[keys],
+		);
+	}
+
+	beforeAll(async () => {
+		const port = await freePort();
+		startDockerContainer([
+			'run',
+			'-d',
+			'--name',
+			CONTAINER,
+			'-e',
+			'POSTGRES_USER=fluxer',
+			'-e',
+			'POSTGRES_PASSWORD=fluxer',
+			'-e',
+			'POSTGRES_DB=fluxer',
+			'-p',
+			`127.0.0.1:${port}:5432`,
+			'postgres:16-alpine',
+			'-c',
+			'fsync=off',
+		]);
+		let ready = false;
+		for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+			await sleep(500);
+			const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
+				stdio: 'ignore',
+			});
+			if (probe.status !== 0) continue;
+			try {
+				await initPostgres({
+					url: `postgres://fluxer:[email protected]:${port}/fluxer`,
+					maxConnections: 4,
+					kvTable: KV_TABLE,
+				});
+				await getDefaultPostgresClient().query('SELECT 1');
+				ready = true;
+			} catch {
+				await shutdownPostgres().catch(() => {});
+			}
+		}
+		if (!ready) throw new Error('postgres never came up');
+		raw = getDefaultPostgresClient();
+		await ensurePostgresKvSchema(raw);
+		executor = new PostgresKvQueryExecutor(raw);
+	}, 900_000);
+
+	beforeEach(async () => {
+		await raw.query(`DELETE FROM ${KV_TABLE}`);
+		setCassandraQueryExecutorForTesting(executor);
+	});
+
+	afterAll(async () => {
+		setCassandraQueryExecutorForTesting(new InMemoryCassandraQueryExecutor());
+		await shutdownPostgres().catch(() => {});
+		spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+	});
+
+	it('writes every ledger row with an expiry and keeps it through the job lifecycle', async () => {
+		const repository = new JobLedgerRepository();
+		const jobId = jobIdAgedDays(0);
+		await repository.createJob({
+			jobId,
+			taskType: 'refreshSearchIndex',
+			payload: {},
+			requestedByUserId: ADMIN_USER_ID,
+			auditLogReason: null,
+			maxAttempts: 5,
+			runAt: null,
+			jetStreamLane: 'batch',
+			jetStreamSeq: null,
+		});
+		await repository.setJetStreamSeq(jobId, '7');
+		await repository.markRunning(jobId, 'batch');
+		await repository.reportProgress(jobId, 1, 2, 'half');
+
+		const running = await jobRows(jobId);
+		expect(running.map((row) => row.table_name)).toEqual(['jobs_active', 'jobs_by_day_bucket', 'jobs_by_id']);
+		for (const row of running) {
+			expect(row.expires_at).not.toBeNull();
+			const remainingSeconds = (row.expires_at!.getTime() - Date.now()) / 1000;
+			expect(remainingSeconds).toBeGreaterThan(JOB_LEDGER_TTL_SECONDS - 60);
+			expect(remainingSeconds).toBeLessThanOrEqual(JOB_LEDGER_TTL_SECONDS);
+		}
+
+		await repository.markSucceeded(jobId, {ok: true});
+		const done = await jobRows(jobId);
+		expect(done.map((row) => row.table_name)).toEqual(['jobs_by_day_bucket', 'jobs_by_id']);
+		expect(done.every((row) => row.expires_at !== null)).toBe(true);
+		expect((await repository.getJob(jobId))?.status).toBe('succeeded');
+	});
+
+	it('never leaves a row without an expiry when a patch lands on a job that is gone', async () => {
+		const repository = new JobLedgerRepository();
+		const patches: Array<(jobId: bigint) => Promise<void>> = [
+			(jobId) => repository.markRunning(jobId, 'batch'),
+			(jobId) => repository.markSucceeded(jobId, null),
+			(jobId) => repository.markCancelled(jobId),
+			(jobId) => repository.markDeadletter(jobId, 'boom'),
+			(jobId) => repository.reportProgress(jobId, 1, null, null),
+			(jobId) => repository.setContextLink(jobId, '/admin/jobs'),
+			(jobId) => repository.setJetStreamSeq(jobId, '1'),
+			(jobId) => repository.requestCancel(jobId),
+		];
+		for (const patch of patches) {
+			const jobId = jobIdAgedDays(0);
+			await patch(jobId);
+			const rows = await jobRows(jobId);
+			expect(rows.length).toBeGreaterThan(0);
+			expect(rows.every((row) => row.expires_at !== null)).toBe(true);
+			expect(await repository.getJob(jobId)).toBeNull();
+		}
+		expect(await repository.listActiveJobs()).toEqual([]);
+	});
+
+	it('discards every row of a job that never reached the stream', async () => {
+		const repository = new JobLedgerRepository();
+		const jobId = jobIdAgedDays(0);
+		const createdAt = await repository.createJob({
+			jobId,
+			taskType: 'batchGuildAuditLogMessageDeletes',
+			payload: {guildId: '1'},
+			requestedByUserId: null,
+			auditLogReason: null,
+			maxAttempts: 3,
+			runAt: new Date(Date.now() + 30_000),
+			jetStreamLane: 'batch',
+			jetStreamSeq: null,
+		});
+		expect(await jobRows(jobId)).toHaveLength(3);
+
+		await repository.discardJob(jobId, createdAt);
+
+		expect(await jobRows(jobId)).toEqual([]);
+	});
+
+	it('clears legacy rows by the same rules the expiry now enforces', async () => {
+		const repository = new JobLedgerRepository();
+		const cronDone = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'succeeded',
+			requestedBy: null,
+			active: false,
+		});
+		const cronStuck = await seedLegacyJob(executor, {ageDays: 20, status: 'queued', requestedBy: null, active: true});
+		const partialId = jobIdAgedDays(20);
+		await executor.executeQuery(
+			JobsById.patchByPk({job_id: partialId}, {status: Db.set('succeeded'), completed_at: Db.set(new Date())}),
+		);
+		const adminDone = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'succeeded',
+			requestedBy: ADMIN_USER_ID,
+			active: false,
+		});
+		const adminStuck = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'queued',
+			requestedBy: ADMIN_USER_ID,
+			active: true,
+		});
+		const adminRunning = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'running',
+			requestedBy: ADMIN_USER_ID,
+			active: true,
+		});
+		const adminAncient = await seedLegacyJob(executor, {
+			ageDays: 100,
+			status: 'succeeded',
+			requestedBy: ADMIN_USER_ID,
+			active: false,
+		});
+		const cronInFlight = await seedLegacyJob(executor, {
+			ageDays: 2,
+			status: 'queued',
+			requestedBy: null,
+			active: true,
+		});
+		await forgetLedgerExpiry();
+		const fresh = jobIdAgedDays(0);
+		await repository.createJob({
+			jobId: fresh,
+			taskType: 'syncUrlBlocklists',
+			payload: {},
+			requestedByUserId: null,
+			auditLogReason: null,
+			maxAttempts: 5,
+			runAt: null,
+			jetStreamLane: 'batch',
+			jetStreamSeq: null,
+		});
+		const freshBefore = await jobRows(fresh);
+		await raw.query(
+			`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data) VALUES ('users', 'u1', 'u1', '{}'::jsonb)`,
+		);
+
+		expect(await repository.getJob(partialId)).toBeNull();
+
+		const first = await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
+		expect(first).toEqual({deleted: 10, expiring: 6, complete: true});
+
+		expect(await jobRows(cronDone.jobId)).toEqual([]);
+		expect(await jobRows(cronStuck.jobId)).toEqual([]);
+		expect(await jobRows(partialId)).toEqual([]);
+		expect(await jobRows(adminAncient.jobId)).toEqual([]);
+
+		for (const kept of [adminDone, adminStuck, adminRunning]) {
+			const rows = await jobRows(kept.jobId);
+			expect(rows.map((row) => row.table_name)).toEqual(['jobs_by_day_bucket', 'jobs_by_id']);
+			for (const row of rows) {
+				const expected = kept.createdAt.getTime() + JOB_LEDGER_TTL_SECONDS * 1000;
+				expect(Math.abs(row.expires_at!.getTime() - expected)).toBeLessThan(2000);
+			}
+		}
+		expect((await repository.getJob(adminDone.jobId))?.status).toBe('succeeded');
+		for (const stuck of [adminStuck, adminRunning]) {
+			const expired = await repository.getJob(stuck.jobId);
+			expect(expired?.status).toBe('deadletter');
+			expect(expired?.error_message).toBe('Expired from the job queue');
+			expect(expired?.completed_at).toBeInstanceOf(Date);
+		}
+
+		const inFlight = await jobRows(cronInFlight.jobId);
+		expect(inFlight.map((row) => row.table_name)).toEqual(['jobs_active', 'jobs_by_day_bucket', 'jobs_by_id']);
+		expect(inFlight.every((row) => row.expires_at === null)).toBe(true);
+		expect((await repository.getJob(cronInFlight.jobId))?.status).toBe('queued');
+
+		expect(await jobRows(fresh)).toEqual(freshBefore);
+		const users = await raw.query(`SELECT expires_at FROM ${KV_TABLE} WHERE table_name = 'users'`);
+		expect(users.rows).toEqual([{expires_at: null}]);
+
+		const listed = await repository.listJobs({limit: 50, cursor: null, filters: {}, maxLookbackDays: 30});
+		expect(listed.jobs.map((job) => job.job_id).sort()).toEqual(
+			[adminDone.jobId, adminStuck.jobId, adminRunning.jobId, cronInFlight.jobId, fresh].sort(),
+		);
+
+		for (let pass = 0; pass < 2; pass += 1) {
+			expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
+				deleted: 0,
+				expiring: 0,
+				complete: true,
+			});
+		}
+	});
+
+	it('runs again a day after a clean pass and clears rows an older image wrote in between', async () => {
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		const rolledBack = await seedLegacyJob(executor, {ageDays: 30, status: 'queued', requestedBy: null, active: true});
+		await forgetLedgerExpiry();
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
+
+		await raw.query(
+			`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('applied_at', now() - interval '2 days') WHERE row_key = 'job_ledger_expiry_v1'`,
+		);
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 3,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await jobRows(rolledBack.jobId)).toEqual([]);
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
+	});
+
+	it('expires stale active jobs page by page without fighting the legacy pass', async () => {
+		const repository = new JobLedgerRepository();
+		const legacyFirst = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'queued',
+			requestedBy: ADMIN_USER_ID,
+			active: true,
+		});
+		await forgetExpiryOf(legacyFirst.jobId);
+		await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
+		const legacyFirstRows = await jobRows(legacyFirst.jobId);
+
+		const stale: Array<bigint> = [];
+		for (let index = 0; index < 4; index += 1) {
+			const jobId = jobIdAgedDays(9);
+			await repository.createJob({
+				jobId,
+				taskType: 'syncUrlBlocklists',
+				payload: {},
+				requestedByUserId: null,
+				auditLogReason: null,
+				maxAttempts: 5,
+				runAt: null,
+				jetStreamLane: 'batch',
+				jetStreamSeq: null,
+			});
+			stale.push(jobId);
+		}
+		await repository.markRunning(stale[0]!, 'batch');
+		const fresh = jobIdAgedDays(0);
+		await repository.createJob({
+			jobId: fresh,
+			taskType: 'syncUrlBlocklists',
+			payload: {},
+			requestedByUserId: null,
+			auditLogReason: null,
+			maxAttempts: 5,
+			runAt: null,
+			jetStreamLane: 'batch',
+			jetStreamSeq: null,
+		});
+		const legacyStale = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'running',
+			requestedBy: ADMIN_USER_ID,
+			active: true,
+		});
+		const legacyYoung = await seedLegacyJob(executor, {ageDays: 2, status: 'queued', requestedBy: null, active: true});
+		await forgetExpiryOf(legacyStale.jobId, legacyYoung.jobId);
+		const orphan = jobIdAgedDays(9);
+		await executor.executeQuery(
+			JobsActive.patchByPkWithTtl({job_id: orphan}, {status: Db.set('running')}, JOB_LEDGER_TTL_SECONDS),
+		);
+
+		const sweep = () =>
+			repository.expireStaleActiveJobs({staleBeforeMs: Date.now() - JOB_STALE_AFTER_MS, pageSize: 2, maxCleared: 100});
+		expect(await sweep()).toEqual({cleared: 6, expired: 5, complete: true});
+
+		const active = (await repository.listActiveJobs()).map((job) => job.job_id).sort();
+		expect(active).toEqual([fresh, legacyYoung.jobId].sort());
+		for (const jobId of [...stale, legacyStale.jobId]) {
+			const job = await repository.getJob(jobId);
+			expect(job?.status).toBe('deadletter');
+			expect(job?.error_message).toBe(EXPIRED_JOB_ERROR);
+			const byId = (await jobRows(jobId)).find((row) => row.table_name === 'jobs_by_id');
+			expect(Math.abs(byId!.expires_at!.getTime() - (Date.now() + JOB_LEDGER_TTL_SECONDS * 1000))).toBeLessThan(60_000);
+		}
+		expect(await jobRows(orphan)).toEqual([]);
+		expect(await jobRows(legacyFirst.jobId)).toEqual(legacyFirstRows);
+
+		const legacyStaleById = (await jobRows(legacyStale.jobId)).find((row) => row.table_name === 'jobs_by_id');
+		await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
+		expect((await jobRows(legacyStale.jobId)).find((row) => row.table_name === 'jobs_by_id')).toEqual(legacyStaleById);
+		expect((await repository.listActiveJobs()).map((job) => job.job_id)).toContain(legacyYoung.jobId);
+		expect(await sweep()).toEqual({cleared: 0, expired: 0, complete: true});
+	});
+
+	it('leaves rows alone when a live writer gives them an expiry while the pass waits on them', async () => {
+		const cronDone = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'succeeded',
+			requestedBy: null,
+			active: false,
+		});
+		const adminDone = await seedLegacyJob(executor, {
+			ageDays: 20,
+			status: 'succeeded',
+			requestedBy: ADMIN_USER_ID,
+			active: false,
+		});
+		await forgetLedgerExpiry();
+		const liveKeys = [cronDone.jobId, adminDone.jobId].map((jobId) =>
+			JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()}),
+		);
+		let written!: () => void;
+		const writerHoldsRows = new Promise<void>((resolve) => {
+			written = resolve;
+		});
+		let release!: () => void;
+		const released = new Promise<void>((resolve) => {
+			release = resolve;
+		});
+		const writer = raw.transaction(async (db) => {
+			await db.query(
+				`UPDATE ${KV_TABLE} SET expires_at = now() + interval '1 hour', updated_at = now() WHERE table_name = 'jobs_by_id' AND row_key = ANY($1::text[])`,
+				[liveKeys],
+			);
+			written();
+			await released;
+		});
+		await writerHoldsRows;
+		const pass = expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
+		await waitForLockWait();
+		release();
+		await writer;
+
+		expect(await pass).toEqual({deleted: 1, expiring: 1, complete: true});
+		for (const job of [cronDone, adminDone]) {
+			const byId = (await jobRows(job.jobId)).find((row) => row.table_name === 'jobs_by_id');
+			expect(byId).toBeDefined();
+			const remainingSeconds = (byId!.expires_at!.getTime() - Date.now()) / 1000;
+			expect(remainingSeconds).toBeGreaterThan(3000);
+			expect(remainingSeconds).toBeLessThanOrEqual(3660);
+		}
+		expect((await new JobLedgerRepository().getJob(adminDone.jobId))?.status).toBe('succeeded');
+	});
+
+	it('pages through more legacy rows than one page holds and stops at its deadline', async () => {
+		const batch = new BatchBuilder();
+		for (let index = 0; index < 2300; index += 1) {
+			const jobId = jobIdAgedDays(30);
+			const createdAt = new Date(Date.now() - 30 * DAY_MS);
+			batch.addPrepared(
+				JobsByDayBucket.insert({
+					bucket_day: createdAt.toISOString().slice(0, 10),
+					created_at: createdAt,
+					job_id: jobId,
+					task_type: 'flushUserActivityBuffer',
+					status: 'queued',
+					requested_by_user_id: null,
+				}),
+			);
+		}
+		await batch.executeChunked(500, false);
+		await forgetLedgerExpiry();
+
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() - 1)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: false,
+		});
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 2300,
+			expiring: 0,
+			complete: true,
+		});
+		const left = await raw.query(`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = 'jobs_by_day_bucket'`);
+		expect(left.rows[0]).toEqual({n: 0});
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
+			deleted: 0,
+			expiring: 0,
+			complete: true,
+		});
+		expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
+	});
+	describeListJobsPaging();
+});

fluxer_api/src/api/jobs/PostgresJobLedgerExpiry.ts

@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {postgresKvPassIsFresh, recordPostgresKvCleanPass} from '@app/api/database/PostgresKvQueryExecutor';
+import {EXPIRED_JOB_ERROR, JOB_LEDGER_TTL_SECONDS, JOB_STALE_AFTER_MS} from '@app/api/jobs/JobLedgerRepository';
+import {FLUXER_EPOCH} from '@fluxer/constants/src/Core';
+import {TIMESTAMP_SHIFT} from '@fluxer/snowflake/src/Snowflake';
+import {type IPostgresClient, quoteIdentifier} from '@pkgs/postgres/src/Client';
+import {ms} from 'itty-time';
+
+const LEGACY_JOB_LEDGER_MARKER = 'job_ledger_expiry_v1';
+const PAGE_SIZE = 2000;
+const CLEAN_PASS_INTERVAL_MS = ms('1 day');
+const BIGINT_KEY_PREFIX = '{"__fluxer_type":"bigint","value":"';
+const BIGINT_KEY_SUFFIX = '"}';
+
+function bigintKeyExpr(key: string): string {
+	const prefixLength = BIGINT_KEY_PREFIX.length;
+	const affixLength = prefixLength + BIGINT_KEY_SUFFIX.length;
+	return `(CASE WHEN left(${key}, ${prefixLength}) = '${BIGINT_KEY_PREFIX}' AND right(${key}, ${BIGINT_KEY_SUFFIX.length}) = '${BIGINT_KEY_SUFFIX}' THEN substr(${key}, ${prefixLength + 1}, length(${key}) - ${affixLength})::numeric END)`;
+}
+
+interface LedgerTable {
+	name: string;
+	jobIdExpr: string;
+	deleteStale: boolean;
+	markStaleDeadletter: boolean;
+}
+
+const LEDGER_TABLES: ReadonlyArray<LedgerTable> = [
+	{
+		name: 'jobs_active',
+		jobIdExpr: bigintKeyExpr('kv.row_key'),
+		deleteStale: true,
+		markStaleDeadletter: false,
+	},
+	{
+		name: 'jobs_by_id',
+		jobIdExpr: bigintKeyExpr('kv.row_key'),
+		deleteStale: false,
+		markStaleDeadletter: true,
+	},
+	{
+		name: 'jobs_by_day_bucket',
+		jobIdExpr: bigintKeyExpr('split_part(kv.row_key, chr(31), 3)'),
+		deleteStale: false,
+		markStaleDeadletter: false,
+	},
+];
+
+export interface LegacyJobLedgerExpiryResult {
+	deleted: number;
+	expiring: number;
+	complete: boolean;
+}
+
+function pageSql(table: string, target: LedgerTable): string {
+	const staleRemovable = target.deleteStale ? 'c.created_at < $7' : '(c.created_at < $7 AND c.system_job)';
+	const removable = `c.job_id IS NULL OR c.created_at < $6 OR ${staleRemovable}`;
+	const rowData = target.markStaleDeadletter
+		? `CASE WHEN c.unfinished THEN kv.row_data || jsonb_build_object('status', 'deadletter', 'error_message', $9::text, 'completed_at', jsonb_build_object('__fluxer_type', 'date', 'value', $10::text)) ELSE kv.row_data END`
+		: 'kv.row_data';
+	return `
+WITH page AS (
+	SELECT kv.row_key, kv.row_data, ${target.jobIdExpr} AS job_id
+	FROM ${table} kv
+	WHERE kv.table_name = $1 AND kv.expires_at IS NULL AND kv.row_key > $2
+	ORDER BY kv.row_key
+	LIMIT $3
+), classified AS (
+	SELECT
+		page.row_key,
+		page.job_id,
+		to_timestamp(((div(page.job_id, $4::numeric) + $5::numeric) / 1000)::double precision) AS created_at,
+		COALESCE(page.row_data -> 'requested_by_user_id', 'null'::jsonb) = 'null'::jsonb AS system_job,
+		COALESCE(page.row_data ->> 'status' IN ('queued', 'running'), false) AS unfinished
+	FROM page
+), removed AS (
+	DELETE FROM ${table} kv
+	USING classified c
+	WHERE kv.table_name = $1 AND kv.row_key = c.row_key AND kv.expires_at IS NULL AND (${removable})
+	RETURNING kv.row_key
+), expiring AS (
+	UPDATE ${table} kv
+	SET expires_at = c.created_at + make_interval(secs => $8::double precision), updated_at = now(), row_data = ${rowData}
+	FROM classified c
+	WHERE kv.table_name = $1 AND kv.row_key = c.row_key AND kv.expires_at IS NULL AND c.created_at < $7 AND NOT (${removable})
+	RETURNING kv.row_key
+)
+SELECT
+	(SELECT max(row_key) FROM page) AS last_row_key,
+	(SELECT count(*) FROM page) AS scanned,
+	(SELECT count(*) FROM removed) AS deleted,
+	(SELECT count(*) FROM expiring) AS expiring`;
+}
+
+export async function expireLegacyJobLedgerRows(
+	client: IPostgresClient,
+	deadlineMs: number,
+): Promise<LegacyJobLedgerExpiryResult | null> {
+	const table = quoteIdentifier(client.kvTable());
+	if (await postgresKvPassIsFresh(client, LEGACY_JOB_LEDGER_MARKER, CLEAN_PASS_INTERVAL_MS)) {
+		return null;
+	}
+	const now = Date.now();
+	const retentionCutoff = new Date(now - JOB_LEDGER_TTL_SECONDS * 1000);
+	const staleCutoff = new Date(now - JOB_STALE_AFTER_MS);
+	const completedAt = new Date(now).toISOString();
+	let scanned = 0;
+	let deleted = 0;
+	let expiring = 0;
+	for (const target of LEDGER_TABLES) {
+		const sql = pageSql(table, target);
+		const deadletterValues = target.markStaleDeadletter ? [EXPIRED_JOB_ERROR, completedAt] : [];
+		let cursor = '';
+		for (;;) {
+			if (Date.now() >= deadlineMs) {
+				return {deleted, expiring, complete: false};
+			}
+			const result = await client.query<{
+				last_row_key: string | null;
+				scanned: string;
+				deleted: string;
+				expiring: string;
+			}>(sql, [
+				target.name,
+				cursor,
+				PAGE_SIZE,
+				(1n << TIMESTAMP_SHIFT).toString(),
+				FLUXER_EPOCH.toString(),
+				retentionCutoff,
+				staleCutoff,
+				JOB_LEDGER_TTL_SECONDS,
+				...deadletterValues,
+			]);
+			const page = result.rows[0];
+			if (!page || page.last_row_key === null) {
+				break;
+			}
+			scanned += Number(page.scanned);
+			deleted += Number(page.deleted);
+			expiring += Number(page.expiring);
+			cursor = page.last_row_key;
+		}
+	}
+	if (scanned === 0) {
+		await recordPostgresKvCleanPass(client, LEGACY_JOB_LEDGER_MARKER);
+	}
+	return {deleted, expiring, complete: true};
+}

fluxer_api/src/api/middleware/FileShaCache.ts

@@ -1,7 +1,8 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {AdminRepository} from '@app/api/admin/AdminRepository';
-import {BANNED_FILE_SHAS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
+import {Config} from '@app/api/Config';
+import {BANNED_FILE_SHAS_REFRESH_CHANNEL, isBlocklistFeedFileSha} from '@app/api/constants/ContentModeration';
 import {Logger} from '@app/api/Logger';
 import {RefreshSubscription} from '@app/api/utils/RefreshSubscription';
 import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
@@ -38,8 +39,11 @@ class FileShaCache {
 	async refresh(): Promise<void> {
 		const rows = await this.adminRepository.loadAllBannedFileShas();
 		const next = new Set<string>();
+		const includeFeedRows = Config.blocklistFeeds.enabled;
 		for (const row of rows) {
-			if (row.sha256_hex) next.add(row.sha256_hex.toLowerCase());
+			if (!row.sha256_hex) continue;
+			if (!includeFeedRows && isBlocklistFeedFileSha(row)) continue;
+			next.add(row.sha256_hex.toLowerCase());
 		}
 		this.banned = next;
 		this.consecutiveFailures = 0;

fluxer_api/src/api/middleware/UrlBlocklistCache.ts

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {AdminRepository} from '@app/api/admin/AdminRepository';
+import {Config} from '@app/api/Config';
 import {BANNED_URL_DOMAINS_REFRESH_CHANNEL, BANNED_URLS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
 import type {IStorageService} from '@app/api/infrastructure/IStorageService';
 import {Logger} from '@app/api/Logger';
@@ -68,7 +69,7 @@ class UrlBlocklistCache {
 	}
 
 	private async loadFeedUrls(): Promise<Set<string>> {
-		if (!this.storageService) return new Set();
+		if (!this.storageService || !Config.blocklistFeeds.enabled) return new Set();
 		const lines = await readLinesFromS3(this.storageService, RISK_S3_KEYS.feedUrls);
 		return new Set(lines);
 	}

fluxer_api/src/api/risk/RiskBlocklistS3.ts

@@ -47,3 +47,7 @@ export async function readLinesFromS3(storage: IStorageService, key: string): Pr
 		return [];
 	}
 }
+
+export async function deleteRiskS3Object(storage: IStorageService, key: string): Promise<void> {
+	await storage.deleteObject(RISK_S3_BUCKET, key);
+}

fluxer_api/src/api/risk/__tests__/PostgresIpInfoKv.test.ts

@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {IpInfoRequestAuditEvent} from '@pkgs/geoip/src/IpInfoService';
+import {
+	createPostgresIpInfoCache,
+	createPostgresIpInfoRequestAuditLogger,
+	IPINFO_CACHE_TTL_SECONDS,
+	IPINFO_REQUEST_AUDIT_TTL_SECONDS,
+} from '@pkgs/geoip/src/PostgresIpInfoKv';
+import type {IPostgresClient} from '@pkgs/postgres/src/Client';
+import {describe, expect, it} from 'vitest';
+
+function recordingClient(writes: Array<Array<unknown>>): IPostgresClient {
+	return {
+		async query(_text: string, values?: Array<unknown>) {
+			writes.push(values ?? []);
+			return {rows: [], rowCount: 1};
+		},
+		kvTable() {
+			return 'kv';
+		},
+	} as never;
+}
+
+function expectExpiresIn(values: Array<unknown> | undefined, ttlSeconds: number): void {
+	const expiresAt = values?.[4];
+	expect(expiresAt).toBeInstanceOf(Date);
+	const remainingSeconds = ((expiresAt as Date).getTime() - Date.now()) / 1000;
+	expect(remainingSeconds).toBeGreaterThan(ttlSeconds - 10);
+	expect(remainingSeconds).toBeLessThanOrEqual(ttlSeconds);
+}
+
+const EVENT: IpInfoRequestAuditEvent = {
+	requestedAt: new Date('2026-09-21T12:00:00.000Z'),
+	ip: '192.0.2.1',
+	cacheKey: 'ip:192.0.2.1',
+	source: 'test',
+	reason: null,
+	outcome: 'http_success',
+	httpStatus: 200,
+	available: true,
+	riskNote: 'none',
+	latencyMs: 12,
+	requestUrl: 'https://ipinfo.test/192.0.2.1',
+	responseIp: '192.0.2.1',
+	countryCode: 'SE',
+	asnNumber: 64500,
+	isAnonymous: false,
+	isTor: false,
+	isVpn: false,
+	isProxy: false,
+	isResidentialProxy: false,
+};
+
+describe('Postgres ipinfo KV expiry', () => {
+	it('expires request audit rows after 90 days', async () => {
+		const writes: Array<Array<unknown>> = [];
+		await createPostgresIpInfoRequestAuditLogger({client: recordingClient(writes)}).record(EVENT);
+		expect(writes).toHaveLength(1);
+		expect(writes[0]?.[0]).toBe('ipinfo_requests_by_hour');
+		expectExpiresIn(writes[0], IPINFO_REQUEST_AUDIT_TTL_SECONDS);
+	});
+
+	it('falls back to the 14-day cache default', async () => {
+		const writes: Array<Array<unknown>> = [];
+		const cache = createPostgresIpInfoCache({client: recordingClient(writes)});
+		await cache.set('fallback', {ok: true});
+		await cache.set('zero', {ok: true}, 0);
+		await cache.set('short', {ok: true}, 60);
+		expect(writes.map((values) => values[0])).toEqual(['ipinfo_cache', 'ipinfo_cache', 'ipinfo_cache']);
+		expectExpiresIn(writes[0], IPINFO_CACHE_TTL_SECONDS);
+		expectExpiresIn(writes[1], IPINFO_CACHE_TTL_SECONDS);
+		expectExpiresIn(writes[2], 60);
+	});
+});

fluxer_api/src/api/test/InMemoryCassandraQueryExecutor.ts

@@ -352,7 +352,16 @@ export class InMemoryCassandraQueryExecutor implements CassandraQueryExecutorFor
 		let rows = [...this.table(meta).values()].filter((row) => matchesWhere(row, meta.where, params));
 		if (meta.orderBy) {
 			const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
-			rows = rows.sort((a, b) => compareValues(a[meta.orderBy!.col], b[meta.orderBy!.col]) * direction);
+			const column = meta.orderBy.col as string;
+			const primaryKey = meta.table.primaryKey as ReadonlyArray<string>;
+			const columns = [column, ...primaryKey.slice(primaryKey.indexOf(column) + 1)];
+			rows = rows.sort((a, b) => {
+				for (const c of columns) {
+					const cmp = compareValues(a[c], b[c]);
+					if (cmp !== 0) return cmp * direction;
+				}
+				return 0;
+			});
 		}
 		if (typeof meta.limit === 'number') {
 			rows = rows.slice(0, meta.limit);

fluxer_api/src/api/worker/BlocklistFeedStartup.ts

@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {Logger} from '@app/api/Logger';
+import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
+import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError';
+import type {WorkerService} from '@app/api/worker/WorkerService';
+import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
+import {ms} from 'itty-time';
+
+const INITIAL_SYNC_KEY = 'sync:email_domains:initialized';
+const PURGE_KEY = 'sync:blocklist_feeds:purged';
+const CLAIM_TTL_SECONDS = ms('6 hours') / 1000;
+const FEED_TASKS = [
+	'syncDisposableEmailDomains',
+	'syncUrlBlocklists',
+	'syncFileShaBlocklists',
+] as const satisfies ReadonlyArray<WorkerTaskName>;
+
+export async function queueBlocklistFeedStartupJobs(
+	kvClient: Pick<IKVProvider, 'setnx' | 'del'>,
+	workerService: Pick<WorkerService, 'addJob'>,
+	enabled: boolean,
+): Promise<void> {
+	if (enabled) {
+		if (await kvClient.setnx(INITIAL_SYNC_KEY, '1', CLAIM_TTL_SECONDS)) {
+			Logger.info('Triggering initial disposable email domain sync');
+			await queueJobs(workerService, ['syncDisposableEmailDomains']);
+		}
+		return;
+	}
+	const wasEnabled = (await kvClient.del(INITIAL_SYNC_KEY)) > 0;
+	const claimed = await kvClient.setnx(PURGE_KEY, '1', CLAIM_TTL_SECONDS);
+	if (!wasEnabled && !claimed) return;
+	Logger.info('Removing blocklist feed data, blocklist feeds are disabled');
+	await queueJobs(workerService, FEED_TASKS);
+}
+
+async function queueJobs(
+	workerService: Pick<WorkerService, 'addJob'>,
+	tasks: ReadonlyArray<WorkerTaskName>,
+): Promise<void> {
+	for (const task of tasks) {
+		try {
+			await workerService.addJob(task, {});
+		} catch (error) {
+			if (!(error instanceof WorkerQueueOverflowError)) {
+				throw error;
+			}
+			Logger.warn({task}, 'Dropped blocklist feed job, jobs stream is at its limit');
+		}
+	}
+}

fluxer_api/src/api/worker/JetStreamWorkerQueue.ts

@@ -17,13 +17,13 @@ import {
 	StorageType,
 	type StreamConfig,
 } from '@nats-io/jetstream';
-import {nanos} from '@nats-io/transport-node';
+import {millis, nanos} from '@nats-io/transport-node';
 import type {JetStreamConnectionManager} from '@pkgs/nats/src/JetStreamConnectionManager';
 import type {WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes';
 
 const STREAM_NAME = 'JOBS';
 const SUBJECT_PREFIX = 'jobs.';
-const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+export const JOBS_STREAM_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
 const LEGACY_CONSUMER_NAME = 'workers';
 const DLQ_STREAM_NAME = 'JOBS_DLQ';
 const DLQ_SUBJECT_PREFIX = 'dlq.';
@@ -59,7 +59,7 @@ const JOBS_STREAM: WorkerStreamDefinition = {
 	name: STREAM_NAME,
 	subject: `${SUBJECT_PREFIX}>`,
 	retention: RetentionPolicy.Workqueue,
-	maxAgeMs: MAX_AGE_MS,
+	maxAgeMs: JOBS_STREAM_MAX_AGE_MS,
 	minBytes: STREAM_MIN_BYTES,
 	maxMessages: STREAM_MAX_MSGS,
 	maxMessagesPerSubject: STREAM_MAX_MSGS_PER_SUBJECT,
@@ -106,6 +106,7 @@ export class JetStreamWorkerQueue {
 	private consumersReady = false;
 	private streamSetup: Promise<void> | null = null;
 	private dlqStreamSetup: Promise<void> | null = null;
+	private jobsStreamMaxAgeMs = JOBS_STREAM_MAX_AGE_MS;
 
 	constructor(connectionManager: JetStreamConnectionManager) {
 		this.connectionManager = connectionManager;
@@ -125,11 +126,16 @@ export class JetStreamWorkerQueue {
 		if (existingConfig === null) {
 			await this.addStream(jsm);
 		} else {
+			this.jobsStreamMaxAgeMs = millis(existingConfig.max_age);
 			await this.applyStreamLimits(jsm, existingConfig);
 		}
 		this.streamReady = true;
 	}
 
+	getJobsStreamMaxAgeMs(): number {
+		return this.jobsStreamMaxAgeMs;
+	}
+
 	private async oversizedSubjects(jsm: JetStreamManager): Promise<Array<[string, number]> | null> {
 		try {
 			const info = await jsm.streams.info(STREAM_NAME, {subjects_filter: JOBS_STREAM.subject});
@@ -462,7 +468,7 @@ export class JetStreamWorkerQueue {
 			priority?: number;
 			jobKey?: string;
 		},
-	): Promise<string> {
+	): Promise<{seq: string; duplicate: boolean}> {
 		const js = this.connectionManager.getJetStreamClient();
 		const subject = `${SUBJECT_PREFIX}${taskType}`;
 		const body = JSON.stringify({
@@ -477,8 +483,7 @@ export class JetStreamWorkerQueue {
 			const ack = await js.publish(subject, body, {
 				msgID,
 			});
-			const jobId = `${ack.seq}`;
-			return jobId;
+			return {seq: `${ack.seq}`, duplicate: ack.duplicate === true};
 		} catch (error) {
 			const rejection = describeStreamRejection(error);
 			if (rejection === null) {

fluxer_api/src/api/worker/WorkerLaneConfig.ts

@@ -67,6 +67,7 @@ const LANE_CONFIG = {
 		consumerName: 'workers_batch',
 		tasks: [
 			'expireAttachments',
+			'expireStaleJobs',
 			'indexChannelMessages',
 			'indexGuildMembers',
 			'processAssetDeletionQueue',

fluxer_api/src/api/worker/WorkerMain.ts

@@ -23,8 +23,9 @@ import {
 } from '@app/api/middleware/ServiceSingletons';
 import {initializeSearch, shutdownSearch} from '@app/api/SearchFactory';
 import {awaitAll} from '@app/api/utils/ConcurrencyUtils';
+import {queueBlocklistFeedStartupJobs} from '@app/api/worker/BlocklistFeedStartup';
 import {CronScheduler} from '@app/api/worker/CronScheduler';
-import {JetStreamWorkerQueue} from '@app/api/worker/JetStreamWorkerQueue';
+import {JetStreamWorkerQueue, JOBS_STREAM_MAX_AGE_MS} from '@app/api/worker/JetStreamWorkerQueue';
 import {clearWorkerDependencies, setWorkerDependencies} from '@app/api/worker/WorkerContext';
 import {initializeWorkerDependencies, type WorkerDependencies} from '@app/api/worker/WorkerDependencies';
 import {WorkerHeartbeat} from '@app/api/worker/WorkerHeartbeat';
@@ -34,7 +35,6 @@ import {
 	validateLaneCompleteness,
 } from '@app/api/worker/WorkerLaneConfig';
 import {createWorkerProcessErrorHandler} from '@app/api/worker/WorkerProcessErrorHandler';
-import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError';
 import {WorkerRunner} from '@app/api/worker/WorkerRunner';
 import {WorkerService} from '@app/api/worker/WorkerService';
 import {workerTasks} from '@app/api/worker/WorkerTaskRegistry';
@@ -43,9 +43,8 @@ import {BACKGROUND_READ_TIMEOUT_MS, initCassandra, shutdownCassandra} from '@pkg
 import {JetStreamConnectionManager} from '@pkgs/nats/src/JetStreamConnectionManager';
 import {getDefaultPostgresClient, initPostgres, shutdownPostgres} from '@pkgs/postgres/src/Client';
 import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
-import {ms} from 'itty-time';
 
-function registerCronJobs(cron: CronScheduler): void {
+function registerCronJobs(cron: CronScheduler, jobsStreamMaxAgeMs: number): void {
 	cron.upsert('processAssetDeletionQueue', 'processAssetDeletionQueue', {}, '0 */5 * * * *', {ledger: false});
 	if (Config.cachePurge.adapter !== 'none') {
 		cron.upsert('processCachePurgeQueue', 'processCachePurgeQueue', {}, '*/10 * * * * *', {ledger: false});
@@ -62,6 +61,14 @@ function registerCronJobs(cron: CronScheduler): void {
 	}
 	cron.upsert('processInactivityDeletions', 'processInactivityDeletions', {}, '0 0 */6 * * *', {ledger: false});
 	cron.upsert('expireAttachments', 'expireAttachments', {}, '0 0 */12 * * *', {ledger: false});
+	if (jobsStreamMaxAgeMs > 0 && jobsStreamMaxAgeMs <= JOBS_STREAM_MAX_AGE_MS) {
+		cron.upsert('expireStaleJobs', 'expireStaleJobs', {}, '0 45 3 * * *', {ledger: false});
+	} else {
+		Logger.warn(
+			{jobsStreamMaxAgeMs},
+			'Jobs stream keeps jobs past 7 days, stale jobs stay active until their ledger rows expire',
+		);
+	}
 	cron.upsert('prunePostgresKvTtl', 'prunePostgresKvTtl', {}, '0 */5 * * * *', {ledger: false});
 	cron.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *', {ledger: false});
 	if (Config.blocklistFeeds.enabled) {
@@ -240,26 +247,9 @@ export async function startWorkerMain(): Promise<void> {
 		}
 		dependencies = await initializeWorkerDependencies(snowflakeService);
 		setWorkerDependencies(dependencies);
-		if (Config.blocklistFeeds.enabled) {
-			const didClaimEmailSync = await dependencies.kvClient.setnx(
-				'sync:email_domains:initialized',
-				'1',
-				ms('6 hours') / 1000,
-			);
-			if (didClaimEmailSync) {
-				Logger.info('Triggering initial disposable email domain sync');
-				try {
-					await workerService.addJob('syncDisposableEmailDomains', {});
-				} catch (error) {
-					if (!(error instanceof WorkerQueueOverflowError)) {
-						throw error;
-					}
-					Logger.warn('Dropped initial disposable email domain sync, jobs stream is at its limit');
-				}
-			}
-		}
+		await queueBlocklistFeedStartupJobs(dependencies.kvClient, workerService, Config.blocklistFeeds.enabled);
 		cron = new CronScheduler(workerService, Logger, dependencies.kvClient, heartbeat);
-		registerCronJobs(cron);
+		registerCronJobs(cron, queue.getJobsStreamMaxAgeMs());
 		for (const lane of activeWorkerLanes) {
 			const laneTasks: Record<string, WorkerTaskHandler> = {};
 			for (const taskType of lane.taskTypes) {

fluxer_api/src/api/worker/WorkerService.ts

@@ -29,10 +29,10 @@ export class WorkerService implements IWorkerService<WorkerTaskName> {
 		const skipLedger = options?.skipLedger === true;
 		const requireLedger = options?.requireLedger === true;
 		const payloadRecord = payload as Record<string, unknown>;
-		let ledgerWritten = false;
+		let ledgerCreatedAt: Date | null = null;
 		if (!skipLedger) {
 			try {
-				await this.ledger.createJob({
+				ledgerCreatedAt = await this.ledger.createJob({
 					jobId,
 					taskType,
 					payload: payloadRecord,
@@ -43,29 +43,32 @@ export class WorkerService implements IWorkerService<WorkerTaskName> {
 					jetStreamLane: findLaneForTask(taskType),
 					jetStreamSeq: null,
 				});
-				ledgerWritten = true;
 			} catch (ledgerErr) {
 				Logger.error({err: ledgerErr, jobId: jobId.toString(), taskType}, 'Failed to write ledger row for job');
 				if (requireLedger) throw ledgerErr;
 			}
 		}
-		const enrichedPayload = ledgerWritten ? {...payloadRecord, __jobId: jobId.toString()} : payloadRecord;
+		const enrichedPayload = ledgerCreatedAt !== null ? {...payloadRecord, __jobId: jobId.toString()} : payloadRecord;
 		try {
-			const seq = await this.queue.enqueue(taskType, enrichedPayload, {
+			const {seq, duplicate} = await this.queue.enqueue(taskType, enrichedPayload, {
 				...(options?.runAt !== undefined && {runAt: options.runAt}),
 				...(options?.maxAttempts !== undefined && {maxAttempts: options.maxAttempts}),
 				...(options?.priority !== undefined && {priority: options.priority}),
 				...(options?.jobKey !== undefined && {jobKey: options.jobKey}),
 			});
-			if (ledgerWritten) {
+			if (ledgerCreatedAt !== null && duplicate) {
+				await this.ledger
+					.discardJob(jobId, ledgerCreatedAt)
+					.catch((err) => Logger.warn({err, jobId: jobId.toString()}, 'Ledger discardJob failed'));
+			} else if (ledgerCreatedAt !== null) {
 				await this.ledger
 					.setJetStreamSeq(jobId, seq)
 					.catch((err) => Logger.warn({err, jobId: jobId.toString()}, 'Ledger setJetStreamSeq failed'));
 			}
-			Logger.debug({taskType, jobId: jobId.toString(), seq}, 'Job queued successfully');
+			Logger.debug({taskType, jobId: jobId.toString(), seq, duplicate}, 'Job queued successfully');
 			return jobId;
 		} catch (error) {
-			if (ledgerWritten) {
+			if (ledgerCreatedAt !== null) {
 				await this.ledger
 					.markDeadletter(jobId, error instanceof Error ? error.message : String(error))
 					.catch((err) => Logger.warn({err, jobId: jobId.toString()}, 'Ledger markDeadletter failed'));

fluxer_api/src/api/worker/WorkerTaskRegistry.ts

@@ -14,6 +14,7 @@ import bulkDeleteUserMessages from '@app/api/worker/tasks/BulkDeleteUserMessages
 import bulkDeleteUserMessagesScoped from '@app/api/worker/tasks/BulkDeleteUserMessagesScoped';
 import deleteUserMessagesInGuildByTime from '@app/api/worker/tasks/DeleteUserMessagesInGuildByTime';
 import expireAttachments from '@app/api/worker/tasks/ExpireAttachments';
+import expireStaleJobs from '@app/api/worker/tasks/ExpireStaleJobs';
 import extractEmbeds from '@app/api/worker/tasks/ExtractEmbeds';
 import finalizeNcmecAttachmentReport from '@app/api/worker/tasks/FinalizeNcmecAttachmentReport';
 import flushUserActivityBuffer from '@app/api/worker/tasks/FlushUserActivityBuffer';
@@ -59,6 +60,7 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
 	bulkUpdateUserFlags: bulkUpdateUserFlags,
 	deleteUserMessagesInGuildByTime,
 	expireAttachments,
+	expireStaleJobs,
 	extractEmbeds,
 	finalizeNcmecAttachmentReport,
 	handleMentions,

fluxer_api/src/api/worker/tasks/ExpireStaleJobs.ts

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {JOB_STALE_AFTER_MS, JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
+import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+
+const PAGE_SIZE = 500;
+const MAX_CLEARED_PER_RUN = 1000;
+
+const expireStaleJobs: WorkerTaskHandler = async (_payload, helpers) => {
+	const result = await new JobLedgerRepository().expireStaleActiveJobs({
+		staleBeforeMs: Date.now() - JOB_STALE_AFTER_MS,
+		pageSize: PAGE_SIZE,
+		maxCleared: MAX_CLEARED_PER_RUN,
+	});
+	if (!result.complete) {
+		helpers.logger.warn({...result}, 'Stale job sweep reached its per-run cap');
+	} else if (result.cleared > 0) {
+		helpers.logger.info({...result}, 'Expired stale jobs');
+	}
+};
+
+export default expireStaleJobs;

fluxer_api/src/api/worker/tasks/PrunePostgresKvTtl.ts

@@ -1,18 +1,34 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {Config} from '@app/api/Config';
+import {expireLegacyDefaultTtlRows} from '@app/api/database/PostgresKvDefaultTtlExpiry';
 import {pruneExpiredPostgresKvRows} from '@app/api/database/PostgresKvQueryExecutor';
+import {expireLegacyJobLedgerRows} from '@app/api/jobs/PostgresJobLedgerExpiry';
 import {getDefaultPostgresClient} from '@pkgs/postgres/src/Client';
 import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+import {ms} from 'itty-time';
 
 const PRUNE_BATCH_SIZE = 5000;
 const MAX_PRUNE_BATCHES_PER_RUN = 20;
+const LEGACY_EXPIRY_BUDGET_MS = ms('2 minutes');
 
 const prunePostgresKvTtl: WorkerTaskHandler = async (_payload, helpers) => {
 	if (Config.database.backend !== 'postgres') {
 		return;
 	}
 	const client = getDefaultPostgresClient();
+	const deadlineMs = Date.now() + LEGACY_EXPIRY_BUDGET_MS;
+	const legacyJobs = await expireLegacyJobLedgerRows(client, deadlineMs);
+	if (legacyJobs !== null && (legacyJobs.deleted > 0 || legacyJobs.expiring > 0 || !legacyJobs.complete)) {
+		helpers.logger.info({...legacyJobs}, 'Expired legacy job ledger rows');
+	}
+	const legacyDefaults = await expireLegacyDefaultTtlRows(client, deadlineMs);
+	if (
+		legacyDefaults !== null &&
+		(legacyDefaults.deleted > 0 || legacyDefaults.expiring > 0 || !legacyDefaults.complete)
+	) {
+		helpers.logger.info({...legacyDefaults}, 'Expired rows written without their table default TTL');
+	}
 	let deleted = 0;
 	for (let batch = 0; batch < MAX_PRUNE_BATCHES_PER_RUN; batch += 1) {
 		const batchDeleted = await pruneExpiredPostgresKvRows(client, PRUNE_BATCH_SIZE);

fluxer_api/src/api/worker/tasks/SyncDisposableEmailDomains.ts

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {domainToASCII} from 'node:url';
+import {Config} from '@app/api/Config';
 import {isAccountPolicyContactDomainReputationExempt} from '@app/api/risk/AccountPolicyService';
 import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits';
 import * as FetchUtils from '@app/api/utils/FetchUtils';
@@ -131,10 +132,7 @@ async function throwIfCancelled(helpers: WorkerTaskHelpers): Promise<void> {
 	}
 }
 
-const syncDisposableEmailDomains: WorkerTaskHandler = async (_payload, helpers) => {
-	helpers.logger.info('Starting disposable email domain sync');
-	await helpers.setContextLink('/suspicious-email-domains');
-	const {adminRepository} = getWorkerDependencies();
+async function fetchFeedDomains(helpers: WorkerTaskHelpers): Promise<Set<string>> {
 	const freshSet = new Set<string>();
 	const perSourceCounts: Record<string, number> = {};
 	const perSourceRawCounts: Record<string, number> = {};
@@ -164,6 +162,14 @@ const syncDisposableEmailDomains: WorkerTaskHandler = async (_payload, helpers)
 		},
 		'Fetched disposable email domains from all sources',
 	);
+	return freshSet;
+}
+
+const syncDisposableEmailDomains: WorkerTaskHandler = async (_payload, helpers) => {
+	helpers.logger.info('Starting disposable email domain sync');
+	await helpers.setContextLink('/suspicious-email-domains');
+	const {adminRepository} = getWorkerDependencies();
+	const freshSet = Config.blocklistFeeds.enabled ? await fetchFeedDomains(helpers) : new Set<string>();
 	const currentSet = await loadCurrentDisposableEmailDomains();
 	let addCount = 0;
 	for (const domain of freshSet) {

fluxer_api/src/api/worker/tasks/SyncFileShaBlocklists.ts

@@ -1,16 +1,39 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import {BANNED_FILE_SHAS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
+import {Config} from '@app/api/Config';
+import {
+	BANNED_FILE_SHAS_REFRESH_CHANNEL,
+	ContentBlocklistCategory,
+	isBlocklistFeedFileSha,
+} from '@app/api/constants/ContentModeration';
 import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits';
 import * as FetchUtils from '@app/api/utils/FetchUtils';
 import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
-import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+import type {WorkerTaskHandler, WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
 
 const MALWARE_BAZAAR_SHA256_URL = 'https://bazaar.abuse.ch/export/txt/sha256/recent/';
 const SHA256_RE = /^[0-9a-fA-F]{64}$/;
+
+async function removeFeedFileShas(helpers: WorkerTaskHelpers): Promise<void> {
+	const {adminRepository, kvClient} = getWorkerDependencies();
+	let removed = 0;
+	for (const row of await adminRepository.loadAllBannedFileShas()) {
+		if (!isBlocklistFeedFileSha(row)) continue;
+		if (await adminRepository.unbanFeedFileSha(row.sha256_hex)) removed++;
+	}
+	if (removed > 0) {
+		await kvClient.publish(BANNED_FILE_SHAS_REFRESH_CHANNEL, 'refresh');
+	}
+	helpers.logger.info({removed}, 'Removed file-SHA blocklist feed rows');
+}
+
 const syncFileShaBlocklists: WorkerTaskHandler = async (_payload, helpers) => {
 	helpers.logger.info('Starting file-SHA blocklist sync');
 	await helpers.setContextLink('/file-sha-bans');
+	if (!Config.blocklistFeeds.enabled) {
+		await removeFeedFileShas(helpers);
+		return;
+	}
 	const {adminRepository, kvClient} = getWorkerDependencies();
 	let added = 0;
 	try {
@@ -38,7 +61,7 @@ const syncFileShaBlocklists: WorkerTaskHandler = async (_payload, helpers) => {
 			if (existingSet.has(sha)) continue;
 			await adminRepository.banFileSha({
 				sha256_hex: sha,
-				category: 'malware_bazaar',
+				category: ContentBlocklistCategory.MALWARE_BAZAAR,
 				severity: 2,
 				content_type: null,
 				source_url: MALWARE_BAZAAR_SHA256_URL,

fluxer_api/src/api/worker/tasks/SyncUrlBlocklists.ts

@@ -1,7 +1,8 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import {Config} from '@app/api/Config';
 import {BANNED_URLS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
-import {RISK_S3_KEYS, writeLinesToS3} from '@app/api/risk/RiskBlocklistS3';
+import {deleteRiskS3Object, RISK_S3_KEYS, writeLinesToS3} from '@app/api/risk/RiskBlocklistS3';
 import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits';
 import * as FetchUtils from '@app/api/utils/FetchUtils';
 import {canonicalizeUrl} from '@app/api/utils/UrlNormalizer';
@@ -62,10 +63,23 @@ async function fetchFeed(source: FeedSource): Promise<Array<string>> {
 	return source.parse(text);
 }
 
+const MISSING_FEED_FILE_ERRORS = new Set(['NoSuchBucket', 'NoSuchKey', 'NotFound']);
+
 const syncUrlBlocklists: WorkerTaskHandler = async (_payload, helpers) => {
 	helpers.logger.info('Starting URL blocklist sync');
 	await helpers.setContextLink('/url-domain-bans');
 	const {storageService, kvClient} = getWorkerDependencies();
+	if (!Config.blocklistFeeds.enabled) {
+		try {
+			await deleteRiskS3Object(storageService, RISK_S3_KEYS.feedUrls);
+		} catch (error) {
+			if (!(error instanceof Error && MISSING_FEED_FILE_ERRORS.has(error.name))) {
+				helpers.logger.warn({error}, 'Failed to delete the URL blocklist feed file');
+			}
+		}
+		await kvClient.publish(BANNED_URLS_REFRESH_CHANNEL, 'refresh');
+		return;
+	}
 	const results = await Promise.allSettled(
 		FEED_SOURCES.map(async (source) => ({source, rawUrls: await fetchFeed(source)})),
 	);

fluxer_api/src/api/worker/tests/BlocklistFeedStartup.test.ts

@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {MockKVProvider} from '@app/api/test/mocks/MockKVProvider';
+import {queueBlocklistFeedStartupJobs} from '@app/api/worker/BlocklistFeedStartup';
+import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
+import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError';
+import type {WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes';
+import {describe, expect, it, vi} from 'vitest';
+
+const INITIAL_SYNC_KEY = 'sync:email_domains:initialized';
+const FEED_TASKS = ['syncDisposableEmailDomains', 'syncUrlBlocklists', 'syncFileShaBlocklists'];
+
+function createWorkerService() {
+	return {addJob: vi.fn(async (_task: WorkerTaskName, _payload: WorkerJobPayload) => 1n)};
+}
+
+function queuedTasks(workerService: ReturnType<typeof createWorkerService>): Array<string> {
+	return workerService.addJob.mock.calls.map(([task]) => task);
+}
+
+describe('queueBlocklistFeedStartupJobs', () => {
+	it('with feeds on, a fresh start queues only the disposable sync and claims it for six hours', async () => {
+		const kv = new MockKVProvider();
+		const workerService = createWorkerService();
+
+		await queueBlocklistFeedStartupJobs(kv, workerService, true);
+
+		expect(workerService.addJob.mock.calls).toEqual([['syncDisposableEmailDomains', {}]]);
+		expect(kv.setnxSpy.mock.calls).toEqual([[INITIAL_SYNC_KEY, '1', 21600]]);
+		expect(kv.delSpy).not.toHaveBeenCalled();
+	});
+
+	it('with feeds on, a start inside the claim queues nothing', async () => {
+		const kv = new MockKVProvider();
+		const workerService = createWorkerService();
+
+		await queueBlocklistFeedStartupJobs(kv, workerService, true);
+		await queueBlocklistFeedStartupJobs(kv, workerService, true);
+
+		expect(queuedTasks(workerService)).toEqual(['syncDisposableEmailDomains']);
+	});
+
+	it('with feeds off, the first start queues all three feed tasks and a second start queues none', async () => {
+		const kv = new MockKVProvider();
+		const workerService = createWorkerService();
+
+		await queueBlocklistFeedStartupJobs(kv, workerService, false);
+		expect(workerService.addJob.mock.calls).toEqual(FEED_TASKS.map((task) => [task, {}]));
+
+		await queueBlocklistFeedStartupJobs(kv, workerService, false);
+		expect(queuedTasks(workerService)).toEqual(FEED_TASKS);
+	});
+
+	it('with feeds off, a start after a feeds-on start cleans up again inside the claim', async () => {
+		const kv = new MockKVProvider();
+		const workerService = createWorkerService();
+
+		await queueBlocklistFeedStartupJobs(kv, workerService, false);
+		await queueBlocklistFeedStartupJobs(kv, workerService, true);
+		await queueBlocklistFeedStartupJobs(kv, workerService, false);
+
+		expect(queuedTasks(workerService)).toEqual([...FEED_TASKS, 'syncDisposableEmailDomains', ...FEED_TASKS]);
+		expect(await kv.exists(INITIAL_SYNC_KEY)).toBe(0);
+	});
+
+	it('a legacy initial sync key without expiry is cleared, so re-enabling runs the initial sync', async () => {
+		const kv = new MockKVProvider();
+		await kv.set(INITIAL_SYNC_KEY, '1');
+		expect(await kv.ttl(INITIAL_SYNC_KEY)).toBe(-1);
+
+		await queueBlocklistFeedStartupJobs(kv, createWorkerService(), false);
+		expect(await kv.exists(INITIAL_SYNC_KEY)).toBe(0);
+
+		const workerService = createWorkerService();
+		await queueBlocklistFeedStartupJobs(kv, workerService, true);
+		expect(queuedTasks(workerService)).toEqual(['syncDisposableEmailDomains']);
+	});
+
+	it('a full jobs stream drops the job without failing startup', async () => {
+		const overflowing = createWorkerService();
+		overflowing.addJob.mockImplementation(async (task) => {
+			throw new WorkerQueueOverflowError(task, 'maximum messages exceeded');
+		});
+		await expect(queueBlocklistFeedStartupJobs(new MockKVProvider(), overflowing, false)).resolves.toBeUndefined();
+		expect(queuedTasks(overflowing)).toEqual(FEED_TASKS);
+
+		const failing = createWorkerService();
+		failing.addJob.mockRejectedValue(new Error('jetstream unavailable'));
+		await expect(queueBlocklistFeedStartupJobs(new MockKVProvider(), failing, false)).rejects.toThrow(
+			'jetstream unavailable',
+		);
+	});
+});

fluxer_api/src/api/worker/tests/BlocklistFeedsDisabled.test.ts

@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {Config} from '@app/api/Config';
+import {
+	BANNED_FILE_SHAS_REFRESH_CHANNEL,
+	BANNED_URLS_REFRESH_CHANNEL,
+	ContentBlocklistCategory,
+} from '@app/api/constants/ContentModeration';
+import type {BannedFileShaRow} from '@app/api/database/types/AdminArchiveTypes';
+import {fileShaCache} from '@app/api/middleware/FileShaCache';
+import {getAdminRepository} from '@app/api/middleware/ServiceSingletons';
+import {urlBlocklistCache} from '@app/api/middleware/UrlBlocklistCache';
+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
+import type {MockKVProvider} from '@app/api/test/mocks/MockKVProvider';
+import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
+import {canonicalizeUrl} from '@app/api/utils/UrlNormalizer';
+import syncDisposableEmailDomains from '@app/api/worker/tasks/SyncDisposableEmailDomains';
+import syncFileShaBlocklists from '@app/api/worker/tasks/SyncFileShaBlocklists';
+import syncUrlBlocklists from '@app/api/worker/tasks/SyncUrlBlocklists';
+import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/worker/WorkerContext';
+import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
+import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, type MockInstance, vi} from 'vitest';
+
+const FEED_BUCKET = 'fluxer-geoip';
+const FEED_KEY = 'blocklists/feed-urls.txt';
+const FEED_SHA = 'a1'.repeat(32);
+const ADMIN_BAZAAR_SHA = 'b2'.repeat(32);
+const ADMIN_MANUAL_SHA = 'c3'.repeat(32);
+const UNOWNED_NCMEC_SHA = 'd4'.repeat(32);
+const ADMIN_USER_ID = 42n;
+
+function createHelpers(overrides: Partial<WorkerTaskHelpers> = {}): WorkerTaskHelpers {
+	return {
+		logger: new NoopLogger(),
+		jobId: 4242n,
+		addJob: async () => 0n,
+		reportProgress: async () => {},
+		shouldCancel: async () => false,
+		setContextLink: async () => {},
+		...overrides,
+	};
+}
+
+function fileShaRow(sha256Hex: string, category: string, addedBy: bigint | null): BannedFileShaRow {
+	return {
+		sha256_hex: sha256Hex,
+		category,
+		severity: 2,
+		content_type: null,
+		source_url: null,
+		added_at: new Date(),
+		added_by: addedBy,
+		notes: null,
+	};
+}
+
+async function seedFileShas(): Promise<void> {
+	const adminRepository = getAdminRepository();
+	await adminRepository.banFileSha(fileShaRow(FEED_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, null));
+	await adminRepository.banFileSha(
+		fileShaRow(ADMIN_BAZAAR_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, ADMIN_USER_ID),
+	);
+	await adminRepository.banFileSha(fileShaRow(ADMIN_MANUAL_SHA, ContentBlocklistCategory.MANUAL, ADMIN_USER_ID));
+	await adminRepository.banFileSha(fileShaRow(UNOWNED_NCMEC_SHA, ContentBlocklistCategory.NCMEC, null));
+}
+
+describe('blocklist feeds turned off', () => {
+	let harness: ApiTestHarness;
+	let previousFeedsEnabled: boolean;
+	let fetchSpy: MockInstance<typeof fetch>;
+
+	beforeAll(async () => {
+		harness = await createApiTestHarness();
+	});
+
+	beforeEach(async () => {
+		previousFeedsEnabled = Config.blocklistFeeds.enabled;
+		await harness.reset();
+		harness.storageService.reset();
+		setWorkerDependenciesForTest({
+			adminRepository: getAdminRepository(),
+			kvClient: harness.kvProvider,
+			storageService: harness.storageService,
+		});
+		fetchSpy = vi.spyOn(globalThis, 'fetch');
+	});
+
+	afterEach(() => {
+		Config.blocklistFeeds.enabled = previousFeedsEnabled;
+		fetchSpy.mockRestore();
+	});
+
+	afterAll(async () => {
+		clearWorkerDependencies();
+		await harness?.shutdown();
+	});
+
+	function publishCalls(): Array<Array<string>> {
+		return (harness.kvProvider as MockKVProvider).publishSpy.mock.calls;
+	}
+
+	it('removes every stored disposable domain without fetching a feed', async () => {
+		const adminRepository = getAdminRepository();
+		for (const domain of ['mailinator.com', 'guerrillamail.com', 'tempmail.dev']) {
+			await adminRepository.addDisposableEmailDomain(domain);
+		}
+		Config.blocklistFeeds.enabled = false;
+		const reportProgress = vi.fn(async () => {});
+
+		await syncDisposableEmailDomains({}, createHelpers({reportProgress}));
+
+		expect(await adminRepository.listDisposableEmailDomains()).toEqual([]);
+		expect(fetchSpy).not.toHaveBeenCalled();
+		expect(reportProgress).toHaveBeenLastCalledWith(3, 3, '+0 added, -3 removed');
+	});
+
+	it('reports a stored domain as disposable only while feeds are on', async () => {
+		const adminRepository = getAdminRepository();
+		await adminRepository.addDisposableEmailDomain('mailinator.com');
+
+		Config.blocklistFeeds.enabled = true;
+		expect(await adminRepository.isEmailDomainDisposable('mailinator.com')).toBe(true);
+		Config.blocklistFeeds.enabled = false;
+		expect(await adminRepository.isEmailDomainDisposable('mailinator.com')).toBe(false);
+	});
+
+	it('removes MalwareBazaar feed rows and keeps every other file-SHA ban', async () => {
+		await seedFileShas();
+		Config.blocklistFeeds.enabled = false;
+
+		await syncFileShaBlocklists({}, createHelpers());
+
+		const remaining = (await getAdminRepository().loadAllBannedFileShas()).map((row) => row.sha256_hex).sort();
+		expect(remaining).toEqual([ADMIN_BAZAAR_SHA, ADMIN_MANUAL_SHA, UNOWNED_NCMEC_SHA]);
+		expect(publishCalls()).toEqual([[BANNED_FILE_SHAS_REFRESH_CHANNEL, 'refresh']]);
+		expect(fetchSpy).not.toHaveBeenCalled();
+	});
+
+	it('keeps a file-SHA ban an Admin took over after the purge read the table', async () => {
+		const adminRepository = getAdminRepository();
+		await adminRepository.banFileSha(fileShaRow(FEED_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, null));
+		const staleRows = await adminRepository.loadAllBannedFileShas();
+		await adminRepository.banFileSha(fileShaRow(FEED_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, ADMIN_USER_ID));
+		const staleRead = vi.spyOn(adminRepository, 'loadAllBannedFileShas').mockResolvedValueOnce(staleRows);
+		Config.blocklistFeeds.enabled = false;
+
+		await syncFileShaBlocklists({}, createHelpers());
+
+		staleRead.mockRestore();
+		expect(await adminRepository.isFileShaBanned(FEED_SHA)).toBe(true);
+	});
+
+	it('file-SHA cache skips feed rows only while feeds are off', async () => {
+		await seedFileShas();
+
+		Config.blocklistFeeds.enabled = false;
+		await fileShaCache.refresh();
+		expect(fileShaCache.isBanned(FEED_SHA)).toBe(false);
+		expect(fileShaCache.isBanned(ADMIN_BAZAAR_SHA)).toBe(true);
+		expect(fileShaCache.isBanned(ADMIN_MANUAL_SHA)).toBe(true);
+		expect(fileShaCache.isBanned(UNOWNED_NCMEC_SHA)).toBe(true);
+
+		Config.blocklistFeeds.enabled = true;
+		await fileShaCache.refresh();
+		expect(fileShaCache.isBanned(FEED_SHA)).toBe(true);
+	});
+
+	it('deletes the URL feed file and never fetches while feeds are off', async () => {
+		await harness.storageService.uploadObject({
+			bucket: FEED_BUCKET,
+			key: FEED_KEY,
+			body: Buffer.from('https://feed.example/x\n'),
+		});
+		Config.blocklistFeeds.enabled = false;
+
+		await syncUrlBlocklists({}, createHelpers());
+
+		expect(harness.storageService.deleteObjectSpy).toHaveBeenCalledWith(FEED_BUCKET, FEED_KEY);
+		expect(harness.storageService.hasObject(FEED_BUCKET, FEED_KEY)).toBe(false);
+		expect(publishCalls()).toEqual([[BANNED_URLS_REFRESH_CHANNEL, 'refresh']]);
+		expect(fetchSpy).not.toHaveBeenCalled();
+
+		const quietLogger = new NoopLogger();
+		const quietWarn = vi.spyOn(quietLogger, 'warn');
+		vi.spyOn(harness.storageService, 'deleteObject').mockRejectedValueOnce(
+			Object.assign(new Error('The specified bucket does not exist'), {name: 'NoSuchBucket'}),
+		);
+		await expect(syncUrlBlocklists({}, createHelpers({logger: quietLogger}))).resolves.toBeUndefined();
+		expect(quietWarn).not.toHaveBeenCalled();
+
+		const loudLogger = new NoopLogger();
+		const loudWarn = vi.spyOn(loudLogger, 'warn');
+		harness.storageService.configure({shouldFailDelete: true});
+		await expect(syncUrlBlocklists({}, createHelpers({logger: loudLogger}))).resolves.toBeUndefined();
+		expect(loudWarn).toHaveBeenCalledTimes(1);
+	});
+
+	it('URL cache ignores the feed file only while feeds are off', async () => {
+		const feedUrl = canonicalizeUrl('https://feed.example/x');
+		const adminUrl = canonicalizeUrl('https://admin.example/y');
+		expect(feedUrl).not.toBeNull();
+		expect(adminUrl).not.toBeNull();
+		await harness.storageService.uploadObject({
+			bucket: FEED_BUCKET,
+			key: FEED_KEY,
+			body: Buffer.from(`${feedUrl}\n`),
+		});
+		await getAdminRepository().banUrl({
+			url_canonical: adminUrl!,
+			category: ContentBlocklistCategory.MANUAL,
+			severity: 2,
+			source_url: null,
+			added_at: new Date(),
+			added_by: ADMIN_USER_ID,
+			notes: null,
+		});
+		urlBlocklistCache.setStorageService(harness.storageService);
+
+		Config.blocklistFeeds.enabled = false;
+		await urlBlocklistCache.refresh();
+		expect(urlBlocklistCache.isUrlBanned('https://feed.example/x')).toBe(false);
+		expect(urlBlocklistCache.isUrlBanned('https://admin.example/y')).toBe(true);
+
+		Config.blocklistFeeds.enabled = true;
+		await urlBlocklistCache.refresh();
+		expect(urlBlocklistCache.isUrlBanned('https://feed.example/x')).toBe(true);
+	});
+});

fluxer_api/src/api/worker/tests/BlocklistFeedsDisabledPostgres.test.ts

@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {AdminRepository} from '@app/api/admin/AdminRepository';
+import {Config} from '@app/api/Config';
+import {ContentBlocklistCategory} from '@app/api/constants/ContentModeration';
+import {setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
+import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';
+import type {BannedFileShaRow} from '@app/api/database/types/AdminArchiveTypes';
+import {startDockerContainer} from '@app/api/test/DockerTestContainer';
+import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
+import {MockKVProvider} from '@app/api/test/mocks/MockKVProvider';
+import {MockStorageService} from '@app/api/test/mocks/MockStorageService';
+import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
+import syncDisposableEmailDomains from '@app/api/worker/tasks/SyncDisposableEmailDomains';
+import syncFileShaBlocklists from '@app/api/worker/tasks/SyncFileShaBlocklists';
+import {
+	clearWorkerDependencies,
+	getWorkerDependencies,
+	setWorkerDependenciesForTest,
+} from '@app/api/worker/WorkerContext';
+import {
+	getDefaultPostgresClient,
+	type IPostgresClient,
+	initPostgres,
+	shutdownPostgres,
+} from '@pkgs/postgres/src/Client';
+import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
+import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';
+
+const KV_TABLE = 'kv_blocklist_feeds_disabled';
+const CONTAINER = `fluxer-feeds-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+const FEED_SHA = 'a1'.repeat(32);
+const ADMIN_BAZAAR_SHA = 'b2'.repeat(32);
+const ADMIN_MANUAL_SHA = 'c3'.repeat(32);
+const ADMIN_USER_ID = 1_234_567_890_123n;
+
+async function sleep(ms: number): Promise<void> {
+	await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function freePort(): Promise<number> {
+	return new Promise((resolve, reject) => {
+		const server = createServer();
+		server.on('error', reject);
+		server.listen(0, '127.0.0.1', () => {
+			const address = server.address();
+			if (typeof address === 'string' || address === null) {
+				reject(new Error('no port'));
+				return;
+			}
+			const port = address.port;
+			server.close(() => resolve(port));
+		});
+	});
+}
+
+function createHelpers(): WorkerTaskHelpers {
+	return {
+		logger: new NoopLogger(),
+		jobId: 4242n,
+		addJob: async () => 0n,
+		reportProgress: async () => {},
+		shouldCancel: async () => false,
+		setContextLink: async () => {},
+	};
+}
+
+function fileShaRow(sha256Hex: string, category: string, addedBy: bigint | null): BannedFileShaRow {
+	return {
+		sha256_hex: sha256Hex,
+		category,
+		severity: 2,
+		content_type: null,
+		source_url: null,
+		added_at: new Date(),
+		added_by: addedBy,
+		notes: null,
+	};
+}
+
+describe.skipIf(!dockerAvailable)('blocklist feeds turned off against postgres', () => {
+	let raw: IPostgresClient;
+	let executor: PostgresKvQueryExecutor;
+	let previousFeedsEnabled: boolean;
+
+	beforeAll(async () => {
+		const port = await freePort();
+		startDockerContainer([
+			'run',
+			'-d',
+			'--name',
+			CONTAINER,
+			'-e',
+			'POSTGRES_USER=fluxer',
+			'-e',
+			'POSTGRES_PASSWORD=fluxer',
+			'-e',
+			'POSTGRES_DB=fluxer',
+			'-p',
+			`127.0.0.1:${port}:5432`,
+			'postgres:16-alpine',
+			'-c',
+			'fsync=off',
+		]);
+		let ready = false;
+		for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+			await sleep(500);
+			const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
+				stdio: 'ignore',
+			});
+			if (probe.status !== 0) continue;
+			try {
+				await initPostgres({
+					url: `postgres://fluxer:[email protected]:${port}/fluxer`,
+					maxConnections: 4,
+					kvTable: KV_TABLE,
+				});
+				await getDefaultPostgresClient().query('SELECT 1');
+				ready = true;
+			} catch {
+				await shutdownPostgres().catch(() => {});
+			}
+		}
+		if (!ready) throw new Error('postgres never came up');
+		raw = getDefaultPostgresClient();
+		await ensurePostgresKvSchema(raw);
+		executor = new PostgresKvQueryExecutor(raw);
+	}, 900_000);
+
+	beforeEach(async () => {
+		previousFeedsEnabled = Config.blocklistFeeds.enabled;
+		await raw.query(`DELETE FROM ${KV_TABLE}`);
+		setCassandraQueryExecutorForTesting(executor);
+		setWorkerDependenciesForTest({
+			adminRepository: new AdminRepository(),
+			kvClient: new MockKVProvider(),
+			storageService: new MockStorageService(),
+		});
+		Config.blocklistFeeds.enabled = false;
+	});
+
+	afterEach(() => {
+		Config.blocklistFeeds.enabled = previousFeedsEnabled;
+	});
+
+	afterAll(async () => {
+		clearWorkerDependencies();
+		setCassandraQueryExecutorForTesting(new InMemoryCassandraQueryExecutor());
+		await shutdownPostgres().catch(() => {});
+		spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+	});
+
+	it('removes every disposable_email_domains row from the KV table', async () => {
+		const repository = new AdminRepository();
+		for (let i = 0; i < 1200; i++) {
+			await repository.addDisposableEmailDomain(`disposable-${i}.example`);
+		}
+		const seeded = await raw.query<{count: number}>(
+			`SELECT count(*)::int AS count FROM ${KV_TABLE} WHERE table_name = 'disposable_email_domains'`,
+		);
+		expect(seeded.rows[0]?.count).toBe(1200);
+
+		await syncDisposableEmailDomains({}, createHelpers());
+
+		const remaining = await raw.query<{count: number}>(
+			`SELECT count(*)::int AS count FROM ${KV_TABLE} WHERE table_name = 'disposable_email_domains'`,
+		);
+		expect(remaining.rows[0]?.count).toBe(0);
+	});
+
+	it('keeps file-SHA rows with added_by set and removes feed rows stored with a JSON null added_by', async () => {
+		const repository = new AdminRepository();
+		await repository.banFileSha(fileShaRow(FEED_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, null));
+		await repository.banFileSha(fileShaRow(ADMIN_BAZAAR_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, ADMIN_USER_ID));
+		await repository.banFileSha(fileShaRow(ADMIN_MANUAL_SHA, ContentBlocklistCategory.MANUAL, ADMIN_USER_ID));
+		const feedRow = await raw.query<{added_by_type: string}>(
+			`SELECT jsonb_typeof(row_data->'added_by') AS added_by_type FROM ${KV_TABLE}
+WHERE table_name = 'banned_file_shas' AND row_data->>'sha256_hex' = $1`,
+			[FEED_SHA],
+		);
+		expect(feedRow.rows).toEqual([{added_by_type: 'null'}]);
+
+		await syncFileShaBlocklists({}, createHelpers());
+
+		const remaining = await raw.query<{sha: string}>(
+			`SELECT row_data->>'sha256_hex' AS sha FROM ${KV_TABLE} WHERE table_name = 'banned_file_shas' ORDER BY 1`,
+		);
+		expect(remaining.rows.map((row) => row.sha)).toEqual([ADMIN_BAZAAR_SHA, ADMIN_MANUAL_SHA]);
+	});
+
+	it('keeps a file-SHA ban an Admin took over after the purge read the table', async () => {
+		const {adminRepository} = getWorkerDependencies();
+		await adminRepository.banFileSha(fileShaRow(FEED_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, null));
+		const staleRows = await adminRepository.loadAllBannedFileShas();
+		await adminRepository.banFileSha(fileShaRow(FEED_SHA, ContentBlocklistCategory.MALWARE_BAZAAR, ADMIN_USER_ID));
+		const staleRead = vi.spyOn(adminRepository, 'loadAllBannedFileShas').mockResolvedValueOnce(staleRows);
+
+		await syncFileShaBlocklists({}, createHelpers());
+
+		staleRead.mockRestore();
+		expect(await adminRepository.isFileShaBanned(FEED_SHA)).toBe(true);
+	});
+});

fluxer_api/src/api/worker/tests/JetStreamWorkerQueue.test.ts

@@ -94,7 +94,7 @@ function createQueue(params: {
 	dlqExists?: boolean;
 	reject?: (config: Partial<StreamConfig>) => Error | null;
 	updateError?: Error;
-	publish?: (subject: string) => {seq: number};
+	publish?: (subject: string, body: string, options: {msgID: string}) => {seq: number; duplicate?: boolean};
 	subjectCounts?: Record<string, number>;
 	subjectCountsError?: Error;
 }): {
@@ -171,9 +171,9 @@ function createQueue(params: {
 				},
 			}),
 		getJetStreamClient: () => ({
-			publish: (subject: string) => {
+			publish: (subject: string, body: string, options: {msgID: string}) => {
 				const publish = params.publish ?? (() => ({seq: 1}));
-				return Promise.resolve(publish(subject));
+				return Promise.resolve(publish(subject, body, options));
 			},
 		}),
 	} as unknown as JetStreamConnectionManager;
@@ -281,6 +281,27 @@ describe('jobs stream limits', () => {
 	});
 });
 
+describe('jobs stream max age', () => {
+	it('reports the max age of a jobs stream it creates', async () => {
+		const {queue, added} = createQueue({existing: null});
+		await queue.ensureStream();
+		expect(queue.getJobsStreamMaxAgeMs()).toBe(7 * 24 * 60 * 60 * 1000);
+		expect(added[0]?.max_age).toBe(queue.getJobsStreamMaxAgeMs() * 1_000_000);
+	});
+
+	it('reports the max age an existing jobs stream really has and never changes it', async () => {
+		for (const [maxAgeNanos, expectedMs] of [
+			[30 * 24 * 60 * 60 * 1_000_000_000, 30 * 24 * 60 * 60 * 1000],
+			[0, 0],
+		] as const) {
+			const {queue, updated} = createQueue({existing: {...LEGACY_CONFIG, max_age: maxAgeNanos} as StreamConfig});
+			await queue.ensureStream();
+			expect(queue.getJobsStreamMaxAgeMs()).toBe(expectedMs);
+			expect(updated.some((config) => 'max_age' in config)).toBe(false);
+		}
+	});
+});
+
 describe('dead-letter stream', () => {
 	it('keeps startup alive when the dead-letter stream does not fit', async () => {
 		const {queue, dlqAdded} = createQueue({dlqExists: false, reject: () => noStorageError()});
@@ -298,16 +319,16 @@ describe('dead-letter stream', () => {
 describe('jobs stream enqueue shedding', () => {
 	it('rejects enqueues once the stream is at its cap', async () => {
 		const {queue} = createQueue({existing: LEGACY_CONFIG, publish: boundedPublisher(2)});
-		await expect(queue.enqueue('extractEmbeds', {})).resolves.toBe('1');
-		await expect(queue.enqueue('extractEmbeds', {})).resolves.toBe('2');
+		await expect(queue.enqueue('extractEmbeds', {})).resolves.toEqual({seq: '1', duplicate: false});
+		await expect(queue.enqueue('extractEmbeds', {})).resolves.toEqual({seq: '2', duplicate: false});
 		await expect(queue.enqueue('extractEmbeds', {})).rejects.toBeInstanceOf(WorkerQueueOverflowError);
 	});
 
 	it('caps each task type independently', async () => {
 		const {queue} = createQueue({existing: LEGACY_CONFIG, publish: boundedPublisher(1)});
-		await expect(queue.enqueue('extractEmbeds', {})).resolves.toBe('1');
+		await expect(queue.enqueue('extractEmbeds', {})).resolves.toEqual({seq: '1', duplicate: false});
 		await expect(queue.enqueue('extractEmbeds', {})).rejects.toBeInstanceOf(WorkerQueueOverflowError);
-		await expect(queue.enqueue('handleMentions', {})).resolves.toBe('2');
+		await expect(queue.enqueue('handleMentions', {})).resolves.toEqual({seq: '2', duplicate: false});
 	});
 
 	it('sheds enqueues the server refuses for lack of resources', async () => {
@@ -320,6 +341,29 @@ describe('jobs stream enqueue shedding', () => {
 		await expect(queue.enqueue('handleMentions', {})).rejects.toBeInstanceOf(WorkerQueueOverflowError);
 	});
 
+	it('reports a publish the stream deduplicated under the same job key', async () => {
+		const seen = new Map<string, number>();
+		const {queue} = createQueue({
+			existing: LEGACY_CONFIG,
+			publish: (_subject, _body, options) => {
+				const existing = seen.get(options.msgID);
+				if (existing !== undefined) return {seq: existing, duplicate: true};
+				seen.set(options.msgID, seen.size + 1);
+				return {seq: seen.size, duplicate: false};
+			},
+		});
+		const options = {jobKey: 'batch-audit-log-message-deletes:1'};
+		await expect(queue.enqueue('batchGuildAuditLogMessageDeletes', {}, options)).resolves.toEqual({
+			seq: '1',
+			duplicate: false,
+		});
+		await expect(queue.enqueue('batchGuildAuditLogMessageDeletes', {}, options)).resolves.toEqual({
+			seq: '1',
+			duplicate: true,
+		});
+		await expect(queue.enqueue('batchGuildAuditLogMessageDeletes', {})).resolves.toEqual({seq: '2', duplicate: false});
+	});
+
 	it('rethrows publish failures that are not stream limits', async () => {
 		const failure = new Error('no responders');
 		const {queue} = createQueue({

fluxer_api/src/api/worker/tests/WorkerServiceLedger.test.ts

@@ -7,6 +7,7 @@ import {WorkerService} from '@app/api/worker/WorkerService';
 import {describe, expect, test} from 'vitest';
 
 const JOB_ID = 4242n;
+const CREATED_AT = new Date('2026-09-21T12:00:00.000Z');
 
 function createSnowflake(): ISnowflakeService {
 	return {
@@ -14,17 +15,24 @@ function createSnowflake(): ISnowflakeService {
 	} as unknown as ISnowflakeService;
 }
 
-function createHarness(options?: {createJobError?: Error; enqueueError?: Error}) {
+function createHarness(options?: {
+	createJobError?: Error;
+	enqueueError?: Error;
+	duplicate?: boolean;
+	discardError?: Error;
+}) {
 	const calls: Array<string> = [];
 	const createdJobs: Array<CreateJobInput> = [];
 	const enqueued: Array<{taskType: string; payload: Record<string, unknown>}> = [];
 	const seqUpdates: Array<{jobId: bigint; seq: string}> = [];
 	const deadletters: Array<{jobId: bigint; errorMessage: string}> = [];
+	const discarded: Array<{jobId: bigint; createdAt: Date}> = [];
 	const ledger = {
 		createJob: async (input: CreateJobInput) => {
 			calls.push('createJob');
 			if (options?.createJobError) throw options.createJobError;
 			createdJobs.push(input);
+			return CREATED_AT;
 		},
 		setJetStreamSeq: async (jobId: bigint, seq: string) => {
 			calls.push('setJetStreamSeq');
@@ -34,17 +42,22 @@ function createHarness(options?: {createJobError?: Error; enqueueError?: Error})
 			calls.push('markDeadletter');
 			deadletters.push({jobId, errorMessage});
 		},
+		discardJob: async (jobId: bigint, createdAt: Date) => {
+			calls.push('discardJob');
+			if (options?.discardError) throw options.discardError;
+			discarded.push({jobId, createdAt});
+		},
 	} as unknown as IJobLedgerRepository;
 	const queue = {
 		enqueue: async (taskType: string, payload: Record<string, unknown>) => {
 			calls.push('enqueue');
 			if (options?.enqueueError) throw options.enqueueError;
 			enqueued.push({taskType, payload});
-			return 'seq-9';
+			return {seq: 'seq-9', duplicate: options?.duplicate === true};
 		},
 	} as unknown as JetStreamWorkerQueue;
 	const service = new WorkerService(queue, createSnowflake(), ledger);
-	return {service, calls, createdJobs, enqueued, seqUpdates, deadletters};
+	return {service, calls, createdJobs, enqueued, seqUpdates, deadletters, discarded};
 }
 
 describe('WorkerService ledger ordering', () => {
@@ -88,6 +101,42 @@ describe('WorkerService ledger ordering', () => {
 		expect(harness.deadletters).toEqual([{jobId: JOB_ID, errorMessage: 'stream unreachable'}]);
 	});
 
+	test('discards the ledger row when the stream already holds a job under the same key', async () => {
+		const harness = createHarness({duplicate: true});
+
+		const jobId = await harness.service.addJob(
+			'batchGuildAuditLogMessageDeletes',
+			{guildId: '1'},
+			{jobKey: 'batch-audit-log-message-deletes:1'},
+		);
+
+		expect(jobId).toBe(JOB_ID);
+		expect(harness.calls).toEqual(['createJob', 'enqueue', 'discardJob']);
+		expect(harness.discarded).toEqual([{jobId: JOB_ID, createdAt: CREATED_AT}]);
+		expect(harness.seqUpdates).toEqual([]);
+	});
+
+	test('still returns the job id when discarding the duplicate ledger row fails', async () => {
+		const harness = createHarness({duplicate: true, discardError: new Error('write timeout')});
+
+		const jobId = await harness.service.addJob(
+			'batchGuildAuditLogMessageDeletes',
+			{guildId: '1'},
+			{jobKey: 'batch-audit-log-message-deletes:1'},
+		);
+
+		expect(jobId).toBe(JOB_ID);
+		expect(harness.calls).toEqual(['createJob', 'enqueue', 'discardJob']);
+	});
+
+	test('does not touch the ledger for a duplicate the caller never ledgered', async () => {
+		const harness = createHarness({duplicate: true});
+
+		await harness.service.addJob('handleMentions', {}, {skipLedger: true, jobKey: 'mentions:1'});
+
+		expect(harness.calls).toEqual(['enqueue']);
+	});
+
 	test('never touches the ledger when the caller skips it', async () => {
 		const harness = createHarness();