fix(kv): stop multi-key commands spanning cluster slots (#2310)

fluxer_api/src/api/infrastructure/KVActivityTracker.ts

@@ -74,6 +74,10 @@ export class KVActivityTracker {
 		return age > STATE_VERSION_TTL_SECONDS;
 	}
 
+	private async writeActivityBatch(batch: ReadonlyArray<{key: string; value: string}>): Promise<void> {
+		await Promise.all(batch.map(async (entry) => this.kvClient.setex(entry.key, TTL_SECONDS, entry.value)));
+	}
+
 	async rebuildActivities(): Promise<void> {
 		Logger.info('Starting activity tracker rebuild from Cassandra');
 		const userRepository = new UserRepository();
@@ -81,8 +85,7 @@ export class KVActivityTracker {
 			const kvBatchSize = 1000;
 			let processedCount = 0;
 			let usersWithActivity = 0;
-			let pipeline = this.kvClient.pipeline();
-			let pipelineCount = 0;
+			let batch: Array<{key: string; value: string}> = [];
 			let pageState: string | null = null;
 			let iterationCount = 0;
 			while (!this.isShuttingDown) {
@@ -93,15 +96,11 @@ export class KVActivityTracker {
 				}
 				for (const user of users) {
 					if (user.lastActiveAt) {
-						const key = this.getActivityKey(user.id);
-						const value = user.lastActiveAt.getTime().toString();
-						pipeline.setex(key, TTL_SECONDS, value);
-						pipelineCount++;
+						batch.push({key: this.getActivityKey(user.id), value: user.lastActiveAt.getTime().toString()});
 						usersWithActivity++;
-						if (pipelineCount >= kvBatchSize) {
-							await pipeline.exec();
-							pipeline = this.kvClient.pipeline();
-							pipelineCount = 0;
+						if (batch.length >= kvBatchSize) {
+							await this.writeActivityBatch(batch);
+							batch = [];
 						}
 					}
 					processedCount++;
@@ -122,8 +121,8 @@ export class KVActivityTracker {
 				Logger.warn({processedCount, usersWithActivity}, 'Activity tracker rebuild interrupted by shutdown');
 				return;
 			}
-			if (pipelineCount > 0) {
-				await pipeline.exec();
+			if (batch.length > 0) {
+				await this.writeActivityBatch(batch);
 			}
 			await this.kvClient.setex(STATE_VERSION_KEY, STATE_VERSION_TTL_SECONDS, Date.now().toString());
 			Logger.info({processedCount, usersWithActivity}, 'Activity tracker rebuild completed');

fluxer_api/src/api/infrastructure/tests/KVActivityTrackerClusterSlots.test.ts

@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {afterEach, describe, expect, it, vi} from 'vitest';
+import {createUserID} from '../../BrandedTypes';
+import type {User} from '../../models/User';
+import {BatchRecordingKVProvider, hashSlot} from '../../test/mocks/BatchRecordingKVProvider';
+import {UserRepository} from '../../user/repositories/UserRepository';
+import {KVActivityTracker} from '../KVActivityTracker';
+
+function createUser(id: bigint, lastActiveAt: Date): User {
+	return {id: createUserID(id), lastActiveAt} as unknown as User;
+}
+
+describe('KVActivityTracker cluster hash slots', () => {
+	afterEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	it('keeps the rebuild writes off batched commands that span hash slots', async () => {
+		const kvClient = new BatchRecordingKVProvider();
+		const lastActiveAt = new Date('2026-06-01T00:00:00.000Z');
+		const users = [createUser(1234n, lastActiveAt), createUser(5678n, lastActiveAt)];
+		vi.spyOn(UserRepository.prototype, 'scanAllUsersPage').mockResolvedValue({users, pageState: null});
+
+		expect(hashSlot('user_activity:1234')).not.toBe(hashSlot('user_activity:5678'));
+
+		await new KVActivityTracker(kvClient).rebuildActivities();
+
+		expect(await kvClient.get('user_activity:1234')).toBe(lastActiveAt.getTime().toString());
+		expect(await kvClient.get('user_activity:5678')).toBe(lastActiveAt.getTime().toString());
+		expect(kvClient.crossSlotBatches()).toEqual([]);
+	});
+});

fluxer_api/src/api/infrastructure/tests/VoiceRoomStoreClusterSlots.test.ts

@@ -1,116 +1,11 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import type {IKVPipeline} from '@pkgs/kv_client/src/IKVProvider';
 import {describe, expect, it} from 'vitest';
 import {createChannelID, createGuildID} from '../../BrandedTypes';
-import {MockKVProvider} from '../../test/mocks/MockKVProvider';
+import {BatchRecordingKVProvider, hashSlot} from '../../test/mocks/BatchRecordingKVProvider';
 import {VOICE_OCCUPANCY_REGION_KEY_PREFIX, VOICE_OCCUPANCY_SERVER_KEY_PREFIX} from '../../voice/VoiceConstants';
 import {VoiceRoomStore} from '../VoiceRoomStore';
 
-type BatchMode = 'multi' | 'pipeline';
-
-interface RecordedBatch {
-	mode: BatchMode;
-	keys: Array<string>;
-}
-
-function hashSlot(key: string): number {
-	let hashed = key;
-	const start = key.indexOf('{');
-	if (start !== -1) {
-		const end = key.indexOf('}', start + 1);
-		if (end > start + 1) {
-			hashed = key.slice(start + 1, end);
-		}
-	}
-	let crc = 0;
-	for (let index = 0; index < hashed.length; index += 1) {
-		crc ^= (hashed.charCodeAt(index) & 0xff) << 8;
-		for (let bit = 0; bit < 8; bit += 1) {
-			crc = (crc & 0x8000) === 0 ? (crc << 1) & 0xffff : ((crc << 1) ^ 0x1021) & 0xffff;
-		}
-	}
-	return crc % 16384;
-}
-
-class BatchRecordingKVProvider extends MockKVProvider {
-	readonly batches: Array<RecordedBatch> = [];
-
-	override pipeline(): IKVPipeline {
-		return this.recordBatch('pipeline', super.pipeline());
-	}
-
-	override multi(): IKVPipeline {
-		return this.recordBatch('multi', super.multi());
-	}
-
-	private recordBatch(mode: BatchMode, inner: IKVPipeline): IKVPipeline {
-		const batch: RecordedBatch = {mode, keys: []};
-		this.batches.push(batch);
-		const recorded: IKVPipeline = {
-			get: (key) => {
-				batch.keys.push(key);
-				inner.get(key);
-				return recorded;
-			},
-			set: (key, value) => {
-				batch.keys.push(key);
-				inner.set(key, value);
-				return recorded;
-			},
-			setex: (key, ttlSeconds, value) => {
-				batch.keys.push(key);
-				inner.setex(key, ttlSeconds, value);
-				return recorded;
-			},
-			del: (key) => {
-				batch.keys.push(key);
-				inner.del(key);
-				return recorded;
-			},
-			expire: (key, ttlSeconds) => {
-				batch.keys.push(key);
-				inner.expire(key, ttlSeconds);
-				return recorded;
-			},
-			sadd: (key, ...members) => {
-				batch.keys.push(key);
-				inner.sadd(key, ...members);
-				return recorded;
-			},
-			srem: (key, ...members) => {
-				batch.keys.push(key);
-				inner.srem(key, ...members);
-				return recorded;
-			},
-			zadd: (key, score, value) => {
-				batch.keys.push(key);
-				inner.zadd(key, score, value);
-				return recorded;
-			},
-			zrem: (key, ...members) => {
-				batch.keys.push(key);
-				inner.zrem(key, ...members);
-				return recorded;
-			},
-			hgetall: (key) => {
-				batch.keys.push(key);
-				inner.hgetall(key);
-				return recorded;
-			},
-			mset: (...args) => {
-				for (let index = 0; index + 1 < args.length; index += 2) {
-					batch.keys.push(args[index]);
-				}
-				inner.mset(...args);
-				return recorded;
-			},
-			exec: async () => await inner.exec(),
-		};
-		return recorded;
-	}
-}
-
 describe('VoiceRoomStore cluster hash slots', () => {
 	it('keeps occupancy writes off batched commands that span hash slots', async () => {
 		const kvClient = new BatchRecordingKVProvider();
@@ -131,7 +26,6 @@ describe('VoiceRoomStore cluster hash slots', () => {
 		expect(await kvClient.smembers(regionKey)).toEqual([]);
 		expect(await kvClient.smembers(serverKey)).toEqual([]);
 
-		const crossSlotBatches = kvClient.batches.filter((batch) => new Set(batch.keys.map(hashSlot)).size > 1);
-		expect(crossSlotBatches).toEqual([]);
+		expect(kvClient.crossSlotBatches()).toEqual([]);
 	});
 });

fluxer_api/src/api/test/mocks/BatchRecordingKVProvider.ts

@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {IKVPipeline} from '@pkgs/kv_client/src/IKVProvider';
+import {MockKVProvider} from './MockKVProvider';
+
+type BatchMode = 'multi' | 'pipeline';
+
+interface RecordedBatch {
+	mode: BatchMode;
+	keys: Array<string>;
+}
+
+export function hashSlot(key: string): number {
+	let hashed = key;
+	const start = key.indexOf('{');
+	if (start !== -1) {
+		const end = key.indexOf('}', start + 1);
+		if (end > start + 1) {
+			hashed = key.slice(start + 1, end);
+		}
+	}
+	let crc = 0;
+	for (let index = 0; index < hashed.length; index += 1) {
+		crc ^= (hashed.charCodeAt(index) & 0xff) << 8;
+		for (let bit = 0; bit < 8; bit += 1) {
+			crc = (crc & 0x8000) === 0 ? (crc << 1) & 0xffff : ((crc << 1) ^ 0x1021) & 0xffff;
+		}
+	}
+	return crc % 16384;
+}
+
+export class BatchRecordingKVProvider extends MockKVProvider {
+	readonly batches: Array<RecordedBatch> = [];
+
+	override pipeline(): IKVPipeline {
+		return this.recordBatch('pipeline', super.pipeline());
+	}
+
+	override multi(): IKVPipeline {
+		return this.recordBatch('multi', super.multi());
+	}
+
+	crossSlotBatches(): Array<RecordedBatch> {
+		return this.batches.filter((batch) => new Set(batch.keys.map(hashSlot)).size > 1);
+	}
+
+	private recordBatch(mode: BatchMode, inner: IKVPipeline): IKVPipeline {
+		const batch: RecordedBatch = {mode, keys: []};
+		this.batches.push(batch);
+		const recorded: IKVPipeline = {
+			get: (key) => {
+				batch.keys.push(key);
+				inner.get(key);
+				return recorded;
+			},
+			set: (key, value) => {
+				batch.keys.push(key);
+				inner.set(key, value);
+				return recorded;
+			},
+			setex: (key, ttlSeconds, value) => {
+				batch.keys.push(key);
+				inner.setex(key, ttlSeconds, value);
+				return recorded;
+			},
+			del: (key) => {
+				batch.keys.push(key);
+				inner.del(key);
+				return recorded;
+			},
+			expire: (key, ttlSeconds) => {
+				batch.keys.push(key);
+				inner.expire(key, ttlSeconds);
+				return recorded;
+			},
+			sadd: (key, ...members) => {
+				batch.keys.push(key);
+				inner.sadd(key, ...members);
+				return recorded;
+			},
+			srem: (key, ...members) => {
+				batch.keys.push(key);
+				inner.srem(key, ...members);
+				return recorded;
+			},
+			zadd: (key, score, value) => {
+				batch.keys.push(key);
+				inner.zadd(key, score, value);
+				return recorded;
+			},
+			zrem: (key, ...members) => {
+				batch.keys.push(key);
+				inner.zrem(key, ...members);
+				return recorded;
+			},
+			hgetall: (key) => {
+				batch.keys.push(key);
+				inner.hgetall(key);
+				return recorded;
+			},
+			mset: (...args) => {
+				for (let index = 0; index + 1 < args.length; index += 2) {
+					batch.keys.push(args[index]);
+				}
+				inner.mset(...args);
+				return recorded;
+			},
+			exec: async () => await inner.exec(),
+		};
+		return recorded;
+	}
+}