fix(kv): bound multi key fan out by pipelining per hash slot (#2313)

This commit is contained in:
Hampus authored and GitHub committed 2026-09-01 02:59:00 +02:00
1 parent 3b5b20c139
commit cf3af50464
14 files changed
+506 -86

No files matched your search

@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {IKVPipeline} from '@pkgs/kv_client/src/IKVProvider';
import {computeHashSlot} from '@pkgs/kv_client/src/KVHashSlots';
import {MockKVProvider} from './MockKVProvider';
type BatchMode = 'multi' | 'pipeline';
@@ -10,25 +11,6 @@ interface RecordedBatch {
keys: Array<string>;
}
export function hashSlot(key: string): number {
let hashed = key;
const start = key.indexOf('{');
if (start !== -1) {
const end = key.indexOf('}', start + 1);
if (end > start + 1) {
hashed = key.slice(start + 1, end);
}
}
let crc = 0;
for (let index = 0; index < hashed.length; index += 1) {
crc ^= (hashed.charCodeAt(index) & 0xff) << 8;
for (let bit = 0; bit < 8; bit += 1) {
crc = (crc & 0x8000) === 0 ? (crc << 1) & 0xffff : ((crc << 1) ^ 0x1021) & 0xffff;
}
}
return crc % 16384;
}
export class BatchRecordingKVProvider extends MockKVProvider {
readonly batches: Array<RecordedBatch> = [];
@@ -41,7 +23,7 @@ export class BatchRecordingKVProvider extends MockKVProvider {
}
crossSlotBatches(): Array<RecordedBatch> {
return this.batches.filter((batch) => new Set(batch.keys.map(hashSlot)).size > 1);
return this.batches.filter((batch) => new Set(batch.keys.map(computeHashSlot)).size > 1);
}
private recordBatch(mode: BatchMode, inner: IKVPipeline): IKVPipeline {
@@ -111,6 +111,7 @@ export class MockKVProvider implements IKVProvider {
key: string;
values: Array<string>;
}> = [];
clustered = true;
private subscription: MockKVSubscription;
private readonly stringStore = new Map<string, string>();
private readonly setStore = new Map<string, Set<string>>();
@@ -782,6 +783,10 @@ export class MockKVProvider implements IKVProvider {
return this.createPipeline();
}
isClustered(): boolean {
return this.clustered;
}
async health(): Promise<boolean> {
this.healthSpy();
return true;