mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-19 05:55:34 +09:00
fix(kv): bound multi key fan out by pipelining per hash slot (#2313)
This commit is contained in:
1 parent
3b5b20c139
commit
cf3af50464
14 files changed
+506
-86
No files matched your search
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {IKVPipeline} from '@pkgs/kv_client/src/IKVProvider';
|
||||
import {computeHashSlot} from '@pkgs/kv_client/src/KVHashSlots';
|
||||
import {MockKVProvider} from './MockKVProvider';
|
||||
|
||||
type BatchMode = 'multi' | 'pipeline';
|
||||
@@ -10,25 +11,6 @@ interface RecordedBatch {
|
||||
keys: Array<string>;
|
||||
}
|
||||
|
||||
export function hashSlot(key: string): number {
|
||||
let hashed = key;
|
||||
const start = key.indexOf('{');
|
||||
if (start !== -1) {
|
||||
const end = key.indexOf('}', start + 1);
|
||||
if (end > start + 1) {
|
||||
hashed = key.slice(start + 1, end);
|
||||
}
|
||||
}
|
||||
let crc = 0;
|
||||
for (let index = 0; index < hashed.length; index += 1) {
|
||||
crc ^= (hashed.charCodeAt(index) & 0xff) << 8;
|
||||
for (let bit = 0; bit < 8; bit += 1) {
|
||||
crc = (crc & 0x8000) === 0 ? (crc << 1) & 0xffff : ((crc << 1) ^ 0x1021) & 0xffff;
|
||||
}
|
||||
}
|
||||
return crc % 16384;
|
||||
}
|
||||
|
||||
export class BatchRecordingKVProvider extends MockKVProvider {
|
||||
readonly batches: Array<RecordedBatch> = [];
|
||||
|
||||
@@ -41,7 +23,7 @@ export class BatchRecordingKVProvider extends MockKVProvider {
|
||||
}
|
||||
|
||||
crossSlotBatches(): Array<RecordedBatch> {
|
||||
return this.batches.filter((batch) => new Set(batch.keys.map(hashSlot)).size > 1);
|
||||
return this.batches.filter((batch) => new Set(batch.keys.map(computeHashSlot)).size > 1);
|
||||
}
|
||||
|
||||
private recordBatch(mode: BatchMode, inner: IKVPipeline): IKVPipeline {
|
||||
|
||||
@@ -111,6 +111,7 @@ export class MockKVProvider implements IKVProvider {
|
||||
key: string;
|
||||
values: Array<string>;
|
||||
}> = [];
|
||||
clustered = true;
|
||||
private subscription: MockKVSubscription;
|
||||
private readonly stringStore = new Map<string, string>();
|
||||
private readonly setStore = new Map<string, Set<string>>();
|
||||
@@ -782,6 +783,10 @@ export class MockKVProvider implements IKVProvider {
|
||||
return this.createPipeline();
|
||||
}
|
||||
|
||||
isClustered(): boolean {
|
||||
return this.clustered;
|
||||
}
|
||||
|
||||
async health(): Promise<boolean> {
|
||||
this.healthSpy();
|
||||
return true;
|
||||
|
||||
Reference in new issue
Block a user