pepepper/fluxer
1
0 Fork 0
mirror of https://github.com/fluxerapp/fluxer synced 2026-10-12 13:38:31 +09:00
Code Issues Packages Projects Releases Wiki Activity

fix(kv): drop the row_key order probe that cliffed paged scans (#2306)

Browse Source
This commit is contained in:
Hampus authored and GitHub committed 2026-09-01 01:32:33 +02:00
1 parent f1e6e94041
commit 42df4f6731
5 files changed
+82 -153
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
+3 -104
View File
@@ -19,7 +19,6 @@ interface StoredRow {
interface PageState {
offset: number;
after?: string;
keyed?: boolean;
}
interface PageEntry {
@@ -65,7 +64,6 @@ const VALUE_SEPARATOR = '\u001f';
const KEY_RANGE_UPPER = ' ';
const MAX_ROW_KEY_COMBINATIONS = 32_768;
const MAX_PREFIX_RANGES = 256;
const PAGE_PROBE_BATCH = 10_000;
const FULL_SCAN_LOG_INTERVAL_MS = 60_000;
const FULL_SCAN_LOG_KEY_LIMIT = 1024;
const ENCODED_TYPE_KEY = '__fluxer_type';
@@ -671,7 +669,7 @@ function decodePageState(pageState: string | null | undefined): PageState {
throw new Error('Invalid Postgres KV page state');
}
if (typeof decoded.after !== 'string') return {offset: decoded.offset};
return {offset: decoded.offset, after: decoded.after, keyed: decoded.keyed === true};
return {offset: decoded.offset, after: decoded.after};
}
function pageableSelect(meta: KvQueryMeta, pageSize: number): boolean {
@@ -684,13 +682,6 @@ function pageableSelect(meta: KvQueryMeta, pageSize: number): boolean {
);
}
function keysetCandidates(plan: QueryPlan): boolean {
return (
plan.exact &&
(plan.candidates.kind === 'none' || plan.candidates.kind === 'range' || plan.candidates.kind === 'scan')
);
}
function pageStart(meta: KvQueryMeta, entries: ReadonlyArray<PageEntry>, state: PageState): number {
if (state.after === undefined) return state.offset;
const index = entries.findIndex((entry) => entry.key === state.after);
@@ -702,12 +693,6 @@ function pageStart(meta: KvQueryMeta, entries: ReadonlyArray<PageEntry>, state:
return start;
}
function pagedStatementName(prefix: string, plan: CandidatePlan, after: string | null): string | undefined {
const name = planStatementName(prefix, plan);
if (name === undefined) return undefined;
return after === null ? name : `${name}_after`;
}
function parseRawMeta(cql: string): KvQueryMeta<Row> | null {
const normalized = normalizeCql(cql).replace(/;$/, '');
const update =
@@ -930,7 +915,8 @@ export class PostgresKvQueryExecutor {
const state = decodePageState(options.pageState);
const meta = this.meta(query);
if (pageableSelect(meta, options.pageSize) && (state.after !== undefined || state.offset === 0)) {
const page = await this.selectPage(meta, query.params, state, options.pageSize);
const plan = buildCandidatePlan(meta, query.params);
const page = await this.sortedPage(meta, query.params, plan, state, options.pageSize);
return {rows: page.rows as Array<T>, pageState: page.pageState};
}
const rows = await this.executeQuery<T, P>(query);
@@ -942,93 +928,6 @@ export class PostgresKvQueryExecutor {
};
}
private async selectPage(
meta: KvQueryMeta,
params: CassandraParams,
state: PageState,
pageSize: number,
): Promise<{rows: Array<Row>; pageState: string | null}> {
const plan = buildCandidatePlan(meta, params);
if (keysetCandidates(plan)) {
if (state.keyed === true) {
return this.keysetPage(meta, params, plan, state.after ?? null, state.offset, pageSize);
}
if (state.after === undefined && (await this.rowKeyOrderMatchesSort(meta, plan))) {
return this.keysetPage(meta, params, plan, null, 0, pageSize);
}
}
return this.sortedPage(meta, params, plan, state, pageSize);
}
private pagedSql(
meta: KvQueryMeta,
fragments: PlanFragments,
projection: string,
after: string | null,
limit: number,
): {text: string; values: Array<unknown>} {
const values: Array<unknown> = [meta.table.name, ...fragments.params];
let text = `SELECT ${projection} FROM ${this.table} kv WHERE kv.table_name = $1${fragments.predicate}`;
if (after !== null) {
values.push(after);
text += ` AND kv.row_key COLLATE "C" > $${values.length}`;
}
text += ' AND (kv.expires_at IS NULL OR kv.expires_at > now()) ORDER BY kv.row_key COLLATE "C"';
values.push(limit);
return {text: `${text} LIMIT $${values.length}`, values};
}
private async rowKeyOrderMatchesSort(meta: KvQueryMeta, plan: QueryPlan): Promise<boolean> {
if (plan.candidates.kind === 'none') return true;
const fragments = planFragmentGroups(plan.candidates)[0]!;
const columns = (meta.table.primaryKey as ReadonlyArray<string>).length;
let after: string | null = null;
let previous: Array<unknown> | null = null;
for (;;) {
const sql = this.pagedSql(meta, fragments, 'kv.row_key', after, PAGE_PROBE_BATCH);
const name = pagedStatementName('kv_keys', plan.candidates, after);
const batch: Array<{row_key: string}> = (await this.client.query<{row_key: string}>(sql.text, sql.values, name))
.rows;
for (const stored of batch) {
const values = decodeRowKey(stored.row_key, columns);
if (values === null) return false;
if (previous !== null && compareKeyValues(previous, values) > 0) return false;
previous = values;
}
if (batch.length < PAGE_PROBE_BATCH) return true;
after = batch[batch.length - 1]!.row_key;
}
}
private async keysetPage(
meta: KvQueryMeta,
params: CassandraParams,
plan: QueryPlan,
after: string | null,
offset: number,
pageSize: number,
): Promise<{rows: Array<Row>; pageState: string | null}> {
if (plan.candidates.kind === 'none') return {rows: [], pageState: null};
if (plan.candidates.kind === 'scan') logFullScan(meta);
const fragments = planFragmentGroups(plan.candidates)[0]!;
const sql = this.pagedSql(meta, fragments, 'kv.row_key, kv.row_data', after, pageSize + 1);
const result = await this.client.query<StoredRow>(
sql.text,
sql.values,
pagedStatementName('kv_page', plan.candidates, after),
);
const stored = result.rows.slice(0, pageSize);
const entries = this.matchingEntries(meta, stored, params);
const last = entries[entries.length - 1];
return {
rows: entries.map((entry) => projectRow(entry.row, meta.columns as ReadonlyArray<string> | undefined)),
pageState:
result.rows.length > pageSize && last
? encodePageState({offset: offset + entries.length, after: last.key, keyed: true})
: null,
};
}
private async sortedPage(
meta: KvQueryMeta,
params: CassandraParams,