// SPDX-License-Identifier: AGPL-3.0-or-later import {execFileSync, spawnSync} from 'node:child_process'; import {createServer} from 'node:net'; import { getDefaultPostgresClient, type IPostgresClient, initPostgres, type PostgresQueryable, shutdownPostgres, } from '@pkgs/postgres/src/Client'; import cassandra from 'cassandra-driver'; import {afterAll, beforeAll, describe, expect, it} from 'vitest'; import {LegacyPostgresKvQueryExecutor} from './__testref__/LegacyPostgresKvQueryExecutor'; import type {CassandraParams, KvQueryMeta, KvTableSpec, WhereExpr} from './CassandraTypes'; import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor'; type Row = Record; type AnyMeta = KvQueryMeta; const LEGACY_TABLE = 'kvm_legacy'; const NEXT_TABLE = 'kvm_next'; const CONTAINER = `fluxer-kvm-${process.pid.toString(36)}-${Date.now().toString(36)}`; const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0; class TableClient implements IPostgresClient { constructor( private readonly inner: IPostgresClient, private readonly table: string, ) {} async query>(text: string, values: Array = []) { return (await this.inner.query(text, values)) as unknown as Awaited> & { rows: Array; }; } async connect(): Promise { await this.inner.connect(); } async shutdown(): Promise {} isConnected(): boolean { return this.inner.isConnected(); } async transaction(fn: (client: PostgresQueryable) => Promise): Promise { return this.inner.transaction(fn); } kvTable(): string { return this.table; } } const Two: KvTableSpec = { name: 'mx_two', columns: ['k', 'c', 'v'], primaryKey: ['k', 'c'], partitionKey: ['k', 'c'], }; const PartTwo: KvTableSpec = { name: 'mx_part', columns: ['k', 'c', 'v'], primaryKey: ['k', 'c'], partitionKey: ['k'], }; const Three: KvTableSpec = { name: 'mx_three', columns: ['k', 'c', 'd', 'v'], primaryKey: ['k', 'c', 'd'], partitionKey: ['k', 'c', 'd'], }; const KEY_VALUES: Array<[string, unknown]> = [ ['null', null], ['num0', 0], ['num1', 1], ['num10', 10], ['big1', 1n], ['str1', '1'], ['strq', 'a"b\\c'], ['strctl', 'xy'], ['strastral', '\u{1f600}'], ['strspace', 'a b'], ['true', true], ['false', false], ['date', new Date(0)], ['buf', Buffer.from('a')], ['localdate', cassandra.types.LocalDate.fromString('2020-01-01')], ['localdate-str', '2020-01-01'], ['emptystr', ''], ]; function bigintJson(_key: string, value: unknown): unknown { if (typeof value === 'bigint') return `bigint:${value.toString()}`; if (Buffer.isBuffer(value)) return `buf:${value.toString('base64')}`; return value; } describe.skipIf(!dockerAvailable)('postgres kv non-select matrix', () => { let raw: IPostgresClient; let legacy: LegacyPostgresKvQueryExecutor; let next: PostgresKvQueryExecutor; async function freePort(): Promise { return new Promise((resolve, reject) => { const server = createServer(); server.on('error', reject); server.listen(0, '127.0.0.1', () => { const address = server.address(); if (typeof address === 'string' || address === null) { reject(new Error('no port')); return; } const port = address.port; server.close(() => resolve(port)); }); }); } async function sleep(ms: number): Promise { await new Promise((resolve) => setTimeout(resolve, ms)); } async function dump(kv: string, tableName: string): Promise { const result = await raw.query<{row_key: string}>( `SELECT row_key FROM ${kv} WHERE table_name = $1 ORDER BY row_key COLLATE "C"`, [tableName], ); return JSON.stringify(result.rows.map((r) => r.row_key)); } async function seed(spec: KvTableSpec, rows: ReadonlyArray): Promise { for (const kv of [LEGACY_TABLE, NEXT_TABLE]) { await raw.query(`DELETE FROM ${kv} WHERE table_name = $1`, [spec.name]); } const meta = {action: 'upsert', table: spec} as AnyMeta; for (const row of rows) { await legacy.executeQuery({cql: '__s__', params: row as CassandraParams, kvMeta: meta}); await next.executeQuery({cql: '__s__', params: row as CassandraParams, kvMeta: meta}); } } beforeAll(async () => { const port = await freePort(); execFileSync( 'docker', [ 'run', '-d', '--name', CONTAINER, '-e', 'POSTGRES_USER=fluxer', '-e', 'POSTGRES_PASSWORD=fluxer', '-e', 'POSTGRES_DB=fluxer', '-p', `127.0.0.1:${port}:5432`, 'postgres:16-alpine', '-c', 'fsync=off', ], {stdio: 'ignore'}, ); let ready = false; for (let attempt = 0; attempt < 180 && !ready; attempt += 1) { await sleep(500); if ( spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {stdio: 'ignore'}) .status !== 0 ) continue; try { await initPostgres({url: `postgres://fluxer:fluxer@127.0.0.1:${port}/fluxer`, maxConnections: 8}); await getDefaultPostgresClient().query('SELECT 1'); ready = true; } catch { await shutdownPostgres().catch(() => {}); } } if (!ready) throw new Error('postgres never came up'); raw = getDefaultPostgresClient(); await ensurePostgresKvSchema(new TableClient(raw, LEGACY_TABLE)); await ensurePostgresKvSchema(new TableClient(raw, NEXT_TABLE)); legacy = new LegacyPostgresKvQueryExecutor(new TableClient(raw, LEGACY_TABLE)); next = new PostgresKvQueryExecutor(new TableClient(raw, NEXT_TABLE)); }, 900_000); afterAll(async () => { await shutdownPostgres().catch(() => {}); spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'}); }); it('count and delete agree with legacy for every key value type', async () => { const rows = KEY_VALUES.map(([, value], i) => ({k: value, c: BigInt(i), v: i})); const countMismatch: Array = []; const deleteMismatch: Array = []; const selectMismatch: Array = []; for (const [name, value] of KEY_VALUES) { await seed(Two, rows); const where = [{kind: 'eq', col: 'k', param: 'k'}] as Array>; const params = {k: value} as CassandraParams; const sel = {action: 'select', table: Two, where, columns: Two.columns} as AnyMeta; const l = await legacy.executeQuery({cql: '__sel__', params, kvMeta: sel}); const n = await next.executeQuery({cql: '__sel__', params, kvMeta: sel}); if (JSON.stringify(l, bigintJson) !== JSON.stringify(n, bigintJson)) { selectMismatch.push(`${name}: legacy=${JSON.stringify(l, bigintJson)} next=${JSON.stringify(n, bigintJson)}`); } const cnt = {action: 'count', table: Two, where} as AnyMeta; const lc = await legacy.executeQuery({cql: '__cnt__', params, kvMeta: cnt}); const nc = await next.executeQuery({cql: '__cnt__', params, kvMeta: cnt}); if (JSON.stringify(lc) !== JSON.stringify(nc)) { countMismatch.push(`${name}: legacy=${JSON.stringify(lc)} next=${JSON.stringify(nc)}`); } const del = {action: 'delete', table: Two, where} as AnyMeta; await legacy.executeQuery({cql: '__del__', params, kvMeta: del}); await next.executeQuery({cql: '__del__', params, kvMeta: del}); const ld = await dump(LEGACY_TABLE, Two.name); const nd = await dump(NEXT_TABLE, Two.name); if (ld !== nd) deleteMismatch.push(`${name}: legacy=${ld} next=${nd}`); } expect({selectMismatch, countMismatch, deleteMismatch}).toEqual({ selectMismatch: [], countMismatch: [], deleteMismatch: [], }); }, 300_000); it('explicit partition key tables agree for count and delete', async () => { const rows = [ {k: 1n, c: 1n, v: 1}, {k: 1n, c: 2n, v: 2}, {k: 2n, c: 1n, v: 3}, ]; await seed(PartTwo, rows); const where = [{kind: 'eq', col: 'k', param: 'k'}] as Array>; const params = {k: 1n} as CassandraParams; const lc = await legacy.executeQuery({ cql: '__pc__', params, kvMeta: {action: 'count', table: PartTwo, where} as AnyMeta, }); const nc = await next.executeQuery({ cql: '__pc__', params, kvMeta: {action: 'count', table: PartTwo, where} as AnyMeta, }); expect(nc).toEqual(lc); await legacy.executeQuery({cql: '__pd__', params, kvMeta: {action: 'delete', table: PartTwo, where} as AnyMeta}); await next.executeQuery({cql: '__pd__', params, kvMeta: {action: 'delete', table: PartTwo, where} as AnyMeta}); expect(await dump(NEXT_TABLE, PartTwo.name)).toBe(await dump(LEGACY_TABLE, PartTwo.name)); }); it('multi column IN products agree, including above the combination cap', async () => { const rows: Array = []; for (let a = 0; a < 40; a += 1) { for (let b = 0; b < 40; b += 1) rows.push({k: BigInt(a), c: BigInt(b), v: a * b}); } await seed(Two, rows); const ks = Array.from({length: 40}, (_, i) => BigInt(i)); const cs = Array.from({length: 40}, (_, i) => BigInt(i)); const where = [ {kind: 'in', col: 'k', param: 'ks'}, {kind: 'in', col: 'c', param: 'cs'}, ] as Array>; for (const [label, params] of [ ['small', {ks: ks.slice(0, 3), cs: cs.slice(0, 3)}], ['cap', {ks, cs}], ['dupes', {ks: [1n, 1n, 2n], cs: [1n, 1n]}], ['empty', {ks: [], cs}], ] as Array<[string, CassandraParams]>) { const lc = await legacy.executeQuery({ cql: `__mc_${label}__`, params, kvMeta: {action: 'count', table: Two, where} as AnyMeta, }); const nc = await next.executeQuery({ cql: `__mc_${label}__`, params, kvMeta: {action: 'count', table: Two, where} as AnyMeta, }); expect([label, nc]).toEqual([label, lc]); const ls = await legacy.executeQuery({ cql: `__ms_${label}__`, params, kvMeta: {action: 'select', table: Two, where, columns: Two.columns} as AnyMeta, }); const ns = await next.executeQuery({ cql: `__ms_${label}__`, params, kvMeta: {action: 'select', table: Two, where, columns: Two.columns} as AnyMeta, }); expect([label, JSON.stringify(ns, bigintJson)]).toEqual([label, JSON.stringify(ls, bigintJson)]); } await legacy.executeQuery({ cql: '__md__', params: {ks, cs} as CassandraParams, kvMeta: {action: 'delete', table: Two, where} as AnyMeta, }); await next.executeQuery({ cql: '__md__', params: {ks, cs} as CassandraParams, kvMeta: {action: 'delete', table: Two, where} as AnyMeta, }); expect(await dump(NEXT_TABLE, Two.name)).toBe(await dump(LEGACY_TABLE, Two.name)); }, 300_000); it('count with a limit or an order by matches legacy', async () => { await seed(Two, [ {k: 1n, c: 1n, v: 1}, {k: 1n, c: 2n, v: 2}, {k: 1n, c: 3n, v: 3}, ]); const where = [{kind: 'eq', col: 'k', param: 'k'}] as Array>; for (const extra of [{limit: 2}, {orderBy: {col: 'c', direction: 'DESC'}}, {}]) { const meta = {action: 'count', table: Two, where, ...extra} as AnyMeta; const l = await legacy.executeQuery({cql: `__cl__${JSON.stringify(extra)}`, params: {k: 1n}, kvMeta: meta}); const n = await next.executeQuery({cql: `__cl__${JSON.stringify(extra)}`, params: {k: 1n}, kvMeta: meta}); expect([JSON.stringify(extra), n]).toEqual([JSON.stringify(extra), l]); } }); it('count and delete with a gap in the primary key match legacy', async () => { const rows: Array = []; for (const c of [1n, 2n]) { for (const d of [1n, 2n]) rows.push({k: 1n, c, d, v: 1}); } rows.push({k: 2n, c: 1n, d: 1n, v: 9}); await seed(Three, rows); const where = [ {kind: 'eq', col: 'k', param: 'k'}, {kind: 'eq', col: 'd', param: 'd'}, ] as Array>; const params = {k: 1n, d: 1n} as CassandraParams; const lc = await legacy.executeQuery({ cql: '__gc__', params, kvMeta: {action: 'count', table: Three, where} as AnyMeta, }); const nc = await next.executeQuery({ cql: '__gc__', params, kvMeta: {action: 'count', table: Three, where} as AnyMeta, }); expect(nc).toEqual(lc); await legacy.executeQuery({cql: '__gd__', params, kvMeta: {action: 'delete', table: Three, where} as AnyMeta}); await next.executeQuery({cql: '__gd__', params, kvMeta: {action: 'delete', table: Three, where} as AnyMeta}); expect(await dump(NEXT_TABLE, Three.name)).toBe(await dump(LEGACY_TABLE, Three.name)); }); it('delete with a non key where column matches legacy', async () => { await seed(Two, [ {k: 1n, c: 1n, v: 5}, {k: 1n, c: 2n, v: 6}, {k: 2n, c: 1n, v: 5}, ]); const where = [ {kind: 'eq', col: 'k', param: 'k'}, {kind: 'eq', col: 'v', param: 'v'}, ] as Array>; const params = {k: 1n, v: 5} as CassandraParams; await legacy.executeQuery({cql: '__nd__', params, kvMeta: {action: 'delete', table: Two, where} as AnyMeta}); await next.executeQuery({cql: '__nd__', params, kvMeta: {action: 'delete', table: Two, where} as AnyMeta}); expect(await dump(NEXT_TABLE, Two.name)).toBe(await dump(LEGACY_TABLE, Two.name)); }); });