Compare commits

...
Author SHA1 Message Date
HampusandGitHub 8092ad8c4d fix(media-proxy): support current FFmpeg APIs (#2158) 2026-08-30 22:08:55 +02:00
HampusandGitHub b4d9cdc584 refactor(voice): remove heartbeat and debug logging sessions (#2121) 2026-08-30 21:08:46 +02:00
HampusandGitHub 36b85512c6 chore(api): drop knip-unused Postgres KV exports (#2119) 2026-08-30 18:58:50 +02:00
HampusandGitHub 14ae64f5f3 perf(api): stop Postgres KV reads scanning whole tables (#2118) 2026-08-30 18:34:23 +02:00
M0N7Y5andGitHub 990176ac7c feat(markdown): add a binary AST envelope to the native ABI (#2117) 2026-08-30 17:47:58 +02:00
M0N7Y5andGitHub 69ef46356b feat(markdown): add a native C ABI for 64-bit FFI hosts (#2115) 2026-08-30 15:47:34 +02:00
HampusandGitHub bd88c7b04b fix(desktop): don't fail startup on inconclusive native probe (#2114) 2026-08-30 15:22:48 +02:00
HampusandGitHub 03641f622f refactor(voice): flatten participant context menu and extract stream menus (#2112) 2026-08-30 03:02:30 +02:00
HampusandGitHub 3b1eb56713 fix(voice): never auto-select AV1 or HEVC for screen sharing (#2111) 2026-08-30 02:05:21 +02:00
HampusandGitHub 059bcc6c53 chore(app): regenerate theme variable manifest for font fallbacks (#2110) 2026-08-30 01:30:00 +02:00
HampusandGitHub 82043ce2a8 fix(guild): show duplicated role in its final spot without flicker (#2109) 2026-08-30 01:25:34 +02:00
HampusandGitHub 470e752fba chore(i18n): refresh catalogs for role and permission menus (#2108) 2026-08-30 00:17:31 +02:00
HampusandGitHub 3cc7b9050c fix(app): clear stuck spellcheck reload banner (#2107) 2026-08-30 00:12:48 +02:00
HampusandGitHub b1f7c78c7e fix(app): overwrite context menu hover and delete danger item (#2106) 2026-08-30 00:12:41 +02:00
HampusandGitHub 8f20b29b16 feat(guild): duplicate role, plus delete and hover in roles sidebar (#2105) 2026-08-30 00:12:36 +02:00
HampusandGitHub 19efbd3d61 fix(app): scope the show-send-button toggle to the main composer (#2104) 2026-08-30 00:12:31 +02:00
HampusandGitHub f35c0effa2 chore(i18n): drop unused gift redemption string (#2102) 2026-08-29 23:31:28 +02:00
HampusandGitHub 8363cc0844 fix(app): send only the gift link when gifting to a friend (#2101) 2026-08-29 23:29:49 +02:00
HampusandGitHub 5a11cacbae chore(i18n): refresh catalogs for spellcheck copy (#2100) 2026-08-29 23:03:01 +02:00
HampusandGitHub 27151a9487 fix(desktop): restore Linux spellcheck, prefer OS engine (#2099) 2026-08-29 22:58:48 +02:00
HampusandGitHub c152b25deb chore(i18n): refresh catalogs for global shortcut string (#2098) 2026-08-29 22:57:18 +02:00
HampusandGitHub 04d3afaf7b fix(app): default voice shortcuts to global and add a toggle (#2097) 2026-08-29 22:46:30 +02:00
HampusandGitHub dbc63e9ef7 fix(voice): surface stream audio volume for screen-share viewers (#2095) 2026-08-29 22:05:06 +02:00
HampusandGitHub ce91ff95ab fix(app): render raw unicode emoji with OS color emoji fonts (#2094) 2026-08-29 21:09:08 +02:00
HampusandGitHub d75a29f099 feat(app): composer send-button toggle and active button hover (#2093) 2026-08-29 20:25:49 +02:00
HampusandGitHub cb889b1160 fix(api): correct apns clear payload and badge handling (#2092) 2026-08-29 20:07:18 +02:00
HampusandGitHub 8db4f5cb63 feat(app): full-Unicode font fallback with self-hosted Noto (#2091) 2026-08-29 19:27:47 +02:00
HampusandGitHub d297dc5805 feat(api): exempt APP_STORE_REVIEWER accounts from captcha (#2090) 2026-08-29 18:00:52 +02:00
HampusandGitHub 9b8659a40b fix(desktop): stop the updater button flickering after download (#2089) 2026-08-29 17:44:43 +02:00
HampusandGitHub 96c5db3f5d fix(markdown): open code blocks for fences after text (#2088) 2026-08-29 17:32:43 +02:00
HampusandGitHub 78e403819e fix(admin): set delete_message_seconds on ban request (#2087) 2026-08-29 16:35:40 +02:00
HampusandGitHub 805acf4e5e feat(ban): accept delete_message_seconds and app options (#2086) 2026-08-29 16:22:12 +02:00
HampusandGitHub 9f099a9127 fix(api): keep the saved placeholder on sent memes (#2085) 2026-08-29 16:04:58 +02:00
HampusandGitHub 4d15c39cd7 fix(app): give mature media blur the real media dimensions (#2084) 2026-08-29 16:04:55 +02:00
HampusandGitHub 827451d12d fix(unfurl): trust curated klipy media without rescanning (#2083) 2026-08-29 16:04:52 +02:00
HampusandGitHub 03603662c6 fix(media-proxy): require corroborating frames for nsfw (#2082) 2026-08-29 16:04:48 +02:00
HampusandGitHub 34eb10cd88 fix(markdown): only open code fences at line start (#2081) 2026-08-29 15:59:04 +02:00
HampusandGitHub 82941c08c9 fix(api): enforce single-role MFA, fix reindex and NCMEC scans (#2080) 2026-08-29 15:31:42 +02:00
236 changed files with 18861 additions and 13438 deletions
+13 -264
View File
@@ -8905,231 +8905,6 @@
}
}
},
"/admin/voice/diagnostics/objects": {
"get": {
"operationId": "list_voice_diagnostics_objects",
"summary": "List voice diagnostics objects",
"tags": ["Admin"],
"responses": {
"200": {
"description": "Success",
"content": {
"application/json": {"schema": {"$ref": "#/components/schemas/VoiceDiagnosticsObjectListResponse"}}
}
},
"400": {
"description": "Bad Request - The request was malformed or contained invalid data",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"401": {
"description": "Unauthorized - Authentication is required or the token is invalid",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"403": {
"description": "Forbidden - You do not have permission to perform this action",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"429": {
"description": "Too Many Requests - You are being rate limited",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}},
"headers": {
"Retry-After": {
"description": "Number of seconds to wait before retrying (only on 429)",
"schema": {"type": "integer"}
},
"X-RateLimit-Limit": {
"description": "The number of requests that can be made in the current window",
"schema": {"type": "integer"}
},
"X-RateLimit-Remaining": {
"description": "The number of remaining requests that can be made",
"schema": {"type": "integer"}
},
"X-RateLimit-Reset": {
"description": "Unix timestamp when the rate limit resets",
"schema": {"type": "integer"}
}
}
},
"500": {
"description": "Internal Server Error - An unexpected error occurred",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
}
},
"description": "Lists raw voice diagnostics NDJSON S3 objects for a channel and time range. Requires VOICE_DIAGNOSTICS_VIEW permission.",
"security": [{"adminApiKey": []}],
"parameters": [
{
"name": "channel_id",
"in": "query",
"required": true,
"schema": {
"type": "string",
"pattern": "^(0|[1-9][0-9]*)$",
"description": "Channel id to query diagnostics for"
}
},
{
"name": "start_ms",
"in": "query",
"required": true,
"schema": {
"type": "integer",
"minimum": 0,
"maximum": 8640000000000000,
"format": "int53",
"description": "Inclusive start timestamp in milliseconds"
}
},
{
"name": "end_ms",
"in": "query",
"required": true,
"schema": {
"type": "integer",
"minimum": 0,
"maximum": 8640000000000000,
"format": "int53",
"description": "Inclusive end timestamp in milliseconds"
}
},
{
"name": "session_id",
"in": "query",
"required": false,
"schema": {
"type": "string",
"minLength": 1,
"maxLength": 128,
"pattern": "^[A-Za-z0-9_.:-]+$",
"description": "Optional voice diagnostics session id filter"
}
},
{
"name": "limit_objects",
"in": "query",
"required": false,
"schema": {
"type": "integer",
"minimum": 1,
"maximum": 5000,
"format": "int32",
"description": "Maximum matching S3 objects to include"
}
}
]
}
},
"/admin/voice/diagnostics/raw": {
"get": {
"operationId": "stream_voice_diagnostics_raw",
"summary": "Stream raw voice diagnostics",
"tags": ["Admin"],
"responses": {
"204": {"description": "No Content"},
"400": {
"description": "Bad Request - The request was malformed or contained invalid data",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"401": {
"description": "Unauthorized - Authentication is required or the token is invalid",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"403": {
"description": "Forbidden - You do not have permission to perform this action",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"429": {
"description": "Too Many Requests - You are being rate limited",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}},
"headers": {
"Retry-After": {
"description": "Number of seconds to wait before retrying (only on 429)",
"schema": {"type": "integer"}
},
"X-RateLimit-Limit": {
"description": "The number of requests that can be made in the current window",
"schema": {"type": "integer"}
},
"X-RateLimit-Remaining": {
"description": "The number of remaining requests that can be made",
"schema": {"type": "integer"}
},
"X-RateLimit-Reset": {
"description": "Unix timestamp when the rate limit resets",
"schema": {"type": "integer"}
}
}
},
"500": {
"description": "Internal Server Error - An unexpected error occurred",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
}
},
"description": "Streams matching voice diagnostics NDJSON objects for local processing. Requires VOICE_DIAGNOSTICS_VIEW permission.",
"security": [{"adminApiKey": []}],
"parameters": [
{
"name": "channel_id",
"in": "query",
"required": true,
"schema": {
"type": "string",
"pattern": "^(0|[1-9][0-9]*)$",
"description": "Channel id to query diagnostics for"
}
},
{
"name": "start_ms",
"in": "query",
"required": true,
"schema": {
"type": "integer",
"minimum": 0,
"maximum": 8640000000000000,
"format": "int53",
"description": "Inclusive start timestamp in milliseconds"
}
},
{
"name": "end_ms",
"in": "query",
"required": true,
"schema": {
"type": "integer",
"minimum": 0,
"maximum": 8640000000000000,
"format": "int53",
"description": "Inclusive end timestamp in milliseconds"
}
},
{
"name": "session_id",
"in": "query",
"required": false,
"schema": {
"type": "string",
"minLength": 1,
"maxLength": 128,
"pattern": "^[A-Za-z0-9_.:-]+$",
"description": "Optional voice diagnostics session id filter"
}
},
{
"name": "limit_objects",
"in": "query",
"required": false,
"schema": {
"type": "integer",
"minimum": 1,
"maximum": 5000,
"format": "int32",
"description": "Maximum matching S3 objects to include"
}
}
]
}
},
"/admin/voice/regions/create": {
"post": {
"operationId": "create_voice_region",
@@ -9718,7 +9493,7 @@
"acls": {
"type": "array",
"items": {"type": "string"},
"maxItems": 115,
"maxItems": 114,
"description": "List of access control permissions for the key"
}
},
@@ -10078,7 +9853,7 @@
"acls": {
"type": "array",
"items": {"type": "string"},
"maxItems": 115,
"maxItems": 114,
"description": "List of access control permissions for the key"
}
},
@@ -10108,7 +9883,7 @@
"acls": {
"type": "array",
"items": {"type": "string"},
"maxItems": 115,
"maxItems": 114,
"description": "List of access control permissions for the key"
}
},
@@ -11854,7 +11629,14 @@
"minimum": 0,
"maximum": 7,
"format": "int32",
"description": "Number of days of messages to delete from the banned user (0-7)"
"description": "Number of days of messages to delete from the banned user (0-7). Deprecated in favor of delete_message_seconds."
},
"delete_message_seconds": {
"type": "integer",
"minimum": 0,
"maximum": 604800,
"format": "int32",
"description": "Number of seconds of messages to delete for the banned user (0-604800, default 0)"
},
"reason": {"description": "The reason for the ban (max 512 characters)", "nullable": true, "type": "string"},
"ban_duration_seconds": {
@@ -14911,7 +14693,7 @@
"pending_bulk_message_deletion_at": {"nullable": true, "type": "string"},
"deletion_reason_code": {"nullable": true, "allOf": [{"$ref": "#/components/schemas/Int32Type"}]},
"deletion_public_reason": {"nullable": true, "type": "string"},
"acls": {"type": "array", "items": {"type": "string"}, "maxItems": 115},
"acls": {"type": "array", "items": {"type": "string"}, "maxItems": 114},
"traits": {"type": "array", "items": {"type": "string"}, "maxItems": 100},
"has_totp": {"type": "boolean"},
"authenticator_types": {"type": "array", "items": {"$ref": "#/components/schemas/Int32Type"}, "maxItems": 10},
@@ -15479,7 +15261,7 @@
"acls": {
"type": "array",
"items": {"type": "string"},
"maxItems": 115,
"maxItems": 114,
"description": "List of access control permissions to assign"
}
},
@@ -15610,39 +15392,6 @@
"type": "object",
"properties": {"reason": {"type": "string"}, "notes": {"type": "string"}}
},
"VoiceDiagnosticsObjectListResponse": {
"type": "object",
"properties": {
"bucket": {"type": "string", "description": "Diagnostics S3 bucket"},
"objects": {
"type": "array",
"items": {
"type": "object",
"properties": {
"key": {"type": "string", "description": "S3 object key"},
"session_id": {"type": "string", "description": "Voice diagnostics session id"},
"start_ns": {
"type": "string",
"description": "First client event timestamp in the object, in nanoseconds"
},
"end_ns": {
"type": "string",
"description": "Last client event timestamp in the object, in nanoseconds"
},
"last_modified": {
"description": "S3 object last-modified timestamp",
"nullable": true,
"type": "string",
"format": "date-time"
}
},
"required": ["key", "session_id", "start_ns", "end_ns", "last_modified"]
},
"description": "Matching diagnostics objects"
}
},
"required": ["bucket", "objects"]
},
"CreateVoiceRegionResponse": {
"type": "object",
"properties": {
+1
View File
@@ -136,6 +136,7 @@ impl AdminApiClient {
let body = generated_types::BanGuildMemberRequest {
ban_duration_seconds: None,
delete_message_days: None,
delete_message_seconds: None,
guild_id: snowflake(guild_id),
reason: None,
user_id: snowflake(user_id),
@@ -1,93 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Readable} from 'node:stream';
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
import {
VoiceDiagnosticsObjectListResponse,
VoiceDiagnosticsQueryRequest,
} from '@fluxer/schema/src/domains/admin/AdminVoiceSchemas';
import {VOICE_DIAGNOSTICS_BUCKET, VoiceDiagnosticsService} from '../../channel/services/VoiceDiagnosticsService';
import {requireAdminACL} from '../../middleware/AdminMiddleware';
import {RateLimitMiddleware} from '../../middleware/RateLimitMiddleware';
import {OpenAPI} from '../../middleware/ResponseTypeMiddleware';
import {RateLimitConfigs} from '../../RateLimitConfig';
import type {HonoApp} from '../../types/HonoEnv';
import {Validator} from '../../Validator';
export function VoiceDiagnosticsAdminController(app: HonoApp) {
app.get(
'/admin/voice/diagnostics/objects',
RateLimitMiddleware(RateLimitConfigs.ADMIN_LOOKUP),
requireAdminACL(AdminACLs.VOICE_DIAGNOSTICS_VIEW),
Validator('query', VoiceDiagnosticsQueryRequest),
OpenAPI({
operationId: 'list_voice_diagnostics_objects',
summary: 'List voice diagnostics objects',
responseSchema: VoiceDiagnosticsObjectListResponse,
statusCode: 200,
security: 'adminApiKey',
tags: 'Admin',
description:
'Lists raw voice diagnostics NDJSON S3 objects for a channel and time range. Requires VOICE_DIAGNOSTICS_VIEW permission.',
}),
async (ctx) => {
const query = ctx.req.valid('query');
const service = new VoiceDiagnosticsService(
ctx.get('cacheService'),
ctx.get('channelService'),
ctx.get('gatewayService'),
ctx.get('storageService'),
);
const objects = await service.listObjects({
channelId: query.channel_id,
startMs: query.start_ms,
endMs: query.end_ms,
sessionId: query.session_id,
limitObjects: query.limit_objects,
});
return ctx.json({bucket: VOICE_DIAGNOSTICS_BUCKET, objects});
},
);
app.get(
'/admin/voice/diagnostics/raw',
RateLimitMiddleware(RateLimitConfigs.ADMIN_LOOKUP),
requireAdminACL(AdminACLs.VOICE_DIAGNOSTICS_VIEW),
Validator('query', VoiceDiagnosticsQueryRequest),
OpenAPI({
operationId: 'stream_voice_diagnostics_raw',
summary: 'Stream raw voice diagnostics',
responseSchema: null,
statusCode: 200,
security: 'adminApiKey',
tags: 'Admin',
description:
'Streams matching voice diagnostics NDJSON objects for local processing. Requires VOICE_DIAGNOSTICS_VIEW permission.',
}),
async (ctx) => {
const query = ctx.req.valid('query');
const service = new VoiceDiagnosticsService(
ctx.get('cacheService'),
ctx.get('channelService'),
ctx.get('gatewayService'),
ctx.get('storageService'),
);
const objects = await service.listObjects({
channelId: query.channel_id,
startMs: query.start_ms,
endMs: query.end_ms,
sessionId: query.session_id,
limitObjects: query.limit_objects,
});
const stream = service.createRawObjectStream(objects);
return new Response(Readable.toWeb(stream) as ReadableStream, {
status: 200,
headers: {
'Content-Type': 'application/x-ndjson',
'Cache-Control': 'no-store, private',
'X-Fluxer-Diagnostics-Bucket': VOICE_DIAGNOSTICS_BUCKET,
'X-Fluxer-Diagnostics-Object-Count': objects.length.toString(),
},
});
},
);
}
@@ -23,7 +23,6 @@ import {SystemAdminController} from './SystemAdminController';
import {SystemDmAdminController} from './SystemDmAdminController';
import {UserAdminController} from './UserAdminController';
import {VoiceAdminController} from './VoiceAdminController';
import {VoiceDiagnosticsAdminController} from './VoiceDiagnosticsAdminController';
export function registerAdminControllers(app: HonoApp) {
AdminApiKeyAdminController(app);
@@ -42,7 +41,6 @@ export function registerAdminControllers(app: HonoApp) {
ReportAdminController(app);
BillingAdminController(app);
VoiceAdminController(app);
VoiceDiagnosticsAdminController(app);
GatewayAdminController(app);
SearchAdminController(app);
DiscoveryAdminController(app);
@@ -133,6 +133,7 @@ export class AdminGuildMembershipService {
guildId,
targetId,
deleteMessageDays: data.delete_message_days,
deleteMessageSeconds: data.delete_message_seconds,
reason: data.reason ?? undefined,
banDurationSeconds: data.ban_duration_seconds ?? undefined,
skipGuildAuditLog: true,
@@ -0,0 +1,79 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
import {Config} from '../../Config';
import type {ApiTestHarness} from '../../test/ApiTestHarness';
import {HTTP_STATUS} from '../../test/TestConstants';
import {createBuilderWithoutAuth} from '../../test/TestRequestBuilder';
import {createAuthHarness, createUniqueEmail, createUniqueUsername, registerUser} from './AuthTestUtils';
async function withCaptchaEnabled<T>(run: () => Promise<T>): Promise<T> {
const previousEnabled = Config.captcha.enabled;
const previousTestModeEnabled = Config.dev.testModeEnabled;
Config.captcha.enabled = true;
Config.dev.testModeEnabled = true;
try {
return await run();
} finally {
Config.captcha.enabled = previousEnabled;
Config.dev.testModeEnabled = previousTestModeEnabled;
}
}
async function registerAndFlag(
harness: ApiTestHarness,
flags: Array<string>,
): Promise<{email: string; password: string; userId: string}> {
const email = createUniqueEmail('captcha-flags');
const password = 'a-strong-password';
const reg = await registerUser(harness, {
email,
username: createUniqueUsername('captchaflags'),
global_name: 'Captcha Flags User',
password,
date_of_birth: '2000-01-01',
consent: true,
});
if (flags.length > 0) {
await createBuilderWithoutAuth(harness)
.post(`/test/users/${reg.user_id}/security-flags`)
.body({set_flags: flags})
.execute();
}
return {email, password, userId: reg.user_id};
}
describe('Auth Captcha Bypass Flags', () => {
let harness: ApiTestHarness;
beforeAll(async () => {
harness = await createAuthHarness();
});
beforeEach(async () => {
await harness.reset();
});
afterAll(async () => {
await harness?.shutdown();
});
it('lets APP_STORE_REVIEWER accounts log in without solving a captcha', async () => {
const account = await registerAndFlag(harness, ['APP_STORE_REVIEWER']);
await withCaptchaEnabled(async () => {
const resp = await createBuilderWithoutAuth<{token?: string; user_id?: string}>(harness)
.post('/auth/login')
.body({email: account.email, password: account.password})
.execute();
expect(resp.token).toBeTruthy();
expect(resp.user_id).toBe(account.userId);
});
});
it('still requires a captcha for accounts without the APP_STORE_REVIEWER flag', async () => {
const account = await registerAndFlag(harness, []);
await withCaptchaEnabled(async () => {
await createBuilderWithoutAuth(harness)
.post('/auth/login')
.body({email: account.email, password: account.password})
.expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.CAPTCHA_REQUIRED)
.execute();
});
});
});
@@ -1,110 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {UserFlags} from '@fluxer/constants/src/UserConstants';
import {MissingPermissionsError} from '@fluxer/errors/src/domains/core/MissingPermissionsError';
import {
VoiceDebugLoggingEventsBodySchema,
VoiceDebugLoggingToggleBodySchema,
} from '@fluxer/schema/src/domains/channel/ChannelRequestSchemas';
import {
VoiceDebugLoggingEventsResponse,
VoiceDebugLoggingStatusResponse,
} from '@fluxer/schema/src/domains/channel/ChannelSchemas';
import {ChannelIdParam} from '@fluxer/schema/src/domains/common/CommonParamSchemas';
import type {Context} from 'hono';
import {createChannelID} from '../../BrandedTypes';
import {DefaultUserOnly, LoginRequired} from '../../middleware/AuthMiddleware';
import {RateLimitMiddleware} from '../../middleware/RateLimitMiddleware';
import {OpenAPI} from '../../middleware/ResponseTypeMiddleware';
import {RateLimitConfigs} from '../../RateLimitConfig';
import type {HonoApp, HonoEnv} from '../../types/HonoEnv';
import {Validator} from '../../Validator';
import {VoiceDiagnosticsService} from '../services/VoiceDiagnosticsService';
function makeVoiceDiagnosticsService(ctx: Context<HonoEnv>): VoiceDiagnosticsService {
return new VoiceDiagnosticsService(
ctx.get('cacheService'),
ctx.get('channelService'),
ctx.get('gatewayService'),
ctx.get('storageService'),
);
}
export function VoiceDiagnosticsController(app: HonoApp) {
app.get(
'/channels/:channel_id/voice-debug-logging/session',
RateLimitMiddleware(RateLimitConfigs.CHANNEL_VOICE_DEBUG_LOGGING_STATUS),
LoginRequired,
DefaultUserOnly,
Validator('param', ChannelIdParam),
OpenAPI({
operationId: 'get_voice_debug_logging_status',
summary: 'Get voice debug logging status',
description:
'Returns whether staff-enabled voice debug logging is active for this channel. Clients poll this while connected to decide whether to upload diagnostics.',
responseSchema: VoiceDebugLoggingStatusResponse,
statusCode: 200,
security: ['bearerToken', 'sessionToken'],
tags: 'Channels',
}),
async (ctx) => {
const userId = ctx.get('user').id;
const channelId = createChannelID(ctx.req.valid('param').channel_id);
const service = makeVoiceDiagnosticsService(ctx);
return ctx.json(await service.getStatus({userId, channelId}));
},
);
app.put(
'/channels/:channel_id/voice-debug-logging/session',
RateLimitMiddleware(RateLimitConfigs.CHANNEL_VOICE_DEBUG_LOGGING_TOGGLE),
LoginRequired,
DefaultUserOnly,
Validator('param', ChannelIdParam),
Validator('json', VoiceDebugLoggingToggleBodySchema),
OpenAPI({
operationId: 'set_voice_debug_logging_status',
summary: 'Toggle voice debug logging',
description:
'Allows staff to start or stop a channel-scoped voice debug logging session. Non-staff users cannot activate or stop sessions.',
responseSchema: VoiceDebugLoggingStatusResponse,
statusCode: 200,
security: ['bearerToken', 'sessionToken'],
tags: 'Channels',
}),
async (ctx) => {
const user = ctx.get('user');
if ((user.flags & UserFlags.STAFF) === 0n) {
throw new MissingPermissionsError();
}
const channelId = createChannelID(ctx.req.valid('param').channel_id);
const {enabled, duration_ms} = ctx.req.valid('json');
const service = makeVoiceDiagnosticsService(ctx);
return ctx.json(await service.setSession({userId: user.id, channelId, enabled, durationMs: duration_ms}));
},
);
app.post(
'/channels/:channel_id/voice-debug-logging/events',
RateLimitMiddleware(RateLimitConfigs.CHANNEL_VOICE_DEBUG_LOGGING_EVENTS),
LoginRequired,
DefaultUserOnly,
Validator('param', ChannelIdParam),
Validator('json', VoiceDebugLoggingEventsBodySchema),
OpenAPI({
operationId: 'upload_voice_debug_logging_events',
summary: 'Upload voice debug logging events',
description:
'Uploads a small batch of client voice diagnostics events for an active staff-enabled debug logging session.',
responseSchema: VoiceDebugLoggingEventsResponse,
statusCode: 200,
security: ['bearerToken', 'sessionToken'],
tags: 'Channels',
}),
async (ctx) => {
const userId = ctx.get('user').id;
const channelId = createChannelID(ctx.req.valid('param').channel_id);
const body = ctx.req.valid('json');
const service = makeVoiceDiagnosticsService(ctx);
return ctx.json(await service.ingestEvents({userId, channelId, body}));
},
);
}
@@ -1,84 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {VoicePresenceHeartbeatBodySchema} from '@fluxer/schema/src/domains/channel/ChannelRequestSchemas';
import {
VoicePresenceHeartbeatEndResponse,
VoicePresenceHeartbeatResponse,
} from '@fluxer/schema/src/domains/channel/ChannelSchemas';
import {ChannelIdParam} from '@fluxer/schema/src/domains/common/CommonParamSchemas';
import {createChannelID} from '../../BrandedTypes';
import {DefaultUserOnly, LoginRequired} from '../../middleware/AuthMiddleware';
import {RateLimitMiddleware} from '../../middleware/RateLimitMiddleware';
import {OpenAPI} from '../../middleware/ResponseTypeMiddleware';
import {RateLimitConfigs} from '../../RateLimitConfig';
import type {HonoApp} from '../../types/HonoEnv';
import {Validator} from '../../Validator';
import {VoicePresenceHeartbeatStore} from '../../voice/VoicePresenceHeartbeatStore';
export function VoicePresenceController(app: HonoApp) {
app.post(
'/channels/:channel_id/voice-presence/heartbeat',
RateLimitMiddleware(RateLimitConfigs.CHANNEL_VOICE_PRESENCE_HEARTBEAT),
LoginRequired,
DefaultUserOnly,
Validator('param', ChannelIdParam),
Validator('json', VoicePresenceHeartbeatBodySchema),
OpenAPI({
operationId: 'heartbeat_voice_presence',
summary: 'Heartbeat voice presence',
description:
'Refreshes the current user voice presence marker for v2 voice reconciliation. Clients call this while connected to voice.',
responseSchema: VoicePresenceHeartbeatResponse,
statusCode: 200,
security: ['bearerToken', 'sessionToken'],
tags: 'Channels',
}),
async (ctx) => {
const userId = ctx.get('user').id;
const channelId = createChannelID(ctx.req.valid('param').channel_id);
const body = ctx.req.valid('json');
const store = new VoicePresenceHeartbeatStore(ctx.get('apiContext').services.kv);
const heartbeat = await store.recordHeartbeat({
channelId,
userId,
connectionId: body.connection_id,
});
return ctx.json({
ok: true,
heartbeat_interval_ms: heartbeat.heartbeatIntervalMs,
heartbeat_ttl_ms: heartbeat.heartbeatTtlMs,
expires_at_ms: heartbeat.expiresAtMs,
});
},
);
app.delete(
'/channels/:channel_id/voice-presence/heartbeat',
RateLimitMiddleware(RateLimitConfigs.CHANNEL_VOICE_PRESENCE_HEARTBEAT),
LoginRequired,
DefaultUserOnly,
Validator('param', ChannelIdParam),
Validator('json', VoicePresenceHeartbeatBodySchema),
OpenAPI({
operationId: 'end_voice_presence_heartbeat',
summary: 'End voice presence heartbeat',
description:
'Clears the current user active v2 voice presence marker for a voice connection while preserving v2 enrollment for fast reconciliation.',
responseSchema: VoicePresenceHeartbeatEndResponse,
statusCode: 200,
security: ['bearerToken', 'sessionToken'],
tags: 'Channels',
}),
async (ctx) => {
const userId = ctx.get('user').id;
const channelId = createChannelID(ctx.req.valid('param').channel_id);
const body = ctx.req.valid('json');
const store = new VoicePresenceHeartbeatStore(ctx.get('apiContext').services.kv);
await store.markHeartbeatEnded({
channelId,
userId,
connectionId: body.connection_id,
});
return ctx.json({ok: true});
},
);
}
@@ -7,8 +7,6 @@ import {MessageController} from './MessageController';
import {MessageInteractionController} from './MessageInteractionController';
import {ScheduledMessageController} from './ScheduledMessageController';
import {StreamController} from './StreamController';
import {VoiceDiagnosticsController} from './VoiceDiagnosticsController';
import {VoicePresenceController} from './VoicePresenceController';
export function registerChannelControllers(app: HonoApp) {
ChannelController(app);
@@ -17,6 +15,4 @@ export function registerChannelControllers(app: HonoApp) {
ScheduledMessageController(app);
CallController(app);
StreamController(app);
VoiceDiagnosticsController(app);
VoicePresenceController(app);
}
@@ -1,336 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Readable} from 'node:stream';
import {Permissions} from '@fluxer/constants/src/ChannelConstants';
import {MissingPermissionsError} from '@fluxer/errors/src/domains/core/MissingPermissionsError';
import type {
VoiceDebugLoggingEventSchema,
VoiceDebugLoggingEventsBodySchema,
} from '@fluxer/schema/src/domains/channel/ChannelRequestSchemas';
import type {
VoiceDebugLoggingEventsResponse,
VoiceDebugLoggingStatusResponse,
} from '@fluxer/schema/src/domains/channel/ChannelSchemas';
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
import type {ChannelID, UserID} from '../../BrandedTypes';
import type {IGatewayService} from '../../infrastructure/IGatewayService';
import type {IStorageService} from '../../infrastructure/IStorageService';
import type {ChannelService} from './ChannelService';
export const VOICE_DIAGNOSTICS_BUCKET = 'fluxer-voice-diagnostics';
const VOICE_DEBUG_LOGGING_POLL_INTERVAL_MS = 10000;
const VOICE_DEBUG_LOGGING_UPLOAD_INTERVAL_MS = 2000;
const DEFAULT_SESSION_DURATION_MS = 60 * 60 * 1000;
const MAX_SESSION_DURATION_MS = 4 * 60 * 60 * 1000;
const NDJSON_CONTENT_TYPE = 'application/x-ndjson';
const TEXT_ENCODER = new TextEncoder();
interface ActiveVoiceDebugLoggingSession {
session_id: string;
channel_id: string;
activated_by_user_id: string;
started_at_ms: number;
expires_at_ms: number;
}
interface VoiceDiagnosticsObject {
key: string;
session_id: string;
start_ns: string;
end_ns: string;
last_modified: string | null;
}
function sessionCacheKey(channelId: ChannelID): string {
return `voice_debug_logging:channel:${channelId.toString()}`;
}
function createId(prefix: string): string {
const randomUUID = globalThis.crypto?.randomUUID?.bind(globalThis.crypto);
if (randomUUID) return `${prefix}_${randomUUID()}`;
return `${prefix}_${Date.now().toString(36)}_${Math.random().toString(36).slice(2)}`;
}
function datePartFromMs(timestampMs: number): string {
return new Date(timestampMs).toISOString().slice(0, 10);
}
function sanitizeKeyPart(value: string): string {
return value.replace(/[^A-Za-z0-9_.:-]/g, '_').slice(0, 256) || 'unknown';
}
function minMaxEventTimestampNs(events: ReadonlyArray<VoiceDebugLoggingEventSchema>): {
startNs: bigint;
endNs: bigint;
} {
let startNs: bigint | null = null;
let endNs: bigint | null = null;
for (const event of events) {
const timestampNs = BigInt(event.timestamp_ns);
if (startNs === null || timestampNs < startNs) startNs = timestampNs;
if (endNs === null || timestampNs > endNs) endNs = timestampNs;
}
return {startNs: startNs ?? 0n, endNs: endNs ?? 0n};
}
function buildDiagnosticsKey(params: {
channelId: ChannelID;
sessionId: string;
userId: UserID;
connectionId?: string;
participantIdentity?: string;
startNs: bigint;
endNs: bigint;
}): string {
const timestampMs = Number(params.startNs / 1000000n);
const date = datePartFromMs(timestampMs > 0 ? timestampMs : Date.now());
return [
`channel_id=${params.channelId.toString()}`,
`date=${date}`,
`session_id=${sanitizeKeyPart(params.sessionId)}`,
`start_ns=${params.startNs.toString()}_end_ns=${params.endNs.toString()}`,
`participant=${sanitizeKeyPart(params.userId.toString())}`,
`connection=${sanitizeKeyPart(params.connectionId ?? 'unknown')}`,
`identity=${sanitizeKeyPart(params.participantIdentity ?? 'unknown')}`,
`batch=${createId('batch')}.ndjson`,
].join('/');
}
function parseDiagnosticsKey(key: string): VoiceDiagnosticsObject | null {
const match = key.match(/^channel_id=[^/]+\/date=[^/]+\/session_id=([^/]+)\/start_ns=([0-9]+)_end_ns=([0-9]+)\//);
if (!match) return null;
const [, sessionId, startNs, endNs] = match;
if (!sessionId || !startNs || !endNs) return null;
return {
key,
session_id: sessionId,
start_ns: startNs,
end_ns: endNs,
last_modified: null,
};
}
function buildDatePrefixes(params: {
channelId: string;
startMs: number;
endMs: number;
sessionId?: string;
}): Array<string> {
const prefixes: Array<string> = [];
const cursor = new Date(params.startMs);
cursor.setUTCHours(0, 0, 0, 0);
const end = new Date(params.endMs);
end.setUTCHours(0, 0, 0, 0);
while (cursor.getTime() <= end.getTime()) {
const base = `channel_id=${params.channelId}/date=${datePartFromMs(cursor.getTime())}/`;
prefixes.push(params.sessionId ? `${base}session_id=${sanitizeKeyPart(params.sessionId)}/` : base);
cursor.setUTCDate(cursor.getUTCDate() + 1);
}
return prefixes;
}
async function assertVoiceChannelAccess(params: {
channelService: ChannelService;
gatewayService: IGatewayService;
userId: UserID;
channelId: ChannelID;
}): Promise<void> {
const channel = await params.channelService.channelData.operations.getChannel({
userId: params.userId,
channelId: params.channelId,
});
if (!channel.guildId) return;
const hasConnect = await params.gatewayService.checkPermission({
guildId: channel.guildId,
channelId: params.channelId,
userId: params.userId,
permission: Permissions.CONNECT,
});
if (!hasConnect) {
throw new MissingPermissionsError();
}
}
export class VoiceDiagnosticsService {
constructor(
private readonly cacheService: ICacheService,
private readonly channelService: ChannelService,
private readonly gatewayService: IGatewayService,
private readonly storageService: IStorageService,
) {}
private async getActiveSession(channelId: ChannelID): Promise<ActiveVoiceDebugLoggingSession | null> {
const session = await this.cacheService.get<ActiveVoiceDebugLoggingSession>(sessionCacheKey(channelId));
if (!session) return null;
if (session.expires_at_ms <= Date.now()) {
await this.cacheService.delete(sessionCacheKey(channelId));
return null;
}
return session;
}
private statusFromSession(session: ActiveVoiceDebugLoggingSession | null): VoiceDebugLoggingStatusResponse {
return {
active: session !== null,
session_id: session?.session_id ?? null,
activated_by_user_id: session?.activated_by_user_id ?? null,
started_at_ms: session?.started_at_ms ?? null,
expires_at_ms: session?.expires_at_ms ?? null,
poll_interval_ms: VOICE_DEBUG_LOGGING_POLL_INTERVAL_MS,
upload_interval_ms: VOICE_DEBUG_LOGGING_UPLOAD_INTERVAL_MS,
};
}
async getStatus(params: {userId: UserID; channelId: ChannelID}): Promise<VoiceDebugLoggingStatusResponse> {
await assertVoiceChannelAccess({
channelService: this.channelService,
gatewayService: this.gatewayService,
userId: params.userId,
channelId: params.channelId,
});
return this.statusFromSession(await this.getActiveSession(params.channelId));
}
async setSession(params: {
userId: UserID;
channelId: ChannelID;
enabled: boolean;
durationMs?: number;
}): Promise<VoiceDebugLoggingStatusResponse> {
await assertVoiceChannelAccess({
channelService: this.channelService,
gatewayService: this.gatewayService,
userId: params.userId,
channelId: params.channelId,
});
const key = sessionCacheKey(params.channelId);
if (!params.enabled) {
await this.cacheService.delete(key);
return this.statusFromSession(null);
}
const now = Date.now();
const durationMs = Math.min(params.durationMs ?? DEFAULT_SESSION_DURATION_MS, MAX_SESSION_DURATION_MS);
const session: ActiveVoiceDebugLoggingSession = {
session_id: createId('voice_debug'),
channel_id: params.channelId.toString(),
activated_by_user_id: params.userId.toString(),
started_at_ms: now,
expires_at_ms: now + durationMs,
};
await this.cacheService.set(key, session, Math.ceil(durationMs / 1000));
return this.statusFromSession(session);
}
async ingestEvents(params: {
userId: UserID;
channelId: ChannelID;
body: VoiceDebugLoggingEventsBodySchema;
}): Promise<VoiceDebugLoggingEventsResponse> {
await assertVoiceChannelAccess({
channelService: this.channelService,
gatewayService: this.gatewayService,
userId: params.userId,
channelId: params.channelId,
});
const session = await this.getActiveSession(params.channelId);
if (!session || session.session_id !== params.body.session_id) {
return {accepted: false, active: session !== null, stored_event_count: 0};
}
const {startNs, endNs} = minMaxEventTimestampNs(params.body.events);
const key = buildDiagnosticsKey({
channelId: params.channelId,
sessionId: session.session_id,
userId: params.userId,
connectionId: params.body.connection_id,
participantIdentity: params.body.participant_identity,
startNs,
endNs,
});
const serverReceivedAtMs = Date.now();
const serverReceivedMonotonicNs = process.hrtime.bigint().toString();
const lines = params.body.events.map((event) =>
JSON.stringify({
schema_version: 1,
server_received_at_ms: serverReceivedAtMs,
server_received_monotonic_ns: serverReceivedMonotonicNs,
channel_id: params.channelId.toString(),
session_id: session.session_id,
activated_by_user_id: session.activated_by_user_id,
participant_user_id: params.userId.toString(),
connection_id: params.body.connection_id ?? null,
participant_identity: params.body.participant_identity ?? null,
event,
}),
);
await this.storageService.uploadObject({
bucket: VOICE_DIAGNOSTICS_BUCKET,
key,
body: TEXT_ENCODER.encode(`${lines.join('\n')}\n`),
contentType: NDJSON_CONTENT_TYPE,
});
return {accepted: true, active: true, stored_event_count: params.body.events.length};
}
async listObjects(params: {
channelId: string;
startMs: number;
endMs: number;
sessionId?: string;
limitObjects: number;
}): Promise<Array<VoiceDiagnosticsObject>> {
const startNs = BigInt(Math.floor(params.startMs)) * 1000000n;
const endNs = BigInt(Math.floor(params.endMs)) * 1000000n;
const prefixes = buildDatePrefixes({
channelId: params.channelId,
startMs: params.startMs,
endMs: params.endMs,
sessionId: params.sessionId,
});
const listed = await Promise.all(
prefixes.map((prefix) => this.storageService.listObjects({bucket: VOICE_DIAGNOSTICS_BUCKET, prefix})),
);
return listed
.flat()
.flatMap((object): Array<VoiceDiagnosticsObject> => {
const parsed = parseDiagnosticsKey(object.key);
if (!parsed) return [];
return [
{
...parsed,
last_modified: object.lastModified?.toISOString() ?? null,
},
];
})
.filter((object) => {
const objectStartNs = BigInt(object.start_ns);
const objectEndNs = BigInt(object.end_ns);
return objectEndNs >= startNs && objectStartNs <= endNs;
})
.sort((a, b) => {
const delta = BigInt(a.start_ns) - BigInt(b.start_ns);
if (delta < 0n) return -1;
if (delta > 0n) return 1;
return a.key.localeCompare(b.key);
})
.slice(0, params.limitObjects);
}
createRawObjectStream(objects: ReadonlyArray<VoiceDiagnosticsObject>): Readable {
const storageService = this.storageService;
async function* streamObjects(): AsyncGenerator<Uint8Array> {
for (const object of objects) {
const stream = await storageService.streamObject({
bucket: VOICE_DIAGNOSTICS_BUCKET,
key: object.key,
});
if (!stream) continue;
for await (const chunk of stream.body) {
yield typeof chunk === 'string' ? TEXT_ENCODER.encode(chunk) : new Uint8Array(chunk);
}
yield TEXT_ENCODER.encode('\n');
}
}
return Readable.from(streamObjects());
}
}
@@ -225,14 +225,14 @@ export class MessageDeleteService {
async deleteUserMessagesInGuild({
userId,
guildId,
days,
seconds,
}: {
userId: UserID;
guildId: GuildID;
days: number;
seconds: number;
}): Promise<void> {
const channels = await this.deps.channelRepository.channelData.listGuildChannels(guildId);
const cutoffTimestamp = Date.now() - days * ms('1 day');
const cutoffTimestamp = Date.now() - seconds * ms('1 second');
const cutoffSnowflake = createMessageID(createSnowflakeFromTimestamp(cutoffTimestamp));
await Promise.all(
channels.map(async (channel: Channel) => {
@@ -14,6 +14,7 @@ import type {IFavoriteMemeRepository} from '../../../favorite_meme/IFavoriteMeme
import type {IMediaService} from '../../../infrastructure/IMediaService';
import type {ISnowflakeService} from '../../../infrastructure/ISnowflakeService';
import type {IStorageService} from '../../../infrastructure/IStorageService';
import {Logger} from '../../../Logger';
import type {FavoriteMeme} from '../../../models/FavoriteMeme';
import type {Message} from '../../../models/Message';
import type {User} from '../../../models/User';
@@ -90,8 +91,21 @@ export class MessageOperationsHelpers {
}
throw error;
}
const needsAnimationProbe =
!favoriteMeme.isGifv &&
favoriteMeme.contentType !== 'image/gif' &&
favoriteMeme.contentType !== 'image/apng' &&
ANIMATION_PROBE_CONTENT_TYPES.has(favoriteMeme.contentType);
const metadata =
needsAnimationProbe || favoriteMeme.placeholder == null
? await this.probeFavoriteMemeMetadata(favoriteMeme)
: null;
const placeholder = favoriteMeme.placeholder ?? metadata?.placeholder ?? null;
if (placeholder != null && favoriteMeme.placeholder == null) {
await this.repairFavoriteMemePlaceholder(favoriteMeme, placeholder);
}
let flags = 0;
if (await this.isFavoriteMemeAnimated(favoriteMeme)) {
if (this.isFavoriteMemeAnimated(favoriteMeme, metadata)) {
flags |= MessageAttachmentFlags.IS_ANIMATED;
}
return {
@@ -104,7 +118,7 @@ export class MessageOperationsHelpers {
height: favoriteMeme.height,
content_type: favoriteMeme.contentType,
content_hash: favoriteMeme.contentHash,
placeholder: null,
placeholder,
flags,
duration: favoriteMeme.duration,
nsfw: null,
@@ -112,20 +126,31 @@ export class MessageOperationsHelpers {
};
}
private async isFavoriteMemeAnimated(favoriteMeme: FavoriteMeme): Promise<boolean> {
if (favoriteMeme.isGifv) return true;
if (favoriteMeme.contentType === 'image/gif' || favoriteMeme.contentType === 'image/apng') return true;
if (!ANIMATION_PROBE_CONTENT_TYPES.has(favoriteMeme.contentType)) return false;
private async probeFavoriteMemeMetadata(favoriteMeme: FavoriteMeme) {
try {
const metadata = await this.deps.mediaService.getMetadata({
return await this.deps.mediaService.getMetadata({
type: 's3',
bucket: Config.s3.buckets.cdn,
key: favoriteMeme.storageKey,
nsfw: 'allow',
});
return metadata?.animated === true;
} catch {
return false;
return null;
}
}
private async repairFavoriteMemePlaceholder(favoriteMeme: FavoriteMeme, placeholder: string): Promise<void> {
try {
await this.deps.favoriteMemeRepository.updatePlaceholder(favoriteMeme.userId, favoriteMeme.id, placeholder);
} catch (error) {
Logger.warn({error, memeId: favoriteMeme.id.toString()}, 'Failed to backfill favorite meme placeholder');
}
}
private isFavoriteMemeAnimated(favoriteMeme: FavoriteMeme, metadata: {animated?: boolean | null} | null): boolean {
if (favoriteMeme.isGifv) return true;
if (favoriteMeme.contentType === 'image/gif' || favoriteMeme.contentType === 'image/apng') return true;
if (!ANIMATION_PROBE_CONTENT_TYPES.has(favoriteMeme.contentType)) return false;
return metadata?.animated === true;
}
}
@@ -30,6 +30,7 @@ import {Config} from '../Config';
import type {IChannelRepository} from '../channel/IChannelRepository';
import type {AttachmentUploadTraceRepository} from '../channel/repositories/message/AttachmentUploadTraceRepository';
import {
collectMessageAttachments,
makeAttachmentCdnKey,
makeAttachmentCdnUrl,
purgeMessageAttachments,
@@ -43,6 +44,9 @@ import type {IStorageService} from '../infrastructure/IStorageService';
import type {KVAccountDeletionQueueService} from '../infrastructure/KVAccountDeletionQueueService';
import type {UserCacheService} from '../infrastructure/UserCacheService';
import {Logger} from '../Logger';
import type {Embed} from '../models/Embed';
import type {EmbedMedia} from '../models/EmbedMedia';
import type {Message} from '../models/Message';
import type {User} from '../models/User';
import type {IARMessageContext, IARSubmission} from '../report/IReportRepository';
import type {ReportRepository} from '../report/ReportRepository';
@@ -471,18 +475,34 @@ export class NcmecSubmissionService {
private async resolveAttachment(input: SubmitAttachmentToNcmecInput): Promise<ResolvedAttachment> {
const liveMessage = await this.deps.channelRepository.getMessage(input.channelId, input.messageId);
const liveAttachment = liveMessage?.attachments.find(
(attachment) => attachment.id === input.attachmentId && attachment.filename === input.filename,
);
if (liveMessage?.authorId && liveAttachment) {
return this.buildAttachmentContext(input, {
contentType: liveAttachment.contentType,
reportedAt: snowflakeToDate(input.messageId),
userId: liveMessage.authorId,
sourceReportId: input.sourceReportId ?? null,
bucket: Config.s3.buckets.cdn,
storageKey: makeAttachmentCdnKey(input.channelId, input.attachmentId, input.filename),
});
if (liveMessage?.authorId) {
const liveAttachment = collectMessageAttachments(liveMessage).find(
(attachment) => attachment.id === input.attachmentId && attachment.filename === input.filename,
);
if (liveAttachment) {
return this.buildAttachmentContext(input, {
contentType: liveAttachment.contentType,
reportedAt: snowflakeToDate(input.messageId),
userId: liveMessage.authorId,
sourceReportId: input.sourceReportId ?? null,
bucket: Config.s3.buckets.cdn,
storageKey: makeAttachmentCdnKey(input.channelId, input.attachmentId, input.filename),
});
}
const embedMedia = findEmbedReferencedAttachmentMedia(
liveMessage,
makeAttachmentCdnUrl(input.channelId, input.attachmentId, input.filename),
);
if (embedMedia) {
return this.buildAttachmentContext(input, {
contentType: embedMedia.contentType,
reportedAt: snowflakeToDate(input.messageId),
userId: liveMessage.authorId,
sourceReportId: input.sourceReportId ?? null,
bucket: Config.s3.buckets.cdn,
storageKey: makeAttachmentCdnKey(input.channelId, input.attachmentId, input.filename),
});
}
}
if (!input.sourceReportId) {
throw new UnknownMessageError();
@@ -939,6 +959,30 @@ function buildAuditMetadata(args: {
return metadata;
}
function findEmbedReferencedAttachmentMedia(message: Message, targetCdnUrl: string): EmbedMedia | null {
const scan = (embeds: Array<Embed>): EmbedMedia | null => {
for (const embed of embeds) {
for (const media of [embed.image, embed.thumbnail, embed.video, embed.audio]) {
if (media?.url === targetCdnUrl) {
return media;
}
}
}
return null;
};
const direct = scan(message.embeds);
if (direct) {
return direct;
}
for (const snapshot of message.messageSnapshots) {
const found = scan(snapshot.embeds);
if (found) {
return found;
}
}
return null;
}
function findAttachmentInReport(
report: IARSubmission,
channelId: ChannelID,
@@ -0,0 +1,144 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import cassandra from 'cassandra-driver';
import {describe, expect, it} from 'vitest';
import type {CassandraParams, KvQueryMeta, KvTableSpec, WhereExpr} from './CassandraTypes';
import {buildCandidatePlan, keyFromColumns, matchesWhere} from './PostgresKvQueryExecutor';
type Row = Record<string, unknown>;
type AnyMeta = KvQueryMeta<Row>;
const Spec: KvTableSpec<Row> = {
name: 'fuzz_two',
columns: ['k', 'c', 'v'],
primaryKey: ['k', 'c'],
partitionKey: ['k', 'c'],
};
const SpecOne: KvTableSpec<Row> = {
name: 'fuzz_one',
columns: ['k', 'v'],
primaryKey: ['k'],
partitionKey: ['k'],
};
const VALUES: Array<[string, unknown]> = [
['null', null],
['undefined', undefined],
['zero', 0],
['neg-zero', -0],
['one-num', 1],
['one-big', 1n],
['one-str', '1'],
['empty-str', ''],
['true', true],
['false', false],
['date-epoch', new Date(0)],
['date-epoch-2', new Date(0)],
['buf-a', Buffer.from('a')],
['buf-a-2', Buffer.from('a')],
['localdate', cassandra.types.LocalDate.fromString('2020-01-01')],
['localdate-str', '2020-01-01'],
];
function rowKeyMatchesPlan(plan: ReturnType<typeof buildCandidatePlan>, rowKey: string): boolean {
const c = plan.candidates;
const buf = Buffer.from(rowKey, 'utf8');
switch (c.kind) {
case 'none':
return false;
case 'scan':
return true;
case 'rowKeys':
return c.rowKeys.includes(rowKey);
case 'range':
return (
Buffer.compare(buf, Buffer.from(c.lowerBound, 'utf8')) >= 0 &&
Buffer.compare(buf, Buffer.from(c.upperBound, 'utf8')) < 0
);
case 'ranges':
return c.lowerBounds.some(
(lower, i) =>
Buffer.compare(buf, Buffer.from(lower, 'utf8')) >= 0 &&
Buffer.compare(buf, Buffer.from(c.upperBounds[i]!, 'utf8')) < 0,
);
case 'partitionKeys':
return c.partitionKeys.includes(rowKey);
}
}
describe('postgres kv pushdown exactness', () => {
it('never excludes a row the javascript filter would keep, and is equal when exact', () => {
const narrowing: Array<string> = [];
const widening: Array<string> = [];
const meta = {
action: 'count',
table: Spec,
where: [{kind: 'eq', col: 'k', param: 'k'}] as Array<WhereExpr<Row>>,
} as AnyMeta;
for (const [paramName, paramValue] of VALUES) {
const params = {k: paramValue} as CassandraParams;
const plan = buildCandidatePlan(meta, params);
for (const [storedName, storedValue] of VALUES) {
const row: Row = {k: storedValue, c: 'x', v: 1};
const rowKey = keyFromColumns(Spec.primaryKey as Array<string>, row);
const js = matchesWhere(row, meta.where as Array<WhereExpr<Row>>, params);
const sql = rowKeyMatchesPlan(plan, rowKey);
const label = `param=${paramName} stored=${storedName} exact=${plan.exact} plan=${plan.candidates.kind}`;
if (js && !sql) narrowing.push(label);
if (plan.exact && sql && !js) widening.push(label);
}
}
expect({narrowing, widening}).toEqual({narrowing: [], widening: []});
});
it('single-column primary keys behave the same', () => {
const narrowing: Array<string> = [];
const widening: Array<string> = [];
const meta = {
action: 'count',
table: SpecOne,
where: [{kind: 'eq', col: 'k', param: 'k'}] as Array<WhereExpr<Row>>,
} as AnyMeta;
for (const [paramName, paramValue] of VALUES) {
const params = {k: paramValue} as CassandraParams;
const plan = buildCandidatePlan(meta, params);
for (const [storedName, storedValue] of VALUES) {
const row: Row = {k: storedValue, v: 1};
const rowKey = keyFromColumns(SpecOne.primaryKey as Array<string>, row);
const js = matchesWhere(row, meta.where as Array<WhereExpr<Row>>, params);
const sql = rowKeyMatchesPlan(plan, rowKey);
const label = `param=${paramName} stored=${storedName} exact=${plan.exact} plan=${plan.candidates.kind}`;
if (js && !sql) narrowing.push(label);
if (plan.exact && sql && !js) widening.push(label);
}
}
expect({narrowing, widening}).toEqual({narrowing: [], widening: []});
});
it('IN lists behave the same', () => {
const narrowing: Array<string> = [];
const widening: Array<string> = [];
const meta = {
action: 'count',
table: SpecOne,
where: [{kind: 'in', col: 'k', param: 'ks'}] as Array<WhereExpr<Row>>,
} as AnyMeta;
for (let i = 0; i < VALUES.length; i += 1) {
for (let j = i; j < VALUES.length; j += 1) {
const params = {ks: [VALUES[i]![1], VALUES[j]![1]]} as CassandraParams;
const plan = buildCandidatePlan(meta, params);
for (const [storedName, storedValue] of VALUES) {
const row: Row = {k: storedValue, v: 1};
const rowKey = keyFromColumns(SpecOne.primaryKey as Array<string>, row);
const js = matchesWhere(row, meta.where as Array<WhereExpr<Row>>, params);
const sql = rowKeyMatchesPlan(plan, rowKey);
const label = `in=[${VALUES[i]![0]},${VALUES[j]![0]}] stored=${storedName} exact=${plan.exact} plan=${plan.candidates.kind}`;
if (js && !sql) narrowing.push(label);
if (plan.exact && sql && !js) widening.push(label);
}
}
}
expect({narrowing: narrowing.slice(0, 10), widening: widening.slice(0, 10)}).toEqual({narrowing: [], widening: []});
});
});
@@ -0,0 +1,52 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {describe, expect, it, vi} from 'vitest';
import {defineTable} from './CassandraTableDsl';
import type {KvQueryMeta} from './CassandraTypes';
vi.mock('../Logger', () => ({
Logger: new Proxy(
{},
{
get() {
throw new Error('Logger has not been initialized. Call initializeLogger() first.');
},
},
),
}));
type Row = Record<string, unknown>;
const Probe = defineTable<{k: string; v: string}, 'k'>({
name: 'kv_logger_probe',
columns: ['k', 'v'],
primaryKey: ['k'],
});
const client = {
async query() {
return {rows: [], rowCount: 0};
},
async connect() {},
async shutdown() {},
isConnected() {
return true;
},
async transaction(fn: (db: unknown) => Promise<unknown>) {
return fn(client);
},
kvTable() {
return 'kv_logger_probe_table';
},
} as never;
describe('PostgresKvQueryExecutor logging safety', () => {
it('runs full scan queries in a process whose logger was never initialized', async () => {
const {PostgresKvQueryExecutor} = await import('./PostgresKvQueryExecutor');
const executor = new PostgresKvQueryExecutor(client);
for (const action of ['select', 'count', 'delete'] as const) {
const meta = {action, table: Probe, where: [], columns: ['k', 'v']} as unknown as KvQueryMeta<Row>;
await expect(executor.executeQuery({cql: `__probe_${action}`, params: {}, kvMeta: meta})).resolves.toBeDefined();
}
});
});
@@ -0,0 +1,335 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {execFileSync, spawnSync} from 'node:child_process';
import {createServer} from 'node:net';
import {
getDefaultPostgresClient,
type IPostgresClient,
initPostgres,
type PostgresQueryable,
shutdownPostgres,
} from '@pkgs/postgres/src/Client';
import {afterAll, beforeAll, describe, expect, it} from 'vitest';
import {LegacyPostgresKvQueryExecutor} from './__testref__/LegacyPostgresKvQueryExecutor';
import type {CassandraParams, KvQueryMeta, KvTableSpec, WhereExpr} from './CassandraTypes';
import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';
type Row = Record<string, unknown>;
type AnyMeta = KvQueryMeta<Row>;
const LEGACY_TABLE = 'kvns_legacy';
const NEXT_TABLE = 'kvns_next';
const CONTAINER = `fluxer-kvns-${process.pid.toString(36)}-${Date.now().toString(36)}`;
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
class TableClient implements IPostgresClient {
constructor(
private readonly inner: IPostgresClient,
private readonly table: string,
) {}
async query<T extends Record<string, unknown>>(text: string, values: Array<unknown> = []) {
return (await this.inner.query(text, values)) as unknown as Awaited<ReturnType<IPostgresClient['query']>> & {
rows: Array<T>;
};
}
async connect(): Promise<void> {
await this.inner.connect();
}
async shutdown(): Promise<void> {}
isConnected(): boolean {
return this.inner.isConnected();
}
async transaction<T>(fn: (client: PostgresQueryable) => Promise<T>): Promise<T> {
return this.inner.transaction(fn);
}
kvTable(): string {
return this.table;
}
}
const Parent: KvTableSpec<Row> = {
name: 'ns_parent',
columns: ['user_id', 'channel_id', 'note', 'blob_'],
primaryKey: ['user_id', 'channel_id'],
partitionKey: ['user_id', 'channel_id'],
};
const Single: KvTableSpec<Row> = {
name: 'ns_single',
columns: ['token_', 'note'],
primaryKey: ['token_'],
partitionKey: ['token_'],
};
function eq(col: string, param: string): WhereExpr<Row> {
return {kind: 'eq', col, param} as WhereExpr<Row>;
}
function inClause(col: string, param: string): WhereExpr<Row> {
return {kind: 'in', col, param} as WhereExpr<Row>;
}
describe.skipIf(!dockerAvailable)('postgres kv non-select drift', () => {
let raw: IPostgresClient;
let legacy: LegacyPostgresKvQueryExecutor;
let next: PostgresKvQueryExecutor;
async function freePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (typeof address === 'string' || address === null) {
reject(new Error('no port'));
return;
}
const port = address.port;
server.close(() => resolve(port));
});
});
}
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function dump(kv: string, tableName: string): Promise<string> {
const result = await raw.query<{row_key: string; partition_key: string; row_data: unknown; expires_at: Date}>(
`SELECT row_key, partition_key, row_data, expires_at FROM ${kv} WHERE table_name = $1 ORDER BY row_key COLLATE "C"`,
[tableName],
);
return JSON.stringify(
result.rows.map((r) => [
r.row_key,
r.partition_key,
r.row_data,
r.expires_at === null ? null : r.expires_at instanceof Date ? 'ts' : String(r.expires_at),
]),
);
}
async function reset(spec: KvTableSpec<Row>, rows: ReadonlyArray<Row>): Promise<void> {
for (const kv of [LEGACY_TABLE, NEXT_TABLE]) {
await raw.query(`DELETE FROM ${kv} WHERE table_name = $1`, [spec.name]);
}
const meta = {action: 'upsert', table: spec} as AnyMeta;
for (const row of rows) {
await legacy.executeQuery({cql: '__seed__', params: row as CassandraParams, kvMeta: meta});
await next.executeQuery({cql: '__seed__', params: row as CassandraParams, kvMeta: meta});
}
}
beforeAll(async () => {
const port = await freePort();
execFileSync(
'docker',
[
'run',
'-d',
'--name',
CONTAINER,
'-e',
'POSTGRES_USER=fluxer',
'-e',
'POSTGRES_PASSWORD=fluxer',
'-e',
'POSTGRES_DB=fluxer',
'-p',
`127.0.0.1:${port}:5432`,
'postgres:16-alpine',
'-c',
'fsync=off',
],
{stdio: 'ignore'},
);
let ready = false;
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
await sleep(500);
const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
stdio: 'ignore',
});
if (probe.status !== 0) continue;
try {
await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 8});
await getDefaultPostgresClient().query('SELECT 1');
ready = true;
} catch {
await shutdownPostgres().catch(() => {});
}
}
if (!ready) throw new Error('postgres never came up');
raw = getDefaultPostgresClient();
const legacyClient = new TableClient(raw, LEGACY_TABLE);
const nextClient = new TableClient(raw, NEXT_TABLE);
await ensurePostgresKvSchema(legacyClient);
await ensurePostgresKvSchema(nextClient);
legacy = new LegacyPostgresKvQueryExecutor(legacyClient);
next = new PostgresKvQueryExecutor(nextClient);
}, 900_000);
afterAll(async () => {
await shutdownPostgres().catch(() => {});
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
});
const seedRows: Array<Row> = [
{user_id: 1n, channel_id: 10n, note: 'a', blob_: null},
{user_id: 1n, channel_id: 11n, note: 'b', blob_: null},
{user_id: 2n, channel_id: 10n, note: 'c', blob_: null},
];
it('upsert / insert / ifNotExists behave identically', async () => {
await reset(Parent, seedRows);
const insMeta = {action: 'upsert', table: Parent, ifNotExists: true} as AnyMeta;
const fresh = {user_id: 3n, channel_id: 30n, note: 'z', blob_: null};
const l1 = await legacy.executeQuery({cql: '__i__', params: fresh as CassandraParams, kvMeta: insMeta});
const n1 = await next.executeQuery({cql: '__i__', params: fresh as CassandraParams, kvMeta: insMeta});
expect(n1).toEqual(l1);
const l2 = await legacy.executeQuery({cql: '__i__', params: fresh as CassandraParams, kvMeta: insMeta});
const n2 = await next.executeQuery({cql: '__i__', params: fresh as CassandraParams, kvMeta: insMeta});
expect(n2).toEqual(l2);
expect(await dump(NEXT_TABLE, Parent.name)).toBe(await dump(LEGACY_TABLE, Parent.name));
});
it('patch with and without ttl leaves identical state', async () => {
await reset(Parent, seedRows);
const patchMeta = {
action: 'patch',
table: Parent,
patchKeys: ['note'],
pkColumns: ['user_id', 'channel_id'],
} as unknown as AnyMeta;
const p = {user_id: 1n, channel_id: 10n, note: 'patched'} as CassandraParams;
await legacy.executeQuery({cql: '__p__', params: p, kvMeta: patchMeta});
await next.executeQuery({cql: '__p__', params: p, kvMeta: patchMeta});
const ttlMeta = {...patchMeta, ttlParamName: 'ttl_'} as unknown as AnyMeta;
const pt = {user_id: 2n, channel_id: 10n, note: 'ttl', ttl_: 600} as CassandraParams;
await legacy.executeQuery({cql: '__pt__', params: pt, kvMeta: ttlMeta});
await next.executeQuery({cql: '__pt__', params: pt, kvMeta: ttlMeta});
expect(await dump(NEXT_TABLE, Parent.name)).toBe(await dump(LEGACY_TABLE, Parent.name));
});
it('count returns the same value and the same javascript type', async () => {
await reset(Parent, seedRows);
await raw.query(
`UPDATE ${NEXT_TABLE} SET expires_at = now() - interval '1 hour' WHERE table_name = $1 AND row_key LIKE '%11%'`,
[Parent.name],
);
await raw.query(
`UPDATE ${LEGACY_TABLE} SET expires_at = now() - interval '1 hour' WHERE table_name = $1 AND row_key LIKE '%11%'`,
[Parent.name],
);
const meta = {action: 'count', table: Parent, where: [eq('user_id', 'user_id')]} as AnyMeta;
const l = await legacy.executeQuery<{count: unknown}>({cql: '__c__', params: {user_id: 1n}, kvMeta: meta});
const n = await next.executeQuery<{count: unknown}>({cql: '__c__', params: {user_id: 1n}, kvMeta: meta});
expect(n).toEqual(l);
expect(typeof (n[0] as {count: unknown}).count).toBe(typeof (l[0] as {count: unknown}).count);
});
it('count over an empty logical table matches', async () => {
const meta = {action: 'count', table: Single, where: []} as AnyMeta;
for (const kv of [LEGACY_TABLE, NEXT_TABLE]) {
await raw.query(`DELETE FROM ${kv} WHERE table_name = $1`, [Single.name]);
}
const l = await legacy.executeQuery({cql: '__ce__', params: {}, kvMeta: meta});
const n = await next.executeQuery({cql: '__ce__', params: {}, kvMeta: meta});
expect(n).toEqual(l);
});
it('count with a string bound to an IN parameter matches legacy', async () => {
await reset(Single, [
{token_: 'a', note: '1'},
{token_: 'b', note: '2'},
]);
const meta = {action: 'count', table: Single, where: [inClause('token_', 'tokens')]} as AnyMeta;
let legacyOut: unknown;
let nextOut: unknown;
try {
legacyOut = await legacy.executeQuery({cql: '__cs__', params: {tokens: 'ab'} as CassandraParams, kvMeta: meta});
} catch (error) {
legacyOut = `throw:${(error as Error).constructor.name}`;
}
try {
nextOut = await next.executeQuery({cql: '__cs__', params: {tokens: 'ab'} as CassandraParams, kvMeta: meta});
} catch (error) {
nextOut = `throw:${(error as Error).constructor.name}`;
}
expect(nextOut).toEqual(legacyOut);
});
it('rejects malformed page tokens the way legacy does', async () => {
await reset(Single, [
{token_: 'a', note: '1'},
{token_: 'b', note: '2'},
{token_: 'c', note: '3'},
]);
const meta = {action: 'select', table: Single, where: [], columns: Single.columns} as AnyMeta;
const tokens = ['eyJmb28iOjF9', 'e30', 'MTIz', 'InN0ciI', 'eyJvZmZzZXQiOi0xfQ', 'eyJvZmZzZXQiOjF9'];
const legacyOut: Array<string> = [];
const nextOut: Array<string> = [];
for (const token of tokens) {
for (const [exec, sink] of [
[legacy, legacyOut],
[next, nextOut],
] as const) {
try {
const page = await exec.executePagedQuery(
{cql: '__pg__', params: {}, kvMeta: meta},
{pageSize: 1, pageState: token},
);
sink.push(`${token} -> rows=${page.rows.length}`);
} catch (error) {
sink.push(`${token} -> throw:${(error as Error).message}`);
}
}
}
expect(nextOut).toEqual(legacyOut);
});
it('delete inside an atomic batch rolls back identically', async () => {
await reset(Parent, seedRows);
const delMeta = {action: 'delete', table: Parent, where: [eq('user_id', 'user_id')]} as AnyMeta;
for (const [exec, kv] of [
[legacy, LEGACY_TABLE],
[next, NEXT_TABLE],
] as const) {
await expect(
exec.executeBatch([
{query: '__d__', params: {user_id: 1n}, meta: delMeta as KvQueryMeta},
{query: '__boom__', params: {}, meta: {action: 'bogus'} as unknown as KvQueryMeta},
]),
).rejects.toThrow();
void kv;
}
expect(await dump(NEXT_TABLE, Parent.name)).toBe(await dump(LEGACY_TABLE, Parent.name));
});
it('delete of an expired-but-unpruned row matches legacy', async () => {
await reset(Parent, seedRows);
for (const kv of [LEGACY_TABLE, NEXT_TABLE]) {
await raw.query(`UPDATE ${kv} SET expires_at = now() - interval '1 hour' WHERE table_name = $1`, [Parent.name]);
}
const delMeta = {action: 'delete', table: Parent, where: [eq('user_id', 'user_id')]} as AnyMeta;
await legacy.executeQuery({cql: '__d__', params: {user_id: 1n}, kvMeta: delMeta});
await next.executeQuery({cql: '__d__', params: {user_id: 1n}, kvMeta: delMeta});
expect(await dump(NEXT_TABLE, Parent.name)).toBe(await dump(LEGACY_TABLE, Parent.name));
});
it('delete with no where clause matches legacy', async () => {
await reset(Parent, seedRows);
const delMeta = {action: 'delete', table: Parent, where: []} as AnyMeta;
await legacy.executeQuery({cql: '__da__', params: {}, kvMeta: delMeta});
await next.executeQuery({cql: '__da__', params: {}, kvMeta: delMeta});
expect(await dump(NEXT_TABLE, Parent.name)).toBe(await dump(LEGACY_TABLE, Parent.name));
});
it('delete does not touch a neighbouring logical table', async () => {
await reset(Parent, seedRows);
await reset(Single, [{token_: 'a', note: '1'}]);
const delMeta = {action: 'delete', table: Parent, where: []} as AnyMeta;
await next.executeQuery({cql: '__da2__', params: {}, kvMeta: delMeta});
const left = await raw.query<{count: string}>(`SELECT count(*) AS count FROM ${NEXT_TABLE} WHERE table_name = $1`, [
Single.name,
]);
expect(left.rows[0]!.count).toBe('1');
});
});
@@ -0,0 +1,361 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {execFileSync, spawnSync} from 'node:child_process';
import {createServer} from 'node:net';
import {
getDefaultPostgresClient,
type IPostgresClient,
initPostgres,
type PostgresQueryable,
shutdownPostgres,
} from '@pkgs/postgres/src/Client';
import cassandra from 'cassandra-driver';
import {afterAll, beforeAll, describe, expect, it} from 'vitest';
import {LegacyPostgresKvQueryExecutor} from './__testref__/LegacyPostgresKvQueryExecutor';
import type {CassandraParams, KvQueryMeta, KvTableSpec, WhereExpr} from './CassandraTypes';
import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';
type Row = Record<string, unknown>;
type AnyMeta = KvQueryMeta<Row>;
const LEGACY_TABLE = 'kvm_legacy';
const NEXT_TABLE = 'kvm_next';
const CONTAINER = `fluxer-kvm-${process.pid.toString(36)}-${Date.now().toString(36)}`;
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
class TableClient implements IPostgresClient {
constructor(
private readonly inner: IPostgresClient,
private readonly table: string,
) {}
async query<T extends Record<string, unknown>>(text: string, values: Array<unknown> = []) {
return (await this.inner.query(text, values)) as unknown as Awaited<ReturnType<IPostgresClient['query']>> & {
rows: Array<T>;
};
}
async connect(): Promise<void> {
await this.inner.connect();
}
async shutdown(): Promise<void> {}
isConnected(): boolean {
return this.inner.isConnected();
}
async transaction<T>(fn: (client: PostgresQueryable) => Promise<T>): Promise<T> {
return this.inner.transaction(fn);
}
kvTable(): string {
return this.table;
}
}
const Two: KvTableSpec<Row> = {
name: 'mx_two',
columns: ['k', 'c', 'v'],
primaryKey: ['k', 'c'],
partitionKey: ['k', 'c'],
};
const PartTwo: KvTableSpec<Row> = {
name: 'mx_part',
columns: ['k', 'c', 'v'],
primaryKey: ['k', 'c'],
partitionKey: ['k'],
};
const Three: KvTableSpec<Row> = {
name: 'mx_three',
columns: ['k', 'c', 'd', 'v'],
primaryKey: ['k', 'c', 'd'],
partitionKey: ['k', 'c', 'd'],
};
const KEY_VALUES: Array<[string, unknown]> = [
['null', null],
['num0', 0],
['num1', 1],
['num10', 10],
['big1', 1n],
['str1', '1'],
['strq', 'a"b\\c'],
['strctl', 'xy'],
['strastral', '\u{1f600}'],
['strspace', 'a b'],
['true', true],
['false', false],
['date', new Date(0)],
['buf', Buffer.from('a')],
['localdate', cassandra.types.LocalDate.fromString('2020-01-01')],
['localdate-str', '2020-01-01'],
['emptystr', ''],
];
function bigintJson(_key: string, value: unknown): unknown {
if (typeof value === 'bigint') return `bigint:${value.toString()}`;
if (Buffer.isBuffer(value)) return `buf:${value.toString('base64')}`;
return value;
}
describe.skipIf(!dockerAvailable)('postgres kv non-select matrix', () => {
let raw: IPostgresClient;
let legacy: LegacyPostgresKvQueryExecutor;
let next: PostgresKvQueryExecutor;
async function freePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (typeof address === 'string' || address === null) {
reject(new Error('no port'));
return;
}
const port = address.port;
server.close(() => resolve(port));
});
});
}
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function dump(kv: string, tableName: string): Promise<string> {
const result = await raw.query<{row_key: string}>(
`SELECT row_key FROM ${kv} WHERE table_name = $1 ORDER BY row_key COLLATE "C"`,
[tableName],
);
return JSON.stringify(result.rows.map((r) => r.row_key));
}
async function seed(spec: KvTableSpec<Row>, rows: ReadonlyArray<Row>): Promise<void> {
for (const kv of [LEGACY_TABLE, NEXT_TABLE]) {
await raw.query(`DELETE FROM ${kv} WHERE table_name = $1`, [spec.name]);
}
const meta = {action: 'upsert', table: spec} as AnyMeta;
for (const row of rows) {
await legacy.executeQuery({cql: '__s__', params: row as CassandraParams, kvMeta: meta});
await next.executeQuery({cql: '__s__', params: row as CassandraParams, kvMeta: meta});
}
}
beforeAll(async () => {
const port = await freePort();
execFileSync(
'docker',
[
'run',
'-d',
'--name',
CONTAINER,
'-e',
'POSTGRES_USER=fluxer',
'-e',
'POSTGRES_PASSWORD=fluxer',
'-e',
'POSTGRES_DB=fluxer',
'-p',
`127.0.0.1:${port}:5432`,
'postgres:16-alpine',
'-c',
'fsync=off',
],
{stdio: 'ignore'},
);
let ready = false;
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
await sleep(500);
if (
spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {stdio: 'ignore'})
.status !== 0
)
continue;
try {
await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 8});
await getDefaultPostgresClient().query('SELECT 1');
ready = true;
} catch {
await shutdownPostgres().catch(() => {});
}
}
if (!ready) throw new Error('postgres never came up');
raw = getDefaultPostgresClient();
await ensurePostgresKvSchema(new TableClient(raw, LEGACY_TABLE));
await ensurePostgresKvSchema(new TableClient(raw, NEXT_TABLE));
legacy = new LegacyPostgresKvQueryExecutor(new TableClient(raw, LEGACY_TABLE));
next = new PostgresKvQueryExecutor(new TableClient(raw, NEXT_TABLE));
}, 900_000);
afterAll(async () => {
await shutdownPostgres().catch(() => {});
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
});
it('count and delete agree with legacy for every key value type', async () => {
const rows = KEY_VALUES.map(([, value], i) => ({k: value, c: BigInt(i), v: i}));
const countMismatch: Array<string> = [];
const deleteMismatch: Array<string> = [];
const selectMismatch: Array<string> = [];
for (const [name, value] of KEY_VALUES) {
await seed(Two, rows);
const where = [{kind: 'eq', col: 'k', param: 'k'}] as Array<WhereExpr<Row>>;
const params = {k: value} as CassandraParams;
const sel = {action: 'select', table: Two, where, columns: Two.columns} as AnyMeta;
const l = await legacy.executeQuery({cql: '__sel__', params, kvMeta: sel});
const n = await next.executeQuery({cql: '__sel__', params, kvMeta: sel});
if (JSON.stringify(l, bigintJson) !== JSON.stringify(n, bigintJson)) {
selectMismatch.push(`${name}: legacy=${JSON.stringify(l, bigintJson)} next=${JSON.stringify(n, bigintJson)}`);
}
const cnt = {action: 'count', table: Two, where} as AnyMeta;
const lc = await legacy.executeQuery({cql: '__cnt__', params, kvMeta: cnt});
const nc = await next.executeQuery({cql: '__cnt__', params, kvMeta: cnt});
if (JSON.stringify(lc) !== JSON.stringify(nc)) {
countMismatch.push(`${name}: legacy=${JSON.stringify(lc)} next=${JSON.stringify(nc)}`);
}
const del = {action: 'delete', table: Two, where} as AnyMeta;
await legacy.executeQuery({cql: '__del__', params, kvMeta: del});
await next.executeQuery({cql: '__del__', params, kvMeta: del});
const ld = await dump(LEGACY_TABLE, Two.name);
const nd = await dump(NEXT_TABLE, Two.name);
if (ld !== nd) deleteMismatch.push(`${name}: legacy=${ld} next=${nd}`);
}
expect({selectMismatch, countMismatch, deleteMismatch}).toEqual({
selectMismatch: [],
countMismatch: [],
deleteMismatch: [],
});
}, 300_000);
it('explicit partition key tables agree for count and delete', async () => {
const rows = [
{k: 1n, c: 1n, v: 1},
{k: 1n, c: 2n, v: 2},
{k: 2n, c: 1n, v: 3},
];
await seed(PartTwo, rows);
const where = [{kind: 'eq', col: 'k', param: 'k'}] as Array<WhereExpr<Row>>;
const params = {k: 1n} as CassandraParams;
const lc = await legacy.executeQuery({
cql: '__pc__',
params,
kvMeta: {action: 'count', table: PartTwo, where} as AnyMeta,
});
const nc = await next.executeQuery({
cql: '__pc__',
params,
kvMeta: {action: 'count', table: PartTwo, where} as AnyMeta,
});
expect(nc).toEqual(lc);
await legacy.executeQuery({cql: '__pd__', params, kvMeta: {action: 'delete', table: PartTwo, where} as AnyMeta});
await next.executeQuery({cql: '__pd__', params, kvMeta: {action: 'delete', table: PartTwo, where} as AnyMeta});
expect(await dump(NEXT_TABLE, PartTwo.name)).toBe(await dump(LEGACY_TABLE, PartTwo.name));
});
it('multi column IN products agree, including above the combination cap', async () => {
const rows: Array<Row> = [];
for (let a = 0; a < 40; a += 1) {
for (let b = 0; b < 40; b += 1) rows.push({k: BigInt(a), c: BigInt(b), v: a * b});
}
await seed(Two, rows);
const ks = Array.from({length: 40}, (_, i) => BigInt(i));
const cs = Array.from({length: 40}, (_, i) => BigInt(i));
const where = [
{kind: 'in', col: 'k', param: 'ks'},
{kind: 'in', col: 'c', param: 'cs'},
] as Array<WhereExpr<Row>>;
for (const [label, params] of [
['small', {ks: ks.slice(0, 3), cs: cs.slice(0, 3)}],
['cap', {ks, cs}],
['dupes', {ks: [1n, 1n, 2n], cs: [1n, 1n]}],
['empty', {ks: [], cs}],
] as Array<[string, CassandraParams]>) {
const lc = await legacy.executeQuery({
cql: `__mc_${label}__`,
params,
kvMeta: {action: 'count', table: Two, where} as AnyMeta,
});
const nc = await next.executeQuery({
cql: `__mc_${label}__`,
params,
kvMeta: {action: 'count', table: Two, where} as AnyMeta,
});
expect([label, nc]).toEqual([label, lc]);
const ls = await legacy.executeQuery({
cql: `__ms_${label}__`,
params,
kvMeta: {action: 'select', table: Two, where, columns: Two.columns} as AnyMeta,
});
const ns = await next.executeQuery({
cql: `__ms_${label}__`,
params,
kvMeta: {action: 'select', table: Two, where, columns: Two.columns} as AnyMeta,
});
expect([label, JSON.stringify(ns, bigintJson)]).toEqual([label, JSON.stringify(ls, bigintJson)]);
}
await legacy.executeQuery({
cql: '__md__',
params: {ks, cs} as CassandraParams,
kvMeta: {action: 'delete', table: Two, where} as AnyMeta,
});
await next.executeQuery({
cql: '__md__',
params: {ks, cs} as CassandraParams,
kvMeta: {action: 'delete', table: Two, where} as AnyMeta,
});
expect(await dump(NEXT_TABLE, Two.name)).toBe(await dump(LEGACY_TABLE, Two.name));
}, 300_000);
it('count with a limit or an order by matches legacy', async () => {
await seed(Two, [
{k: 1n, c: 1n, v: 1},
{k: 1n, c: 2n, v: 2},
{k: 1n, c: 3n, v: 3},
]);
const where = [{kind: 'eq', col: 'k', param: 'k'}] as Array<WhereExpr<Row>>;
for (const extra of [{limit: 2}, {orderBy: {col: 'c', direction: 'DESC'}}, {}]) {
const meta = {action: 'count', table: Two, where, ...extra} as AnyMeta;
const l = await legacy.executeQuery({cql: `__cl__${JSON.stringify(extra)}`, params: {k: 1n}, kvMeta: meta});
const n = await next.executeQuery({cql: `__cl__${JSON.stringify(extra)}`, params: {k: 1n}, kvMeta: meta});
expect([JSON.stringify(extra), n]).toEqual([JSON.stringify(extra), l]);
}
});
it('count and delete with a gap in the primary key match legacy', async () => {
const rows: Array<Row> = [];
for (const c of [1n, 2n]) {
for (const d of [1n, 2n]) rows.push({k: 1n, c, d, v: 1});
}
rows.push({k: 2n, c: 1n, d: 1n, v: 9});
await seed(Three, rows);
const where = [
{kind: 'eq', col: 'k', param: 'k'},
{kind: 'eq', col: 'd', param: 'd'},
] as Array<WhereExpr<Row>>;
const params = {k: 1n, d: 1n} as CassandraParams;
const lc = await legacy.executeQuery({
cql: '__gc__',
params,
kvMeta: {action: 'count', table: Three, where} as AnyMeta,
});
const nc = await next.executeQuery({
cql: '__gc__',
params,
kvMeta: {action: 'count', table: Three, where} as AnyMeta,
});
expect(nc).toEqual(lc);
await legacy.executeQuery({cql: '__gd__', params, kvMeta: {action: 'delete', table: Three, where} as AnyMeta});
await next.executeQuery({cql: '__gd__', params, kvMeta: {action: 'delete', table: Three, where} as AnyMeta});
expect(await dump(NEXT_TABLE, Three.name)).toBe(await dump(LEGACY_TABLE, Three.name));
});
it('delete with a non key where column matches legacy', async () => {
await seed(Two, [
{k: 1n, c: 1n, v: 5},
{k: 1n, c: 2n, v: 6},
{k: 2n, c: 1n, v: 5},
]);
const where = [
{kind: 'eq', col: 'k', param: 'k'},
{kind: 'eq', col: 'v', param: 'v'},
] as Array<WhereExpr<Row>>;
const params = {k: 1n, v: 5} as CassandraParams;
await legacy.executeQuery({cql: '__nd__', params, kvMeta: {action: 'delete', table: Two, where} as AnyMeta});
await next.executeQuery({cql: '__nd__', params, kvMeta: {action: 'delete', table: Two, where} as AnyMeta});
expect(await dump(NEXT_TABLE, Two.name)).toBe(await dump(LEGACY_TABLE, Two.name));
});
});
@@ -0,0 +1,544 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {execFileSync, spawnSync} from 'node:child_process';
import {createServer} from 'node:net';
import {
getDefaultPostgresClient,
type IPostgresClient,
initPostgres,
type PostgresQueryable,
shutdownPostgres,
} from '@pkgs/postgres/src/Client';
import cassandra from 'cassandra-driver';
import {afterAll, beforeAll, describe, expect, it} from 'vitest';
import {LegacyPostgresKvQueryExecutor} from './__testref__/LegacyPostgresKvQueryExecutor';
import {defineTable} from './CassandraTableDsl';
import type {CassandraParam, CassandraParams, KvQueryMeta, KvTableSpec, WhereExpr} from './CassandraTypes';
import {buildCandidatePlan, ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';
type Row = Record<string, unknown>;
type AnyMeta = KvQueryMeta<Row>;
type AnyWhere = WhereExpr<Row>;
const KV = 'kv_order_attack';
const POSTGRES_IMAGE = 'postgres:16-alpine';
const CONTAINER = `fluxer-kvorder-${process.pid.toString(36)}-${Date.now().toString(36)}`;
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
const BigChild = defineTable<{owner: unknown; seq: unknown; v: unknown}, 'owner' | 'seq'>({
name: 'attack_bigint_child',
columns: ['owner', 'seq', 'v'],
primaryKey: ['owner', 'seq'],
});
const NumChild = defineTable<{owner: unknown; seq: unknown; v: unknown}, 'owner' | 'seq'>({
name: 'attack_number_child',
columns: ['owner', 'seq', 'v'],
primaryKey: ['owner', 'seq'],
});
const DateChild = defineTable<{owner: unknown; at: unknown; id: unknown; v: unknown}, 'owner' | 'at' | 'id'>({
name: 'attack_date_child',
columns: ['owner', 'at', 'id', 'v'],
primaryKey: ['owner', 'at', 'id'],
});
const StrChild = defineTable<{owner: unknown; seq: unknown; v: unknown}, 'owner' | 'seq'>({
name: 'attack_string_child',
columns: ['owner', 'seq', 'v'],
primaryKey: ['owner', 'seq'],
});
const ScanBig = defineTable<{id: unknown; v: unknown}, 'id'>({
name: 'attack_scan_bigint',
columns: ['id', 'v'],
primaryKey: ['id'],
});
const Messages = defineTable<
{channel_id: unknown; bucket: unknown; message_id: unknown; content: unknown},
'channel_id' | 'bucket' | 'message_id',
'channel_id' | 'bucket'
>({
name: 'attack_messages',
columns: ['channel_id', 'bucket', 'message_id', 'content'],
primaryKey: ['channel_id', 'bucket', 'message_id'],
partitionKey: ['channel_id', 'bucket'],
});
const LocalDateChild = defineTable<{owner: unknown; seq: unknown; v: unknown}, 'owner' | 'seq'>({
name: 'attack_localdate_child',
columns: ['owner', 'seq', 'v'],
primaryKey: ['owner', 'seq'],
});
function eq(col: string, param: string): AnyWhere {
return {kind: 'eq', col, param} as AnyWhere;
}
function inClause(col: string, param: string): AnyWhere {
return {kind: 'in', col, param} as AnyWhere;
}
function cmp(kind: 'lt' | 'lte' | 'gt' | 'gte', col: string, param: string): AnyWhere {
return {kind, col, param} as AnyWhere;
}
function spec(table: unknown): KvTableSpec<Row> {
return table as unknown as KvTableSpec<Row>;
}
function selectMeta(table: unknown, where: Array<AnyWhere>, extra: Partial<AnyMeta> = {}): AnyMeta {
const s = spec(table);
return {action: 'select', table: s, where, columns: s.columns, ...extra} as AnyMeta;
}
function canonical(value: unknown): unknown {
if (value === undefined) return {__c: 'undefined'};
if (value === null) return null;
if (typeof value === 'bigint') return {__c: 'bigint', v: value.toString()};
if (typeof value === 'number') {
if (Number.isNaN(value)) return {__c: 'nan'};
if (!Number.isFinite(value)) return {__c: 'inf', v: value > 0 ? 1 : -1};
return value;
}
if (value instanceof Date) return {__c: 'date', v: value.toISOString()};
if (Buffer.isBuffer(value)) return {__c: 'buffer', v: value.toString('base64')};
if (typeof value === 'object' && value.constructor?.name === 'LocalDate') return {__c: 'localdate', v: String(value)};
if (Array.isArray(value)) return value.map(canonical);
if (typeof value === 'object') {
const source = value as Record<string, unknown>;
return {
__c: 'obj',
v: Object.keys(source)
.sort()
.map((k) => [k, canonical(source[k])]),
};
}
return value;
}
function fp(rows: ReadonlyArray<unknown>): string {
return JSON.stringify(rows.map(canonical));
}
function multiset(rows: ReadonlyArray<unknown>): string {
return JSON.stringify(rows.map((r) => JSON.stringify(canonical(r))).sort());
}
const suite = dockerAvailable ? describe : describe.skip;
suite('PostgresKvQueryExecutor ordering and limit attack', () => {
let raw: IPostgresClient;
let legacy: LegacyPostgresKvQueryExecutor;
let next: PostgresKvQueryExecutor;
async function freePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (typeof address === 'string' || address === null) {
reject(new Error('no port'));
return;
}
const port = address.port;
server.close(() => resolve(port));
});
});
}
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
function client(): IPostgresClient {
const inner = raw;
return {
query: (text: string, values?: Array<unknown>) => inner.query(text, values),
connect: () => inner.connect(),
shutdown: async () => {},
isConnected: () => inner.isConnected(),
transaction: <T>(fn: (db: PostgresQueryable) => Promise<T>) => inner.transaction(fn),
kvTable: () => KV,
} as unknown as IPostgresClient;
}
async function seed(table: unknown, rows: ReadonlyArray<Row>): Promise<void> {
const s = spec(table);
await raw.query(`DELETE FROM ${KV} WHERE table_name = $1`, [s.name]);
const meta = {action: 'upsert', table: s} as AnyMeta;
for (const row of rows) {
await legacy.executeQuery({cql: `__seed__${s.name}`, params: row as CassandraParams, kvMeta: meta});
}
}
async function collectPages(
executor: LegacyPostgresKvQueryExecutor | PostgresKvQueryExecutor,
meta: AnyMeta,
params: CassandraParams,
pageSize: number,
): Promise<Array<Array<unknown>>> {
const pages: Array<Array<unknown>> = [];
let pageState: string | null = null;
for (let guard = 0; guard < 200; guard += 1) {
const page: {rows: Array<unknown>; pageState: string | null} = await executor.executePagedQuery(
{cql: '__page__', params, kvMeta: meta},
{pageSize, pageState},
);
pages.push(page.rows);
pageState = page.pageState;
if (pageState === null) break;
}
return pages;
}
beforeAll(async () => {
const port = await freePort();
execFileSync(
'docker',
[
'run',
'-d',
'--name',
CONTAINER,
'-e',
'POSTGRES_USER=fluxer',
'-e',
'POSTGRES_PASSWORD=fluxer',
'-e',
'POSTGRES_DB=fluxer',
'-p',
`127.0.0.1:${port}:5432`,
POSTGRES_IMAGE,
'-c',
'fsync=off',
'-c',
'synchronous_commit=off',
'-c',
'full_page_writes=off',
],
{stdio: 'ignore'},
);
let ready = false;
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
await sleep(500);
const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
stdio: 'ignore',
});
if (probe.status !== 0) continue;
try {
await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 6});
await getDefaultPostgresClient().query('SELECT 1');
ready = true;
} catch {
await shutdownPostgres().catch(() => {});
}
}
if (!ready) throw new Error('postgres not ready');
raw = getDefaultPostgresClient();
const c = client();
await ensurePostgresKvSchema(c);
legacy = new LegacyPostgresKvQueryExecutor(c);
next = new PostgresKvQueryExecutor(c);
}, 900_000);
afterAll(async () => {
await shutdownPostgres().catch(() => {});
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
});
it('A1 keyset paging over a bigint clustering column keeps legacy page boundaries', async () => {
const seqs = [1n, 2n, 9n, 10n, 11n, 100n, 101n, 999n, 1000n];
await seed(
BigChild,
seqs.map((seq) => ({owner: 'o', seq, v: seq.toString()})),
);
const meta = selectMeta(BigChild, [eq('owner', 'owner')]);
const params: CassandraParams = {owner: 'o'};
const plan = buildCandidatePlan(meta, params);
expect(plan.exact).toBe(true);
expect(plan.candidates.kind).toBe('range');
const left = await collectPages(legacy, meta, params, 3);
const right = await collectPages(next, meta, params, 3);
expect(multiset(left.flat())).toBe(multiset(right.flat()));
expect(right.map((p) => p.map((r) => String((r as Row).seq)))).toStrictEqual(
left.map((p) => p.map((r) => String((r as Row).seq))),
);
}, 120_000);
it('A2 keyset paging over a plain number clustering column keeps legacy page boundaries', async () => {
const seqs = [1, 2, 9, 10, 11, 100, 1e21, -1];
await seed(
NumChild,
seqs.map((seq) => ({owner: 'o', seq, v: String(seq)})),
);
const meta = selectMeta(NumChild, [eq('owner', 'owner')]);
const params: CassandraParams = {owner: 'o'};
const left = await collectPages(legacy, meta, params, 3);
const right = await collectPages(next, meta, params, 3);
expect(multiset(left.flat())).toBe(multiset(right.flat()));
expect(fp(right.flat())).toBe(fp(left.flat()));
}, 120_000);
it('A3 keyset paging over a whole-table bigint scan keeps legacy page boundaries', async () => {
const ids = [1n, 2n, 9n, 10n, 11n, 100n, 1000n];
await seed(
ScanBig,
ids.map((id) => ({id, v: id.toString()})),
);
const meta = selectMeta(ScanBig, []);
const plan = buildCandidatePlan(meta, {});
expect(plan.exact).toBe(true);
expect(plan.candidates.kind).toBe('scan');
const left = await collectPages(legacy, meta, {}, 3);
const right = await collectPages(next, meta, {}, 3);
expect(multiset(left.flat())).toBe(multiset(right.flat()));
expect(fp(right.flat())).toBe(fp(left.flat()));
}, 120_000);
it('A4 keyset paging over a Date clustering column keeps legacy page boundaries', async () => {
const dates = [
new Date('1969-12-31T23:59:59.999Z'),
new Date(0),
new Date('2020-01-02T03:04:05.006Z'),
new Date('2024-06-07T08:09:10.011Z'),
new Date('2999-01-01T00:00:00.000Z'),
new Date(-62167219200000),
new Date(8640000000000000),
];
await seed(
DateChild,
dates.map((at, index) => ({owner: 'o', at, id: BigInt(index), v: at.toISOString()})),
);
const meta = selectMeta(DateChild, [eq('owner', 'owner')]);
const params: CassandraParams = {owner: 'o'};
const left = await collectPages(legacy, meta, params, 2);
const right = await collectPages(next, meta, params, 2);
expect(multiset(left.flat())).toBe(multiset(right.flat()));
expect(fp(right.flat())).toBe(fp(left.flat()));
}, 120_000);
it('A5 keyset paging over a string clustering column keeps legacy page boundaries', async () => {
const seqs = ['a', 'b', 'A', 'ab', 'a b', '\u{1f600}', '�', 'a"b', 'a\\b', ''];
await seed(
StrChild,
seqs.map((seq, index) => ({owner: 'o', seq, v: index})),
);
const meta = selectMeta(StrChild, [eq('owner', 'owner')]);
const params: CassandraParams = {owner: 'o'};
const left = await collectPages(legacy, meta, params, 3);
const right = await collectPages(next, meta, params, 3);
expect(multiset(left.flat())).toBe(multiset(right.flat()));
expect(fp(right.flat())).toBe(fp(left.flat()));
}, 120_000);
it('A6 a keyset page is never empty while it reports more pages', async () => {
const seqs = [1n, 2n, 9n, 10n, 11n, 100n, 101n];
await seed(
BigChild,
seqs.map((seq) => ({owner: 'o', seq, v: seq.toString()})),
);
const meta = selectMeta(BigChild, [eq('owner', 'owner')]);
const pages = await collectPages(next, meta, {owner: 'o'}, 2);
for (let index = 0; index < pages.length - 1; index += 1) {
expect(pages[index]!.length).toBeGreaterThan(0);
}
}, 120_000);
it('B1 select with a LIMIT returns the same rows as the legacy executor', async () => {
const seqs = [1n, 2n, 9n, 10n, 11n, 100n, 101n, 999n, 1000n];
const rows: Array<Row> = [];
for (const owner of ['o', 'p']) for (const seq of seqs) rows.push({owner, seq, v: seq.toString()});
await seed(BigChild, rows);
const mismatches: Array<string> = [];
for (const limit of [1, 2, 3, 5, 100]) {
for (const direction of [undefined, 'ASC', 'DESC'] as const) {
const extra: Partial<AnyMeta> = {limit};
if (direction) extra.orderBy = {col: 'seq', direction} as AnyMeta['orderBy'];
const meta = selectMeta(BigChild, [eq('owner', 'owner')], extra);
const params: CassandraParams = {owner: 'o'};
const l = await legacy.executeQuery({cql: '__l__', params, kvMeta: meta});
const r = await next.executeQuery({cql: '__l__', params, kvMeta: meta});
if (fp(l) !== fp(r)) mismatches.push(`limit=${limit} order=${direction}\n legacy ${fp(l)}\n next ${fp(r)}`);
}
}
expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
}, 120_000);
it('B2 select with a LIMIT and an IN over the leading key returns the same rows', async () => {
const rows: Array<Row> = [];
for (const owner of ['o', 'p', 'q']) {
for (const seq of [1n, 2n, 9n, 10n, 11n, 100n]) rows.push({owner, seq, v: `${owner}${seq}`});
}
await seed(BigChild, rows);
const mismatches: Array<string> = [];
for (const limit of [1, 2, 4, 7, 100]) {
for (const direction of [undefined, 'ASC', 'DESC'] as const) {
const extra: Partial<AnyMeta> = {limit};
if (direction) extra.orderBy = {col: 'seq', direction} as AnyMeta['orderBy'];
const meta = selectMeta(BigChild, [inClause('owner', 'owners')], extra);
const params: CassandraParams = {owners: ['o', 'q', 'o'] as unknown as CassandraParam};
const l = await legacy.executeQuery({cql: '__l2__', params, kvMeta: meta});
const r = await next.executeQuery({cql: '__l2__', params, kvMeta: meta});
if (fp(l) !== fp(r)) mismatches.push(`limit=${limit} order=${direction}\n legacy ${fp(l)}\n next ${fp(r)}`);
}
}
expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
}, 120_000);
it('B3 select with a LIMIT and a range clause on the clustering column returns the same rows', async () => {
const rows: Array<Row> = [];
for (const owner of ['o', 'p']) {
for (const seq of [1n, 2n, 9n, 10n, 11n, 100n, 101n, 1000n]) rows.push({owner, seq, v: `${owner}${seq}`});
}
await seed(BigChild, rows);
const mismatches: Array<string> = [];
for (const kind of ['lt', 'lte', 'gt', 'gte'] as const) {
for (const bound of [1n, 10n, 11n, 100n]) {
for (const limit of [1, 2, 3, 100]) {
for (const direction of ['ASC', 'DESC'] as const) {
const meta = selectMeta(BigChild, [eq('owner', 'owner'), cmp(kind, 'seq', 'bound')], {
limit,
orderBy: {col: 'seq', direction} as AnyMeta['orderBy'],
});
const params: CassandraParams = {owner: 'o', bound};
const l = await legacy.executeQuery({cql: '__l3__', params, kvMeta: meta});
const r = await next.executeQuery({cql: '__l3__', params, kvMeta: meta});
if (fp(l) !== fp(r)) {
mismatches.push(`${kind} ${bound} limit=${limit} ${direction}\n legacy ${fp(l)}\n next ${fp(r)}`);
}
}
}
}
}
expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
}, 240_000);
it('B4 message history shaped select returns the same page as the legacy executor', async () => {
const rows: Array<Row> = [];
for (const channel of [10n, 11n]) {
for (const bucket of [0, 1, 2, 10, 11]) {
for (const messageId of [1n, 2n, 9n, 10n, 11n, 100n, 101n, 999n, 1000n, 9999999999999999999n]) {
rows.push({channel_id: channel, bucket, message_id: messageId, content: `${channel}/${bucket}/${messageId}`});
}
}
}
await seed(Messages, rows);
const mismatches: Array<string> = [];
for (const bucket of [0, 1, 10]) {
for (const cursor of [1n, 10n, 11n, 100n, 1000n, 9999999999999999999n]) {
for (const kind of ['lt', 'gt'] as const) {
for (const direction of ['ASC', 'DESC'] as const) {
for (const limit of [1, 3, 50]) {
const meta = selectMeta(
Messages,
[eq('channel_id', 'channel_id'), eq('bucket', 'bucket'), cmp(kind, 'message_id', 'cursor')],
{limit, orderBy: {col: 'message_id', direction} as AnyMeta['orderBy']},
);
const params: CassandraParams = {channel_id: 10n, bucket, cursor};
const l = await legacy.executeQuery({cql: '__b4__', params, kvMeta: meta});
const r = await next.executeQuery({cql: '__b4__', params, kvMeta: meta});
if (fp(l) !== fp(r)) {
mismatches.push(`bucket=${bucket} ${kind} ${cursor} ${direction} limit=${limit}`);
}
}
}
}
}
}
expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
}, 300_000);
it('B5 select with a LIMIT under a demoted over-cap cartesian plan returns the same rows', async () => {
const rows: Array<Row> = [];
for (const owner of ['o', 'p', 'q']) {
for (const seq of [1n, 2n, 9n, 10n, 11n, 100n]) rows.push({owner, seq, v: `${owner}${seq}`});
}
await seed(BigChild, rows);
const owners = Array.from({length: 40}, (_unused, index) => `x${index}`);
owners.push('o', 'q');
const seqs = Array.from({length: 40}, (_unused, index) => BigInt(index));
seqs.push(100n);
const mismatches: Array<string> = [];
for (const limit of [1, 3, 100]) {
for (const direction of ['ASC', 'DESC'] as const) {
const meta = selectMeta(BigChild, [inClause('owner', 'owners'), inClause('seq', 'seqs')], {
limit,
orderBy: {col: 'seq', direction} as AnyMeta['orderBy'],
});
const params: CassandraParams = {
owners: owners as unknown as CassandraParam,
seqs: seqs as unknown as CassandraParam,
};
const l = await legacy.executeQuery({cql: '__b5__', params, kvMeta: meta});
const r = await next.executeQuery({cql: '__b5__', params, kvMeta: meta});
if (fp(l) !== fp(r)) mismatches.push(`limit=${limit} ${direction}\n legacy ${fp(l)}\n next ${fp(r)}`);
}
}
expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
}, 180_000);
it('B6 select ordered by a column outside the projection returns the same rows', async () => {
const rows: Array<Row> = [];
for (const seq of [1n, 2n, 9n, 10n, 11n, 100n]) rows.push({owner: 'o', seq, v: `v${seq}`});
await seed(BigChild, rows);
const mismatches: Array<string> = [];
for (const limit of [1, 3, 100]) {
for (const direction of ['ASC', 'DESC'] as const) {
const meta = selectMeta(BigChild, [eq('owner', 'owner')], {
columns: ['v'],
limit,
orderBy: {col: 'seq', direction} as AnyMeta['orderBy'],
});
const params: CassandraParams = {owner: 'o'};
const l = await legacy.executeQuery({cql: '__b6__', params, kvMeta: meta});
const r = await next.executeQuery({cql: '__b6__', params, kvMeta: meta});
if (fp(l) !== fp(r)) mismatches.push(`limit=${limit} ${direction}\n legacy ${fp(l)}\n next ${fp(r)}`);
}
}
expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
}, 180_000);
it('B7 keyset page sizes match legacy page sizes exactly', async () => {
const rows: Array<Row> = [];
for (const seq of [1n, 2n, 9n, 10n, 11n, 100n, 101n]) rows.push({owner: 'o', seq, v: `v${seq}`});
await seed(BigChild, rows);
const meta = selectMeta(BigChild, [eq('owner', 'owner')]);
const mismatches: Array<string> = [];
for (const pageSize of [1, 2, 3, 6, 7, 8, 100]) {
const left = await collectPages(legacy, meta, {owner: 'o'}, pageSize);
const right = await collectPages(next, meta, {owner: 'o'}, pageSize);
const l = left.map((p) => p.length).join(',');
const r = right.map((p) => p.length).join(',');
if (l !== r) mismatches.push(`pageSize=${pageSize} legacy=${l} next=${r}`);
if (multiset(left.flat()) !== multiset(right.flat())) mismatches.push(`pageSize=${pageSize} rowset`);
}
expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
}, 180_000);
it('C1 a LocalDate key value is still found by an equal string parameter', async () => {
await seed(LocalDateChild, [
{owner: cassandra.types.LocalDate.fromString('2020-01-01'), seq: 1n, v: 'localdate'},
{owner: '2020-01-01', seq: 2n, v: 'string'},
]);
const meta = selectMeta(LocalDateChild, [eq('owner', 'owner')], {limit: 10});
const params: CassandraParams = {owner: '2020-01-01'};
const l = await legacy.executeQuery({cql: '__c1__', params, kvMeta: meta});
const r = await next.executeQuery({cql: '__c1__', params, kvMeta: meta});
expect(fp(r), `legacy ${fp(l)} next ${fp(r)}`).toBe(fp(l));
}, 120_000);
it('C2 a string key value is still found by an equal LocalDate parameter', async () => {
await seed(LocalDateChild, [
{owner: cassandra.types.LocalDate.fromString('2020-01-01'), seq: 1n, v: 'localdate'},
{owner: '2020-01-01', seq: 2n, v: 'string'},
]);
const meta = selectMeta(LocalDateChild, [eq('owner', 'owner')], {limit: 10});
const params: CassandraParams = {owner: cassandra.types.LocalDate.fromString('2020-01-01')};
const l = await legacy.executeQuery({cql: '__c2__', params, kvMeta: meta});
const r = await next.executeQuery({cql: '__c2__', params, kvMeta: meta});
expect(fp(r), `legacy ${fp(l)} next ${fp(r)}`).toBe(fp(l));
}, 120_000);
});
@@ -0,0 +1,562 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {execFileSync, spawnSync} from 'node:child_process';
import {createServer} from 'node:net';
import {
getDefaultPostgresClient,
type IPostgresClient,
initPostgres,
type PostgresQueryable,
shutdownPostgres,
} from '@pkgs/postgres/src/Client';
import {afterAll, beforeAll, describe, expect, it} from 'vitest';
import {LegacyPostgresKvQueryExecutor} from './__testref__/LegacyPostgresKvQueryExecutor';
import type {CassandraParams, KvQueryMeta, KvTableSpec, WhereExpr} from './CassandraTypes';
import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';
type Row = Record<string, unknown>;
type AnyMeta = KvQueryMeta<Row>;
type AnyExec = LegacyPostgresKvQueryExecutor | PostgresKvQueryExecutor;
const KV = 'kv_pagead';
const ICU_KV = 'kv_pagead_icu';
const ICU_COLLATION = 'kvpagead_icu';
const POSTGRES_IMAGE = 'postgres:16-alpine';
const CONTAINER = `fluxer-kvpage-${process.pid.toString(36)}-${Date.now().toString(36)}`;
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
const PagedTable: KvTableSpec<Row> = {
name: 'pagead_items',
columns: ['owner_id', 'item_id', 'payload'],
primaryKey: ['owner_id', 'item_id'],
partitionKey: ['owner_id', 'item_id'],
};
const FlatTable: KvTableSpec<Row> = {
name: 'pagead_flat',
columns: ['k', 'v'],
primaryKey: ['k'],
partitionKey: ['k'],
};
class PlainClient implements IPostgresClient {
constructor(
private readonly inner: IPostgresClient,
private readonly table: string,
) {}
async query<T extends Record<string, unknown>>(text: string, values: Array<unknown> = []) {
return (await this.inner.query(text, values)) as unknown as Awaited<ReturnType<IPostgresClient['query']>> & {
rows: Array<T>;
};
}
async connect(): Promise<void> {
await this.inner.connect();
}
async shutdown(): Promise<void> {}
isConnected(): boolean {
return this.inner.isConnected();
}
async transaction<T>(fn: (client: PostgresQueryable) => Promise<T>): Promise<T> {
return this.inner.transaction(fn);
}
kvTable(): string {
return this.table;
}
}
function selectMeta(spec: KvTableSpec<Row>, where: Array<WhereExpr<Row>> = [], extra: Partial<AnyMeta> = {}): AnyMeta {
return {action: 'select', table: spec, where, columns: spec.columns, ...extra} as AnyMeta;
}
const upsertMeta = (spec: KvTableSpec<Row>) => ({action: 'upsert', table: spec}) as AnyMeta;
const deleteMeta = (spec: KvTableSpec<Row>, where: Array<WhereExpr<Row>>) =>
({action: 'delete', table: spec, where}) as AnyMeta;
function token(value: unknown): string {
return Buffer.from(JSON.stringify(value)).toString('base64url');
}
describe.skipIf(!dockerAvailable)('postgres kv paging adversarial', () => {
let raw: IPostgresClient;
let legacy: LegacyPostgresKvQueryExecutor;
let next: PostgresKvQueryExecutor;
let icuNext: PostgresKvQueryExecutor;
let icuLegacy: LegacyPostgresKvQueryExecutor;
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function freePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (typeof address === 'string' || address === null) {
reject(new Error('no port'));
return;
}
const port = address.port;
server.close(() => resolve(port));
});
});
}
async function upsert(exec: AnyExec, spec: KvTableSpec<Row>, row: Row) {
await exec.executeQuery({cql: `__seed__${spec.name}`, params: row as CassandraParams, kvMeta: upsertMeta(spec)});
}
async function wipe(kv: string) {
await raw.query(`DELETE FROM ${kv}`);
}
async function pageAll(
exec: AnyExec,
meta: AnyMeta,
params: CassandraParams,
pageSize: number,
hook?: (pageIndex: number) => Promise<void>,
): Promise<{pages: Array<Array<Row>>; error: string | null}> {
const pages: Array<Array<Row>> = [];
let pageState: string | null = null;
try {
for (let guard = 0; guard < 400; guard += 1) {
const page: {rows: Array<Row>; pageState: string | null} = await exec.executePagedQuery<Row>(
{cql: `__page__${meta.table.name}`, params, kvMeta: meta},
{pageSize, pageState},
);
pages.push(page.rows);
pageState = page.pageState;
if (hook) await hook(guard);
if (pageState === null) break;
if (guard === 399) return {pages, error: 'NON_TERMINATING'};
}
} catch (error) {
return {pages, error: (error as Error).message};
}
return {pages, error: null};
}
beforeAll(async () => {
const port = await freePort();
execFileSync(
'docker',
[
'run',
'-d',
'--name',
CONTAINER,
'-e',
'POSTGRES_USER=fluxer',
'-e',
'POSTGRES_PASSWORD=fluxer',
'-e',
'POSTGRES_DB=fluxer',
'-p',
`127.0.0.1:${port}:5432`,
POSTGRES_IMAGE,
'-c',
'fsync=off',
'-c',
'synchronous_commit=off',
],
{stdio: 'ignore'},
);
let ready = false;
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
await sleep(500);
const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
stdio: 'ignore',
});
if (probe.status !== 0) continue;
try {
await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 8});
await getDefaultPostgresClient().query('SELECT 1');
ready = true;
} catch {
await shutdownPostgres().catch(() => {});
}
}
if (!ready) throw new Error('postgres not ready');
raw = getDefaultPostgresClient();
const kvClient = new PlainClient(raw, KV);
const icuClient = new PlainClient(raw, ICU_KV);
await ensurePostgresKvSchema(kvClient);
await raw.query(`CREATE COLLATION ${ICU_COLLATION} (provider = icu, locale = 'en-US')`);
await ensurePostgresKvSchema(icuClient);
for (const column of ['row_key', 'partition_key']) {
await raw.query(`ALTER TABLE ${ICU_KV} ALTER COLUMN ${column} TYPE text COLLATE ${ICU_COLLATION}`);
}
legacy = new LegacyPostgresKvQueryExecutor(kvClient);
next = new PostgresKvQueryExecutor(kvClient);
icuLegacy = new LegacyPostgresKvQueryExecutor(icuClient);
icuNext = new PostgresKvQueryExecutor(icuClient);
}, 900_000);
afterAll(async () => {
await shutdownPostgres().catch(() => {});
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
});
it('pages a bare scan without skipping or repeating rows', async () => {
await wipe(KV);
for (let i = 0; i < 23; i += 1) await upsert(next, FlatTable, {k: BigInt(i), v: `v${i}`});
for (const pageSize of [1, 2, 3, 5, 7, 23, 24, 100]) {
const result = await pageAll(next, selectMeta(FlatTable), {}, pageSize);
expect(result.error, `pageSize=${pageSize}`).toBeNull();
const keys = result.pages.flat().map((row) => String(row.k));
expect(new Set(keys).size, `dupes at pageSize=${pageSize}: ${keys.join(',')}`).toBe(23);
expect(keys.length, `count at pageSize=${pageSize}`).toBe(23);
}
}, 300_000);
it('pages a prefix range without skipping or repeating rows', async () => {
await wipe(KV);
for (let owner = 0; owner < 3; owner += 1) {
for (let i = 0; i < 17; i += 1) {
await upsert(next, PagedTable, {owner_id: BigInt(owner), item_id: BigInt(i), payload: `p${owner}-${i}`});
}
}
const meta = selectMeta(PagedTable, [{kind: 'eq', col: 'owner_id', param: 'owner_id'} as WhereExpr<Row>]);
for (const pageSize of [1, 2, 4, 17, 18]) {
const result = await pageAll(next, meta, {owner_id: 1n}, pageSize);
expect(result.error).toBeNull();
const rows = result.pages.flat();
expect(rows.length, `pageSize=${pageSize}`).toBe(17);
expect(new Set(rows.map((r) => String(r.item_id))).size).toBe(17);
expect(rows.every((r) => String(r.owner_id) === '1')).toBe(true);
}
}, 300_000);
it('never loses a row that existed for the whole scan while rows are inserted between pages', async () => {
await wipe(KV);
const stable = new Set<string>();
for (let i = 0; i < 30; i += 1) {
await upsert(next, FlatTable, {k: BigInt(i), v: `v${i}`});
stable.add(String(i));
}
let inserted = 100;
const result = await pageAll(next, selectMeta(FlatTable), {}, 4, async () => {
await upsert(next, FlatTable, {k: BigInt(inserted), v: `late${inserted}`});
inserted += 1;
});
expect(result.error).toBeNull();
const seen = result.pages.flat().map((r) => String(r.k));
const dupes = seen.filter((key, index) => seen.indexOf(key) !== index);
const missing = [...stable].filter((key) => !seen.includes(key));
expect(missing, `missing stable rows: ${missing.join(',')}`).toEqual([]);
expect(dupes, `duplicate rows: ${dupes.join(',')}`).toEqual([]);
}, 300_000);
it('pages identically to the legacy executor while rows are deleted between pages', async () => {
const run = async (exec: AnyExec) => {
await wipe(KV);
for (let i = 0; i < 30; i += 1) await upsert(exec, FlatTable, {k: BigInt(i), v: `v${i}`});
let pageIndex = 0;
const result = await pageAll(exec, selectMeta(FlatTable), {}, 4, async () => {
const victim = pageIndex;
pageIndex += 1;
await exec.executeQuery({
cql: '__del__',
params: {k: BigInt(victim)},
kvMeta: deleteMeta(FlatTable, [{kind: 'eq', col: 'k', param: 'k'} as WhereExpr<Row>]),
});
});
return {
error: result.error,
pages: result.pages.map((page) => page.map((row) => String(row.k)).join(',')).join('|'),
};
};
const legacyRun = await run(legacy);
const nextRun = await run(next);
expect(nextRun.error).toBeNull();
expect(nextRun, `legacy=${legacyRun.pages} next=${nextRun.pages}`).toEqual(legacyRun);
}, 300_000);
it('pages identically to the legacy executor when the last row of each page is deleted', async () => {
const run = async (exec: AnyExec) => {
await wipe(KV);
for (let i = 0; i < 20; i += 1)
await upsert(exec, FlatTable, {k: `key${String(i).padStart(2, '0')}`, v: `v${i}`});
const meta = selectMeta(FlatTable);
const seen: Array<string> = [];
let pageState: string | null = null;
for (let guard = 0; guard < 100; guard += 1) {
const page: {rows: Array<Row>; pageState: string | null} = await exec.executePagedQuery<Row>(
{cql: '__cur__', params: {}, kvMeta: meta},
{pageSize: 3, pageState},
);
for (const row of page.rows) seen.push(String(row.k));
const cursorRow = page.rows[page.rows.length - 1];
pageState = page.pageState;
if (cursorRow) {
await exec.executeQuery({
cql: '__delcur__',
params: {k: cursorRow.k as string},
kvMeta: deleteMeta(FlatTable, [{kind: 'eq', col: 'k', param: 'k'} as WhereExpr<Row>]),
});
}
if (pageState === null) break;
}
return seen.join(',');
};
const legacySeen = await run(legacy);
const nextSeen = await run(next);
expect(nextSeen, `legacy=${legacySeen} next=${nextSeen}`).toBe(legacySeen);
}, 300_000);
it('never returns an empty page together with a non-null page state', async () => {
await wipe(KV);
for (let owner = 0; owner < 4; owner += 1) {
for (let i = 0; i < 11; i += 1) {
await upsert(next, PagedTable, {owner_id: BigInt(owner), item_id: BigInt(i), payload: `p${owner}-${i}`});
}
}
const shapes: Array<[string, AnyMeta, CassandraParams]> = [
['scan', selectMeta(PagedTable), {}],
[
'eq-owner',
selectMeta(PagedTable, [{kind: 'eq', col: 'owner_id', param: 'owner_id'} as WhereExpr<Row>]),
{owner_id: 2n},
],
[
'in-owner-single',
selectMeta(PagedTable, [{kind: 'in', col: 'owner_id', param: 'owner_ids'} as WhereExpr<Row>]),
{owner_ids: [2n]},
],
[
'in-owner-multi',
selectMeta(PagedTable, [{kind: 'in', col: 'owner_id', param: 'owner_ids'} as WhereExpr<Row>]),
{owner_ids: [1n, 3n]},
],
[
'in-owner-empty',
selectMeta(PagedTable, [{kind: 'in', col: 'owner_id', param: 'owner_ids'} as WhereExpr<Row>]),
{owner_ids: []},
],
];
const offenders: Array<string> = [];
for (const [name, meta, params] of shapes) {
for (const pageSize of [1, 2, 3, 5, 11]) {
let pageState: string | null = null;
const flat: Array<Row> = [];
for (let guard = 0; guard < 200; guard += 1) {
const page: {rows: Array<Row>; pageState: string | null} = await next.executePagedQuery<Row>(
{cql: `__empty__${name}`, params, kvMeta: meta},
{pageSize, pageState},
);
if (page.rows.length === 0 && page.pageState !== null) {
offenders.push(`${name}/size${pageSize} page ${guard}`);
}
flat.push(...page.rows);
pageState = page.pageState;
if (pageState === null) break;
}
const unpaged = await next.executeQuery<Row>({cql: `__whole__${name}`, params, kvMeta: meta});
expect(flat.length, `${name}/size${pageSize} row count`).toBe(unpaged.length);
const pagedKeys = flat.map((r) => `${String(r.owner_id)}:${String(r.item_id)}`).sort();
const wholeKeys = unpaged.map((r) => `${String(r.owner_id)}:${String(r.item_id)}`).sort();
expect(pagedKeys, `${name}/size${pageSize} multiset`).toEqual(wholeKeys);
}
}
expect(offenders, `empty page with continuation: ${offenders.join(', ')}`).toEqual([]);
}, 300_000);
it('matches legacy throw/no-throw behaviour for hand-crafted and corrupt page states', async () => {
await wipe(KV);
for (let i = 0; i < 9; i += 1) await upsert(next, FlatTable, {k: BigInt(i), v: `v${i}`});
const meta = selectMeta(FlatTable);
const query = {cql: '__tok__', params: {}, kvMeta: meta};
const tokens: Array<[string, string]> = [
['empty-object', token({})],
['number', token(123)],
['string', token('hello')],
['array', token([1, 2])],
['unknown-key', token({foo: 1})],
['offset-negative', token({offset: -1})],
['offset-fractional', token({offset: 1.5})],
['offset-string', token({offset: '2'})],
['offset-huge', token({offset: Number.MAX_SAFE_INTEGER})],
['after-number', token({after: 5})],
['after-null', token({after: null})],
['after-object', token({after: {a: 1}})],
['after-and-offset', token({after: 'x', offset: 3})],
['garbage', 'not-base64-at-all'],
];
const deltas: Array<string> = [];
for (const [name, value] of tokens) {
const legacyResult = await legacy
.executePagedQuery<Row>(query, {pageSize: 3, pageState: value})
.then((r) => ({err: null as string | null, rows: r.rows.length}))
.catch((e: Error) => ({err: e.message, rows: -1}));
const nextResult = await next
.executePagedQuery<Row>(query, {pageSize: 3, pageState: value})
.then((r) => ({err: null as string | null, rows: r.rows.length}))
.catch((e: Error) => ({err: e.message, rows: -1}));
if ((legacyResult.err === null) !== (nextResult.err === null)) {
deltas.push(
`${name}: legacy=${legacyResult.err ?? `ok(${legacyResult.rows})`} next=${nextResult.err ?? `ok(${nextResult.rows})`}`,
);
}
}
expect(deltas, `page-state error behaviour deltas:\n${deltas.join('\n')}`).toEqual([]);
}, 300_000);
it('matches legacy behaviour for degenerate page sizes', async () => {
await wipe(KV);
for (let i = 0; i < 9; i += 1) await upsert(next, FlatTable, {k: BigInt(i), v: `v${i}`});
const query = {cql: '__ps__', params: {}, kvMeta: selectMeta(FlatTable)};
const deltas: Array<string> = [];
for (const pageSize of [0, -1, 1.5, 2.9, Number.MAX_SAFE_INTEGER, 2 ** 31, Number.NaN]) {
const l = await legacy
.executePagedQuery<Row>(query, {pageSize})
.then((r) => `ok rows=${r.rows.length} more=${r.pageState !== null}`)
.catch((e: Error) => `throw ${e.message}`);
const n = await next
.executePagedQuery<Row>(query, {pageSize})
.then((r) => `ok rows=${r.rows.length} more=${r.pageState !== null}`)
.catch((e: Error) => `throw ${e.message}`);
if (l !== n) deltas.push(`pageSize=${String(pageSize)}: legacy=${l} next=${n}`);
}
expect(deltas, `page size deltas:\n${deltas.join('\n')}`).toEqual([]);
}, 300_000);
it('does not silently truncate when a page token is reused across a different query', async () => {
await wipe(KV);
for (let owner = 0; owner < 3; owner += 1) {
for (let i = 0; i < 9; i += 1) {
await upsert(next, PagedTable, {owner_id: BigInt(owner), item_id: BigInt(i), payload: `p${owner}-${i}`});
}
}
const meta = selectMeta(PagedTable, [{kind: 'eq', col: 'owner_id', param: 'owner_id'} as WhereExpr<Row>]);
const firstNext = await next.executePagedQuery<Row>(
{cql: '__x__', params: {owner_id: 0n}, kvMeta: meta},
{pageSize: 3},
);
const firstLegacy = await legacy.executePagedQuery<Row>(
{cql: '__x__', params: {owner_id: 0n}, kvMeta: meta},
{pageSize: 3},
);
expect(firstNext.pageState).not.toBeNull();
const crossNext = await next.executePagedQuery<Row>(
{cql: '__x__', params: {owner_id: 2n}, kvMeta: meta},
{pageSize: 3, pageState: firstNext.pageState},
);
const crossLegacy = await legacy.executePagedQuery<Row>(
{cql: '__x__', params: {owner_id: 2n}, kvMeta: meta},
{pageSize: 3, pageState: firstLegacy.pageState},
);
expect(crossNext.rows.length, 'cross-query token row count diverges from legacy').toBe(crossLegacy.rows.length);
}, 300_000);
it('does not restart the scan when a keyset token reaches a non-keyset plan', async () => {
await wipe(KV);
for (let owner = 0; owner < 3; owner += 1) {
for (let i = 0; i < 9; i += 1) {
await upsert(next, PagedTable, {owner_id: BigInt(owner), item_id: BigInt(i), payload: `p${owner}-${i}`});
}
}
const inMeta = selectMeta(PagedTable, [{kind: 'in', col: 'owner_id', param: 'owner_ids'} as WhereExpr<Row>]);
const first = await next.executePagedQuery<Row>(
{cql: '__flip__', params: {owner_ids: [1n]}, kvMeta: inMeta},
{pageSize: 3},
);
expect(first.pageState).not.toBeNull();
const flipped = await next.executePagedQuery<Row>(
{cql: '__flip__', params: {owner_ids: [1n, 2n]}, kvMeta: inMeta},
{pageSize: 3, pageState: first.pageState},
);
const firstKeys = first.rows.map((r) => String(r.item_id));
const flippedKeys = flipped.rows.map((r) => String(r.item_id));
const overlap = flippedKeys.filter((key) => firstKeys.includes(key));
expect(overlap, `keyset token silently restarted the scan: first=${firstKeys} flipped=${flippedKeys}`).toEqual([]);
}, 300_000);
it('pages correctly on a database whose row_key column has a linguistic collation', async () => {
await wipe(ICU_KV);
const keys = ['a', 'A', 'a b', 'ab', 'B', 'b', '\u{1f600}', 'z', 'Z', '', '"', '\\', 'ab'];
for (const key of keys) await upsert(icuNext, FlatTable, {k: key, v: `v:${key}`});
for (const pageSize of [1, 2, 3, 5]) {
const result = await pageAll(icuNext, selectMeta(FlatTable), {}, pageSize);
expect(result.error, `icu pageSize=${pageSize}`).toBeNull();
const seen = result.pages.flat().map((r) => String(r.k));
const dupes = seen.filter((key, index) => seen.indexOf(key) !== index);
expect(dupes, `icu dupes pageSize=${pageSize}: ${JSON.stringify(dupes)}`).toEqual([]);
expect(seen.length, `icu count pageSize=${pageSize}`).toBe(keys.length);
}
const legacyAll = await pageAll(icuLegacy, selectMeta(FlatTable), {}, 3);
expect(legacyAll.pages.flat().length).toBe(keys.length);
}, 300_000);
it('reports the paged order delta against legacy for key shapes that reach real callers', async () => {
const DatedTable: KvTableSpec<Row> = {
name: 'pagead_dated',
columns: ['customer_id', 'created_at', 'provider_id', 'total'],
primaryKey: ['customer_id', 'created_at', 'provider_id'],
partitionKey: ['customer_id'],
};
const NumericTable: KvTableSpec<Row> = {
name: 'pagead_numeric',
columns: ['owner_id', 'seq', 'payload'],
primaryKey: ['owner_id', 'seq'],
partitionKey: ['owner_id'],
};
await wipe(KV);
for (let i = 0; i < 15; i += 1) {
await upsert(next, DatedTable, {
customer_id: 'cus_1',
created_at: new Date(Date.UTC(2024, 0, 1 + i, i, i)),
provider_id: `in_${String(i).padStart(3, '0')}`,
total: BigInt(i),
});
await upsert(next, NumericTable, {owner_id: 7n, seq: BigInt(i), payload: `p${i}`});
}
const datedMeta = selectMeta(DatedTable, [
{kind: 'eq', col: 'customer_id', param: 'customer_id'} as WhereExpr<Row>,
]);
const numericMeta = selectMeta(NumericTable, [{kind: 'eq', col: 'owner_id', param: 'owner_id'} as WhereExpr<Row>]);
const datedLegacy = await pageAll(legacy, datedMeta, {customer_id: 'cus_1'}, 4);
const datedNext = await pageAll(next, datedMeta, {customer_id: 'cus_1'}, 4);
const numericLegacy = await pageAll(legacy, numericMeta, {owner_id: 7n}, 4);
const numericNext = await pageAll(next, numericMeta, {owner_id: 7n}, 4);
const datedLegacyOrder = datedLegacy.pages.flat().map((r) => String(r.provider_id));
const datedNextOrder = datedNext.pages.flat().map((r) => String(r.provider_id));
const numericLegacyOrder = numericLegacy.pages.flat().map((r) => String(r.seq));
const numericNextOrder = numericNext.pages.flat().map((r) => String(r.seq));
const deltas: Array<string> = [];
if (datedLegacyOrder.join(',') !== datedNextOrder.join(',')) {
deltas.push(`date-keyed: legacy=${datedLegacyOrder.join(',')} next=${datedNextOrder.join(',')}`);
}
if (numericLegacyOrder.join(',') !== numericNextOrder.join(',')) {
deltas.push(`bigint-keyed: legacy=${numericLegacyOrder.join(',')} next=${numericNextOrder.join(',')}`);
}
expect(datedNextOrder.slice().sort(), 'date-keyed row set').toEqual(datedLegacyOrder.slice().sort());
expect(numericNextOrder.slice().sort(), 'bigint-keyed row set').toEqual(numericLegacyOrder.slice().sort());
expect(deltas, `paged order deltas:\n${deltas.join('\n')}`).toEqual([]);
}, 300_000);
it('pages a prefix range whose keys sit adjacent to the range bounds', async () => {
await wipe(KV);
const owners = ['a', 'a b', 'ab', 'a', 'a"'];
for (const owner of owners) {
for (let i = 0; i < 5; i += 1) {
await upsert(next, PagedTable, {owner_id: owner, item_id: BigInt(i), payload: `${owner}#${i}`});
}
}
const meta = selectMeta(PagedTable, [{kind: 'eq', col: 'owner_id', param: 'owner_id'} as WhereExpr<Row>]);
for (const owner of owners) {
for (const pageSize of [1, 2, 5]) {
const result = await pageAll(next, meta, {owner_id: owner}, pageSize);
expect(result.error).toBeNull();
const rows = result.pages.flat();
expect(rows.length, `owner=${JSON.stringify(owner)} pageSize=${pageSize}`).toBe(5);
expect(
rows.every((r) => r.owner_id === owner),
`owner leak for ${JSON.stringify(owner)}`,
).toBe(true);
}
}
}, 300_000);
});
File diff suppressed because it is too large Load Diff
@@ -2,11 +2,14 @@
import {type IPostgresClient, type PostgresQueryable, quoteIdentifier} from '@pkgs/postgres/src/Client';
import cassandra from 'cassandra-driver';
import {Logger} from '../Logger';
import {getKvMeta, getTableMetadata} from './CassandraMetaRegistry';
import type {CassandraParams, ColumnName, KvQueryMeta, PreparedQuery, WhereExpr} from './CassandraTypes';
type Row = Record<string, unknown>;
type EqWhereExpr = Extract<WhereExpr<Row>, {kind: 'eq'}>;
type InWhereExpr = Extract<WhereExpr<Row>, {kind: 'in'}>;
type PinnedWhereExpr = EqWhereExpr | InWhereExpr;
interface StoredRow {
row_key: string;
@@ -17,7 +20,40 @@ interface PageState {
offset: number;
}
export type CandidatePlan =
| {kind: 'none'}
| {kind: 'rowKeys'; rowKeys: Array<string>}
| {kind: 'range'; lowerBound: string; upperBound: string}
| {kind: 'ranges'; lowerBounds: Array<string>; upperBounds: Array<string>}
| {kind: 'partitionKeys'; partitionKeys: Array<string>}
| {kind: 'scan'};
interface QueryPlan {
candidates: CandidatePlan;
exact: boolean;
}
interface QueryShape {
leadingClauses: ReadonlyArray<PinnedWhereExpr>;
partitionClauses: ReadonlyArray<PinnedWhereExpr> | null;
inClauses: ReadonlyArray<InWhereExpr>;
whereColumns: ReadonlyArray<string>;
requiredColumns: ReadonlyArray<string> | null;
clauseCount: number;
summary: string;
}
interface PlanFragments {
predicate: string;
params: Array<unknown>;
}
const VALUE_SEPARATOR = '\u001f';
const KEY_RANGE_UPPER = ' ';
const MAX_ROW_KEY_COMBINATIONS = 32_768;
const MAX_PREFIX_RANGES = 256;
const FULL_SCAN_LOG_INTERVAL_MS = 60_000;
const FULL_SCAN_LOG_KEY_LIMIT = 1024;
const ENCODED_TYPE_KEY = '__fluxer_type';
const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE = 0x46584b56;
const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT = '120s';
@@ -93,11 +129,24 @@ function decodeRow(value: unknown): Row {
return decoded;
}
function decodeRowColumns(value: unknown, columns: ReadonlyArray<string>): Row {
if (!isPlainObject(value)) {
throw new Error('Postgres KV row payload is not an object');
}
const decoded: Row = {};
for (const column of columns) {
if (column in value) {
decoded[column] = decodeValue(value[column]);
}
}
return decoded;
}
function valueKey(value: unknown): string {
return JSON.stringify(encodeValue(value));
}
function keyFromColumns(columns: ReadonlyArray<string>, row: Row): string {
export function keyFromColumns(columns: ReadonlyArray<string>, row: Row): string {
return columns.map((column) => valueKey(row[column])).join(VALUE_SEPARATOR);
}
@@ -137,10 +186,6 @@ function rowKeyFromParams(meta: KvQueryMeta, params: CassandraParams): string {
return rowKey(meta, paramsRow(params, (meta.pkColumns ?? meta.table.primaryKey) as ReadonlyArray<string>));
}
function partitionKeyFromParams(meta: KvQueryMeta, params: CassandraParams): string {
return partitionKey(meta, paramsRow(params, meta.table.partitionKey as ReadonlyArray<string>));
}
function compareValues(left: unknown, right: unknown): number {
if (typeof left === 'bigint' || typeof right === 'bigint') {
const l = typeof left === 'bigint' ? left : BigInt(left as number | string);
@@ -160,8 +205,8 @@ function valuesEqual(left: unknown, right: unknown): boolean {
if (left == null && right == null) return true;
if (left instanceof Date && right instanceof Date) return left.getTime() === right.getTime();
if (Buffer.isBuffer(left) && Buffer.isBuffer(right)) return left.equals(right);
if (left?.constructor?.name === 'LocalDate' || right?.constructor?.name === 'LocalDate') {
return left?.toString() === right?.toString();
if (left?.constructor?.name === 'LocalDate' && right?.constructor?.name === 'LocalDate') {
return left.toString() === right.toString();
}
return left === right;
}
@@ -170,7 +215,11 @@ function getParam(params: CassandraParams, param: string): unknown {
return params[param];
}
function matchesWhere(row: Row, where: ReadonlyArray<WhereExpr<Row>> | undefined, params: CassandraParams): boolean {
export function matchesWhere(
row: Row,
where: ReadonlyArray<WhereExpr<Row>> | undefined,
params: CassandraParams,
): boolean {
for (const clause of where ?? []) {
switch (clause.kind) {
case 'eq':
@@ -238,39 +287,269 @@ function sortRows(meta: KvQueryMeta, rows: Array<Row>): Array<Row> {
});
}
function equalityParam(where: ReadonlyArray<WhereExpr<Row>> | undefined, column: string): string | null {
const clause = (where ?? []).find((entry) => entry.kind === 'eq' && entry.col === column);
return clause && clause.kind === 'eq' ? clause.param : null;
function whereClauses(meta: KvQueryMeta): ReadonlyArray<WhereExpr<Row>> {
return (meta.where ?? []) as ReadonlyArray<WhereExpr<Row>>;
}
function inParam(where: ReadonlyArray<WhereExpr<Row>> | undefined, column: string): string | null {
const clause = (where ?? []).find((entry) => entry.kind === 'in' && entry.col === column);
return clause && clause.kind === 'in' ? clause.param : null;
}
function fullRowKeysFromWhere(meta: KvQueryMeta, params: CassandraParams): Array<string> | null {
const pk = meta.table.primaryKey as ReadonlyArray<string>;
const eqParams = pk.map((column) => equalityParam(meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, column));
if (eqParams.every((param) => param !== null)) {
const row: Row = {};
for (let i = 0; i < pk.length; i += 1) row[pk[i]!] = params[eqParams[i]!];
return [rowKey(meta, row)];
}
if (pk.length === 1) {
const param = inParam(meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, pk[0]!);
if (param) {
const values = params[param] as ReadonlyArray<unknown> | Set<unknown>;
const haystack = values instanceof Set ? [...values] : values;
return haystack.map((value) => rowKey(meta, {[pk[0]!]: value}));
}
function pinnedClause(where: ReadonlyArray<WhereExpr<Row>>, column: string): PinnedWhereExpr | null {
for (const clause of where) {
if ((clause.kind === 'eq' || clause.kind === 'in') && clause.col === column) return clause;
}
return null;
}
function hasFullPartition(meta: KvQueryMeta): boolean {
return meta.table.partitionKey.every((column) =>
equalityParam(meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, column),
);
function clauseColumns(clause: WhereExpr<Row>): ReadonlyArray<string> {
return clause.kind === 'tupleGt' ? (clause.cols as ReadonlyArray<string>) : [clause.col as string];
}
function describeClause(clause: WhereExpr<Row>): string {
return `${clauseColumns(clause).join('+')} ${clause.kind}`;
}
function requiredColumnsFor(meta: KvQueryMeta, whereColumns: ReadonlyArray<string>): ReadonlyArray<string> | null {
if (!meta.columns) return null;
const required = new Set<string>(meta.columns as ReadonlyArray<string>);
for (const column of whereColumns) required.add(column);
if (meta.orderBy) required.add(meta.orderBy.col as string);
for (const column of meta.table.primaryKey as ReadonlyArray<string>) required.add(column);
for (const column of meta.table.columns as ReadonlyArray<string>) {
if (!required.has(column)) return [...required];
}
return null;
}
const QUERY_SHAPES = new WeakMap<KvQueryMeta, QueryShape>();
function queryShape(meta: KvQueryMeta): QueryShape {
const cached = QUERY_SHAPES.get(meta);
if (cached) return cached;
const where = whereClauses(meta);
const leadingClauses: Array<PinnedWhereExpr> = [];
for (const column of meta.table.primaryKey as ReadonlyArray<string>) {
const clause = pinnedClause(where, column);
if (!clause) break;
leadingClauses.push(clause);
}
const partitionColumns = meta.table.partitionKey as ReadonlyArray<string>;
const partitionClauses: Array<PinnedWhereExpr> = [];
for (const column of partitionColumns) {
const clause = pinnedClause(where, column);
if (!clause) {
partitionClauses.length = 0;
break;
}
partitionClauses.push(clause);
}
const whereColumns = [...new Set(where.flatMap(clauseColumns))];
const shape: QueryShape = {
leadingClauses,
partitionClauses:
partitionColumns.length > 0 && partitionClauses.length === partitionColumns.length ? partitionClauses : null,
inClauses: where.filter((clause): clause is InWhereExpr => clause.kind === 'in'),
whereColumns,
requiredColumns: requiredColumnsFor(meta, whereColumns),
clauseCount: where.length,
summary: where.map(describeClause).join(', '),
};
QUERY_SHAPES.set(meta, shape);
return shape;
}
function clauseValues(clause: PinnedWhereExpr, params: CassandraParams): Array<unknown> | null {
if (clause.kind === 'eq') return [getParam(params, clause.param)];
const values = getParam(params, clause.param);
if (values === null || values === undefined) return [];
if (values instanceof Set) return [...values];
if (Array.isArray(values)) return [...values];
return null;
}
function isKeyComparableValue(value: unknown): boolean {
if (value === null || value === undefined) return true;
if (typeof value === 'string' || typeof value === 'boolean' || typeof value === 'bigint') return true;
if (typeof value === 'number') return Number.isFinite(value);
if (Buffer.isBuffer(value)) return true;
if (value instanceof Date) return !Number.isNaN(value.getTime());
return false;
}
function keySegments(values: ReadonlyArray<unknown>): Array<string> | null {
const segments: Array<string> = [];
for (const value of values) {
let segment: string;
try {
segment = valueKey(value);
} catch {
return null;
}
if (typeof segment !== 'string') return null;
segments.push(segment);
}
return segments;
}
function combinationCount(segmentLists: ReadonlyArray<ReadonlyArray<string>>): number {
let total = 1;
for (const segments of segmentLists) total *= segments.length;
return total;
}
function keyPrefixes(segmentLists: ReadonlyArray<ReadonlyArray<string>>): Array<string> {
let prefixes: Array<string> | null = null;
for (const segments of segmentLists) {
const next: Array<string> = [];
for (const segment of segments) {
if (prefixes === null) {
next.push(segment);
continue;
}
for (const prefix of prefixes) next.push(`${prefix}${VALUE_SEPARATOR}${segment}`);
}
prefixes = next;
}
return [...new Set(prefixes ?? [])];
}
function keyRangeLowerBound(prefix: string): string {
return `${prefix}${VALUE_SEPARATOR}`;
}
function keyRangeUpperBound(prefix: string): string {
return `${prefix}${KEY_RANGE_UPPER}`;
}
interface PinnedColumn {
values: Array<unknown>;
segments: Array<string>;
}
function pinnedColumns(clauses: ReadonlyArray<PinnedWhereExpr>, params: CassandraParams): Array<PinnedColumn> {
const pinned: Array<PinnedColumn> = [];
for (const clause of clauses) {
const values = clauseValues(clause, params);
if (values === null || values.length === 0) break;
const segments = keySegments(values);
if (segments === null) break;
pinned.push({values, segments});
}
return pinned;
}
function planIsExact(meta: KvQueryMeta, shape: QueryShape, pinned: ReadonlyArray<PinnedColumn>): boolean {
if (pinned.length !== shape.clauseCount) return false;
if (meta.limit !== undefined || meta.orderBy !== undefined) return false;
for (const column of pinned) {
for (const value of column.values) {
if (!isKeyComparableValue(value)) return false;
}
}
return true;
}
export function buildCandidatePlan(meta: KvQueryMeta, params: CassandraParams): QueryPlan {
const shape = queryShape(meta);
for (const clause of shape.inClauses) {
const values = clauseValues(clause, params);
if (values !== null && values.length === 0) {
return {candidates: {kind: 'none'}, exact: true};
}
}
const primaryKey = meta.table.primaryKey as ReadonlyArray<string>;
const pinned = pinnedColumns(shape.leadingClauses, params);
let leading = pinned.length;
while (leading > 0) {
const segmentLists = pinned.slice(0, leading).map((column) => column.segments);
const cap = leading === primaryKey.length ? MAX_ROW_KEY_COMBINATIONS : MAX_PREFIX_RANGES;
if (combinationCount(segmentLists) <= cap) break;
leading -= 1;
}
if (leading > 0) {
const columns = pinned.slice(0, leading);
const prefixes = keyPrefixes(columns.map((column) => column.segments));
const exact = planIsExact(meta, shape, columns);
if (leading === primaryKey.length) {
return {candidates: {kind: 'rowKeys', rowKeys: prefixes}, exact};
}
if (prefixes.length === 1) {
const prefix = prefixes[0]!;
return {
candidates: {kind: 'range', lowerBound: keyRangeLowerBound(prefix), upperBound: keyRangeUpperBound(prefix)},
exact,
};
}
return {
candidates: {
kind: 'ranges',
lowerBounds: prefixes.map(keyRangeLowerBound),
upperBounds: prefixes.map(keyRangeUpperBound),
},
exact,
};
}
if (shape.partitionClauses) {
const partition = pinnedColumns(shape.partitionClauses, params);
if (
partition.length === shape.partitionClauses.length &&
combinationCount(partition.map((column) => column.segments)) <= MAX_ROW_KEY_COMBINATIONS
) {
return {
candidates: {kind: 'partitionKeys', partitionKeys: keyPrefixes(partition.map((column) => column.segments))},
exact: planIsExact(meta, shape, partition),
};
}
}
return {candidates: {kind: 'scan'}, exact: planIsExact(meta, shape, [])};
}
export function planFragments(plan: CandidatePlan): PlanFragments {
switch (plan.kind) {
case 'rowKeys':
return {predicate: ' AND kv.row_key = ANY($2::text[])', params: [plan.rowKeys]};
case 'range':
return {
predicate: ' AND kv.row_key COLLATE "C" >= $2 AND kv.row_key COLLATE "C" < $3',
params: [plan.lowerBound, plan.upperBound],
};
case 'ranges': {
const params: Array<unknown> = [];
const arms = plan.lowerBounds.map((lowerBound, index) => {
params.push(lowerBound, plan.upperBounds[index]);
return `(kv.row_key COLLATE "C" >= $${params.length} AND kv.row_key COLLATE "C" < $${params.length + 1})`;
});
return {predicate: ` AND (${arms.join(' OR ')})`, params};
}
case 'partitionKeys':
return plan.partitionKeys.length === 1
? {predicate: ' AND kv.partition_key = $2', params: [plan.partitionKeys[0]]}
: {predicate: ' AND kv.partition_key = ANY($2::text[])', params: [plan.partitionKeys]};
default:
return {predicate: '', params: []};
}
}
function logWarn(details: Record<string, unknown>, message: string): void {
try {
Logger.warn(details, message);
} catch {}
}
function logError(details: Record<string, unknown>, message: string): void {
try {
Logger.error(details, message);
} catch {}
}
const fullScanLoggedAt = new Map<string, number>();
function logFullScan(meta: KvQueryMeta): void {
const shape = queryShape(meta);
const key = `${meta.table.name}|${meta.action}|${shape.summary}`;
const now = Date.now();
const last = fullScanLoggedAt.get(key);
if (last !== undefined && now - last < FULL_SCAN_LOG_INTERVAL_MS) return;
if (fullScanLoggedAt.size >= FULL_SCAN_LOG_KEY_LIMIT) fullScanLoggedAt.clear();
fullScanLoggedAt.set(key, now);
logWarn({table: meta.table.name, action: meta.action, where: shape.summary || 'none'}, 'Postgres KV full table scan');
}
function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null | undefined {
@@ -355,6 +634,24 @@ function parseEqWhere(whereSql: string, cql: string): ReadonlyArray<EqWhereExpr>
});
}
async function repairInvalidPostgresKvIndexes(db: PostgresQueryable, kvTable: string): Promise<void> {
const invalid = await db.query<{index_name: string; index_def: string}>(
`SELECT cls.relname AS index_name, pg_get_indexdef(idx.indexrelid) AS index_def
FROM pg_index idx
JOIN pg_class cls ON cls.oid = idx.indexrelid
WHERE idx.indrelid = to_regclass($1)
AND NOT idx.indisvalid
AND NOT idx.indisprimary
AND NOT idx.indisunique`,
[kvTable],
);
for (const row of invalid.rows) {
logError({table: kvTable, index: row.index_name}, 'Postgres KV index is invalid, rebuilding it');
await db.query(`DROP INDEX IF EXISTS ${quoteIdentifier(row.index_name)}`);
await db.query(row.index_def);
}
}
export async function ensurePostgresKvSchema(client: IPostgresClient): Promise<void> {
const kvTable = client.kvTable();
const table = quoteIdentifier(kvTable);
@@ -387,12 +684,22 @@ CREATE TABLE IF NOT EXISTS ${table} (
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`,
);
await db.query(`
await repairInvalidPostgresKvIndexes(db, kvTable);
const pending = await db.query(`
SELECT 1
FROM ${table}
WHERE table_name = 'messages'
AND partition_key = row_key
AND split_part(row_key, chr(31), 3) <> ''
LIMIT 1`);
if (pending.rows.length > 0) {
await db.query(`
UPDATE ${table}
SET partition_key = split_part(row_key, chr(31), 1) || chr(31) || split_part(row_key, chr(31), 2)
WHERE table_name = 'messages'
AND partition_key = row_key
AND split_part(row_key, chr(31), 3) <> ''`);
}
await db.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${kvTable}_partition_idx`)}`);
});
}
@@ -434,9 +741,9 @@ export class PostgresKvQueryExecutor {
const meta = this.meta(query);
switch (meta.action) {
case 'select':
return (await this.select(meta, query.params, db)) as Array<T>;
return (await this.select(meta, query.params, buildCandidatePlan(meta, query.params), db)) as Array<T>;
case 'count':
return [{count: (await this.select(meta, query.params, db)).length}] as Array<T>;
return (await this.count(meta, query.params, db)) as Array<T>;
case 'upsert':
return (await this.upsert(meta, query.params, db)) as Array<T>;
case 'insert':
@@ -495,42 +802,51 @@ export class PostgresKvQueryExecutor {
return meta;
}
private async candidates(
meta: KvQueryMeta,
params: CassandraParams,
db: PostgresQueryable,
): Promise<Array<StoredRow>> {
const rowKeys = fullRowKeysFromWhere(meta, params);
if (rowKeys) {
const result = await db.query<StoredRow>(
`SELECT row_key, row_data FROM ${this.table} WHERE table_name = $1 AND row_key = ANY($2::text[]) AND (expires_at IS NULL OR expires_at > now())`,
[meta.table.name, rowKeys],
);
return result.rows;
}
if (hasFullPartition(meta)) {
const result = await db.query<StoredRow>(
`SELECT row_key, row_data FROM ${this.table} WHERE table_name = $1 AND partition_key = $2 AND (expires_at IS NULL OR expires_at > now())`,
[meta.table.name, partitionKeyFromParams(meta, params)],
);
return result.rows;
}
private async candidates(meta: KvQueryMeta, plan: QueryPlan, db: PostgresQueryable): Promise<Array<StoredRow>> {
if (plan.candidates.kind === 'none') return [];
if (plan.candidates.kind === 'scan') logFullScan(meta);
const fragments = planFragments(plan.candidates);
const result = await db.query<StoredRow>(
`SELECT row_key, row_data FROM ${this.table} WHERE table_name = $1 AND (expires_at IS NULL OR expires_at > now())`,
[meta.table.name],
`SELECT kv.row_key, kv.row_data FROM ${this.table} kv WHERE kv.table_name = $1${fragments.predicate} AND (kv.expires_at IS NULL OR kv.expires_at > now())`,
[meta.table.name, ...fragments.params],
);
return result.rows;
}
private async select(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<Array<Row>> {
let rows = (await this.candidates(meta, params, db))
.map((stored) => decodeRow(stored.row_data))
private matchingRows(meta: KvQueryMeta, stored: ReadonlyArray<StoredRow>, params: CassandraParams): Array<Row> {
const required = queryShape(meta).requiredColumns;
return stored
.map((entry) => (required ? decodeRowColumns(entry.row_data, required) : decodeRow(entry.row_data)))
.filter((row) => matchesWhere(row, meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, params));
}
private async select(
meta: KvQueryMeta,
params: CassandraParams,
plan: QueryPlan,
db: PostgresQueryable,
): Promise<Array<Row>> {
let rows = this.matchingRows(meta, await this.candidates(meta, plan, db), params);
rows = sortRows(meta, rows);
if (typeof meta.limit === 'number') rows = rows.slice(0, meta.limit);
return rows.map((row) => projectRow(row, meta.columns as ReadonlyArray<string> | undefined));
}
private async count(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<Array<Row>> {
const plan = buildCandidatePlan(meta, params);
if (!plan.exact) {
return [{count: (await this.select(meta, params, plan, db)).length}];
}
if (plan.candidates.kind === 'none') return [{count: 0}];
if (plan.candidates.kind === 'scan') logFullScan(meta);
const fragments = planFragments(plan.candidates);
const result = await db.query<{count: string}>(
`SELECT count(*) AS count FROM ${this.table} kv WHERE kv.table_name = $1${fragments.predicate} AND (kv.expires_at IS NULL OR kv.expires_at > now())`,
[meta.table.name, ...fragments.params],
);
return [{count: Number(result.rows[0]?.count ?? 0)}];
}
private async upsert(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<Array<Row>> {
const incoming = rowFromParams(meta, params);
const key = rowKey(meta, incoming);
@@ -587,10 +903,26 @@ DO UPDATE SET partition_key = EXCLUDED.partition_key, row_data = EXCLUDED.row_da
}
private async delete(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<void> {
const rows = await this.candidates(meta, params, db);
const plan = buildCandidatePlan(meta, params);
if (plan.candidates.kind === 'none') return;
if (plan.exact) {
if (plan.candidates.kind === 'scan') logFullScan(meta);
const fragments = planFragments(plan.candidates);
await db.query(
`DELETE FROM ${this.table} kv WHERE kv.table_name = $1${fragments.predicate} AND (kv.expires_at IS NULL OR kv.expires_at > now())`,
[meta.table.name, ...fragments.params],
);
return;
}
const whereColumns = queryShape(meta).whereColumns;
const rows = await this.candidates(meta, plan, db);
const matchingKeys = rows
.filter((stored) =>
matchesWhere(decodeRow(stored.row_data), meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, params),
matchesWhere(
decodeRowColumns(stored.row_data, whereColumns),
meta.where as ReadonlyArray<WhereExpr<Row>> | undefined,
params,
),
)
.map((stored) => stored.row_key);
if (matchingKeys.length === 0) return;
@@ -0,0 +1,441 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import cassandra from 'cassandra-driver';
import {describe, expect, it} from 'vitest';
import {defineTable} from './CassandraTableDsl';
import type {CassandraParams, PreparedQuery} from './CassandraTypes';
import {buildCandidatePlan, type CandidatePlan, keyFromColumns, matchesWhere} from './PostgresKvQueryExecutor';
type Row = Record<string, unknown>;
const SEPARATOR = '\u001f';
const TRICKY_STRINGS = [
'',
'a',
'a"b',
'a\\b',
'a\u0001b',
'a\u001fb',
'a b',
'a\u007fb',
'\u{1f600}',
'\ufffd',
'a\u0000b',
'b',
];
function byteCompare(left: string, right: string): number {
return Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8'));
}
function withinRange(rowKey: string, lowerBound: string, upperBound: string): boolean {
return byteCompare(rowKey, lowerBound) >= 0 && byteCompare(rowKey, upperBound) < 0;
}
function planAccepts(plan: CandidatePlan, rowKey: string, partitionKey: string): boolean {
switch (plan.kind) {
case 'none':
return false;
case 'rowKeys':
return plan.rowKeys.includes(rowKey);
case 'range':
return withinRange(rowKey, plan.lowerBound, plan.upperBound);
case 'ranges':
return plan.lowerBounds.some((lower, index) => withinRange(rowKey, lower, plan.upperBounds[index]!));
case 'partitionKeys':
return plan.partitionKeys.includes(partitionKey);
case 'scan':
return true;
}
}
function matchingRangeCount(plan: CandidatePlan, rowKey: string): number {
if (plan.kind !== 'ranges') return 0;
return plan.lowerBounds.filter((lower, index) => withinRange(rowKey, lower, plan.upperBounds[index]!)).length;
}
function expectKeyShape(key: string, separators: number): void {
let seen = 0;
for (const byte of Buffer.from(key, 'utf8')) {
if (byte === 0x1f) {
seen += 1;
continue;
}
expect(byte).toBeGreaterThanOrEqual(0x20);
}
expect(seen).toBe(separators);
}
function planOf(query: PreparedQuery) {
const meta = query.kvMeta;
if (!meta) {
throw new Error('Query is missing kv metadata');
}
return {meta, plan: buildCandidatePlan(meta, query.params)};
}
function checkQuery(query: PreparedQuery, rows: Array<Row>): CandidatePlan {
const {meta, plan} = planOf(query);
const primaryKey = meta.table.primaryKey as ReadonlyArray<string>;
const partitionKey = meta.table.partitionKey as ReadonlyArray<string>;
const accepted: Array<Row> = [];
const matched: Array<Row> = [];
for (const row of rows) {
const rowKey = keyFromColumns(primaryKey, row);
expectKeyShape(rowKey, primaryKey.length - 1);
expect(matchingRangeCount(plan.candidates, rowKey)).toBeLessThanOrEqual(1);
if (planAccepts(plan.candidates, rowKey, keyFromColumns(partitionKey, row))) accepted.push(row);
if (matchesWhere(row, meta.where, query.params)) matched.push(row);
}
for (const row of matched) {
expect(accepted).toContain(row);
}
if (plan.exact) {
expect(accepted).toEqual(matched);
}
return plan.candidates;
}
interface SingleRow {
a: string;
payload: string;
}
const SingleKey = defineTable<SingleRow, 'a'>({
name: 'kv_plan_single',
columns: ['a', 'payload'],
primaryKey: ['a'],
});
const singleRows: Array<Row> = TRICKY_STRINGS.map((a) => ({a, payload: 'p'}));
interface PairRow {
a: string;
b: bigint;
payload: string;
}
const PairKey = defineTable<PairRow, 'a' | 'b'>({
name: 'kv_plan_pair',
columns: ['a', 'b', 'payload'],
primaryKey: ['a', 'b'],
});
const pairRows: Array<Row> = TRICKY_STRINGS.flatMap((a) =>
[-1n, 0n, 9n, 10n, 1000n].map((b) => ({a, b, payload: 'p'})),
);
interface TripleRow {
a: string;
b: bigint;
c: boolean;
payload: string;
}
const TripleKey = defineTable<TripleRow, 'a' | 'b' | 'c'>({
name: 'kv_plan_triple',
columns: ['a', 'b', 'c', 'payload'],
primaryKey: ['a', 'b', 'c'],
});
const tripleRows: Array<Row> = TRICKY_STRINGS.flatMap((a) =>
[9n, 10n].flatMap((b) => [true, false].map((c) => ({a, b, c, payload: 'p'}))),
);
interface WideRow {
a: string;
b: bigint;
c: boolean;
d: Date | null;
e: Buffer;
f: number;
payload: string;
}
const WideKey = defineTable<WideRow, 'a' | 'b' | 'c' | 'd' | 'e' | 'f'>({
name: 'kv_plan_wide',
columns: ['a', 'b', 'c', 'd', 'e', 'f', 'payload'],
primaryKey: ['a', 'b', 'c', 'd', 'e', 'f'],
});
const wideRows: Array<Row> = ['a', 'a"b', '\u{1f600}'].flatMap((a) =>
[9n, 10n].flatMap((b) =>
[true, false].flatMap((c) =>
[null, new Date('2020-01-01T00:00:00.000Z')].flatMap((d) =>
[Buffer.from('one'), Buffer.from([0x00, 0x1f, 0xff])].flatMap((e) =>
[1, 2].map((f) => ({a, b, c, d, e, f, payload: 'p'})),
),
),
),
),
);
interface PartitionedRow {
a: string;
b: string;
payload: string;
}
const PartitionedKey = defineTable<PartitionedRow, 'a' | 'b', 'b'>({
name: 'kv_plan_partitioned',
columns: ['a', 'b', 'payload'],
primaryKey: ['a', 'b'],
partitionKey: ['b'],
});
const partitionedRows: Array<Row> = TRICKY_STRINGS.flatMap((a) => TRICKY_STRINGS.map((b) => ({a, b, payload: 'p'})));
describe('PostgresKvQueryExecutor candidate planner', () => {
it('pins a single-column primary key by equality', () => {
for (const a of TRICKY_STRINGS) {
expect(checkQuery(SingleKey.select({where: SingleKey.where.eq('a')}).bind({a}), singleRows).kind).toBe('rowKeys');
}
});
it('pins a single-column primary key by IN', () => {
const query = SingleKey.select({where: SingleKey.where.in('a', 'ids')}).bind({
ids: ['a', 'a\u0001b', 'a"b', 'a'],
});
expect(checkQuery(query, singleRows).kind).toBe('rowKeys');
});
it('returns nothing for an empty IN list', () => {
for (const params of [{ids: []}, {ids: new Set<string>()}, {}]) {
const query = SingleKey.select({where: SingleKey.where.in('a', 'ids')}).bind(params as CassandraParams);
expect(checkQuery(query, singleRows).kind).toBe('none');
}
});
it('ranges over the leading column of a composite primary key', () => {
for (const a of TRICKY_STRINGS) {
expect(checkQuery(PairKey.select({where: PairKey.where.eq('a')}).bind({a}), pairRows).kind).toBe('range');
}
});
it('ranges over the leading column when a comparison follows', () => {
const query = PairKey.select({where: [PairKey.where.eq('a'), PairKey.where.lt('b')]}).bind({
a: 'a\u0001b',
b: 10n,
});
expect(checkQuery(query, pairRows).kind).toBe('range');
});
it('emits one range per value when the leading column uses IN', () => {
const query = PairKey.select({where: PairKey.where.in('a', 'ids')}).bind({
ids: ['a', 'a\u0001b', 'a\u001fb', 'a'],
});
expect(checkQuery(query, pairRows).kind).toBe('ranges');
});
it('pins the full primary key across a cartesian product of IN lists', () => {
const query = PairKey.select({where: [PairKey.where.in('a', 'ids'), PairKey.where.in('b', 'bs')]}).bind({
ids: ['a', 'a\u007fb'],
bs: [9n, 10n],
});
expect(checkQuery(query, pairRows).kind).toBe('rowKeys');
});
it('ranges over a two-column prefix of a three-column primary key', () => {
for (const a of TRICKY_STRINGS) {
const query = TripleKey.select({where: [TripleKey.where.eq('a'), TripleKey.where.eq('b')]}).bind({a, b: 9n});
expect(checkQuery(query, tripleRows).kind).toBe('range');
}
});
it('falls back to a scan when the leading column is unpinned', () => {
const query = TripleKey.select({where: [TripleKey.where.eq('b'), TripleKey.where.eq('c')]}).bind({b: 9n, c: true});
expect(checkQuery(query, tripleRows).kind).toBe('scan');
});
it('pins every column of a six-column primary key', () => {
const query = WideKey.select({
where: [
WideKey.where.eq('a'),
WideKey.where.eq('b'),
WideKey.where.eq('c'),
WideKey.where.eq('d'),
WideKey.where.eq('e'),
WideKey.where.eq('f'),
],
}).bind({a: '\u{1f600}', b: 10n, c: false, d: null, e: Buffer.from([0x00, 0x1f, 0xff]), f: 2});
expect(checkQuery(query, wideRows).kind).toBe('rowKeys');
});
it('stops at the first gap in the primary key', () => {
const query = WideKey.select({
where: [WideKey.where.eq('a'), WideKey.where.eq('b'), WideKey.where.eq('f')],
}).bind({a: 'a', b: 9n, f: 1});
expect(checkQuery(query, wideRows).kind).toBe('range');
});
it('uses the partition key when it is not a prefix of the primary key', () => {
for (const b of TRICKY_STRINGS) {
const query = PartitionedKey.select({where: PartitionedKey.where.eq('b')}).bind({b});
expect(checkQuery(query, partitionedRows).kind).toBe('partitionKeys');
}
});
it('uses the partition key for an IN on the partition column', () => {
const query = PartitionedKey.select({where: PartitionedKey.where.in('b', 'bs')}).bind({
bs: new Set(['a', 'a\u0001b', 'a\\b']),
});
expect(checkQuery(query, partitionedRows).kind).toBe('partitionKeys');
});
it('scans a table with no predicate', () => {
expect(checkQuery(SingleKey.select().bind({}), singleRows).kind).toBe('scan');
});
});
describe('PostgresKvQueryExecutor plan exactness', () => {
it('is exact when the key predicate consumes every clause', () => {
expect(planOf(PairKey.select({where: PairKey.where.eq('a')}).bind({a: 'a'})).plan.exact).toBe(true);
expect(planOf(SingleKey.select().bind({})).plan.exact).toBe(true);
expect(planOf(PartitionedKey.select({where: PartitionedKey.where.eq('b')}).bind({b: 'a'})).plan.exact).toBe(true);
});
it('is inexact when a clause is left for the JavaScript filter', () => {
const bounded = PairKey.select({where: [PairKey.where.eq('a'), PairKey.where.lt('b')]}).bind({a: 'a', b: 9n});
expect(planOf(bounded).plan.exact).toBe(false);
expect(planOf(TripleKey.select({where: TripleKey.where.eq('b')}).bind({b: 9n})).plan.exact).toBe(false);
});
it('is inexact when a limit or an order is present', () => {
expect(planOf(PairKey.select({where: PairKey.where.eq('a'), limit: 5}).bind({a: 'a'})).plan.exact).toBe(false);
const ordered = PairKey.select({where: PairKey.where.eq('a'), orderBy: {col: 'b'}}).bind({a: 'a'});
expect(planOf(ordered).plan.exact).toBe(false);
});
it('is inexact when a pinned value has no stable key encoding', () => {
expect(planOf(SingleKey.select({where: SingleKey.where.eq('a')}).bind({a: {}})).plan.exact).toBe(false);
expect(planOf(SingleKey.select({where: SingleKey.where.eq('a')}).bind({a: Number.NaN})).plan.exact).toBe(false);
});
it('is exact for an empty IN list regardless of the other clauses', () => {
const query = PairKey.select({where: [PairKey.where.in('a', 'ids'), PairKey.where.lt('b')], limit: 3}).bind({
ids: [],
b: 9n,
});
expect(planOf(query).plan.exact).toBe(true);
});
});
describe('PostgresKvQueryExecutor plan totality', () => {
it('leaves a column unpinned when its value has no key encoding at all', () => {
const invalid = PairKey.select({where: PairKey.where.eq('a')}).bind({a: new Date('nope')});
const {plan} = planOf(invalid);
expect(plan.candidates.kind).toBe('scan');
expect(plan.exact).toBe(false);
});
it('leaves a trailing column unpinned without losing the leading prefix', () => {
const query = PairKey.select({where: [PairKey.where.eq('a'), PairKey.where.eq('b')]}).bind({
a: 'a',
b: new Date('nope') as never,
});
const {plan} = planOf(query);
expect(plan.candidates.kind).toBe('range');
expect(plan.exact).toBe(false);
});
it('does not treat a scalar bound to an IN parameter as an iterable', () => {
const query = SingleKey.select({where: SingleKey.where.in('a', 'ids')}).bind({ids: 'ab' as never});
const {plan} = planOf(query);
expect(plan.candidates.kind).toBe('scan');
expect(plan.exact).toBe(false);
});
});
describe('PostgresKvQueryExecutor plan size caps', () => {
it('keeps a full primary key product as row keys rather than demoting it to ranges', () => {
const ids = Array.from({length: 30}, (_, i) => `id${i}`);
const seqs = Array.from({length: 30}, (_, i) => BigInt(i));
const query = PairKey.select({where: [PairKey.where.in('a', 'ids'), PairKey.where.in('b', 'seqs')]}).bind({
ids,
seqs,
});
const {plan} = planOf(query);
expect(plan.candidates.kind).toBe('rowKeys');
expect(plan.exact).toBe(true);
if (plan.candidates.kind !== 'rowKeys') return;
expect(plan.candidates.rowKeys.length).toBe(900);
});
it('caps the number of prefix ranges and degrades to a scan instead', () => {
const under = TripleKey.select({where: TripleKey.where.in('a', 'ids')}).bind({
ids: Array.from({length: 256}, (_, i) => `id${i}`),
});
expect(planOf(under).plan.candidates.kind).toBe('ranges');
const over = TripleKey.select({where: TripleKey.where.in('a', 'ids')}).bind({
ids: Array.from({length: 257}, (_, i) => `id${i}`),
});
const {plan} = planOf(over);
expect(plan.candidates.kind).toBe('scan');
expect(plan.exact).toBe(false);
});
it('enforces the cap even when the trailing pinned column is an equality', () => {
const ids = Array.from({length: 40}, (_, i) => `id${i}`);
const query = WideKey.select({
where: [WideKey.where.in('a', 'ids'), WideKey.where.in('b', 'seqs'), WideKey.where.eq('c')],
}).bind({ids, seqs: Array.from({length: 40}, (_, i) => BigInt(i)), c: true});
const {plan} = planOf(query);
expect(plan.candidates.kind).toBe('ranges');
expect(plan.exact).toBe(false);
if (plan.candidates.kind !== 'ranges') return;
expect(plan.candidates.lowerBounds.length).toBe(40);
});
});
describe('PostgresKvQueryExecutor prefix bounds', () => {
it('brackets exactly the rows sharing the pinned prefix', () => {
const {plan} = planOf(PairKey.select({where: PairKey.where.eq('a')}).bind({a: 'a'}));
expect(plan.candidates.kind).toBe('range');
if (plan.candidates.kind !== 'range') return;
expect(plan.candidates.lowerBound).toBe(`"a"${SEPARATOR}`);
expect(plan.candidates.upperBound).toBe('"a" ');
expect(byteCompare(plan.candidates.lowerBound, plan.candidates.upperBound)).toBeLessThan(0);
});
});
describe('PostgresKvQueryExecutor key equality', () => {
const eqWhere = [{kind: 'eq', col: 'a', param: 'a'}] as never;
const corpus: Array<[string, unknown]> = [
['null', null],
['undefined', undefined],
['empty-string', ''],
['date-string', '2020-01-01'],
['days-string', '18262'],
['zero', 0],
['neg-zero', -0],
['number', 18262],
['bigint', 18262n],
['true', true],
['false', false],
['date', new Date('2020-01-01T00:00:00.000Z')],
['buffer', Buffer.from('2020-01-01')],
['local-date', cassandra.types.LocalDate.fromString('2020-01-01')],
['other-local-date', cassandra.types.LocalDate.fromString('2021-06-05')],
];
it('accepts an equality exactly when both sides share a row key', () => {
const deltas: Array<string> = [];
for (const [storedName, stored] of corpus) {
for (const [paramName, param] of corpus) {
const accepted = matchesWhere({a: stored}, eqWhere, {a: param} as CassandraParams);
const sameKey = keyFromColumns(['a'], {a: stored}) === keyFromColumns(['a'], {a: param});
if (accepted !== sameKey)
deltas.push(`stored=${storedName} param=${paramName} match=${accepted} key=${sameKey}`);
}
}
expect(deltas).toEqual([]);
});
it('does not equate a LocalDate with its string rendering', () => {
const localDate = cassandra.types.LocalDate.fromString('2020-01-01');
expect(matchesWhere({a: localDate}, eqWhere, {a: localDate} as CassandraParams)).toBe(true);
expect(matchesWhere({a: localDate}, eqWhere, {a: '2020-01-01'} as CassandraParams)).toBe(false);
expect(matchesWhere({a: '2020-01-01'}, eqWhere, {a: localDate} as CassandraParams)).toBe(false);
});
});
@@ -0,0 +1,484 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {execFileSync, spawnSync} from 'node:child_process';
import {createServer} from 'node:net';
import {
getDefaultPostgresClient,
type IPostgresClient,
initPostgres,
type PostgresQueryable,
shutdownPostgres,
} from '@pkgs/postgres/src/Client';
import {afterAll, beforeAll, describe, expect, it} from 'vitest';
import {GuildMembers, ReadStates, Users} from '../Tables';
import {LegacyPostgresKvQueryExecutor, legacyEnsurePostgresKvSchema} from './__testref__/LegacyPostgresKvQueryExecutor';
import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';
type Row = Record<string, unknown>;
const POSTGRES_IMAGE = 'postgres:16-alpine';
const CONTAINER = `fluxer-kvupgrade-${process.pid.toString(36)}-${Date.now().toString(36)}`;
const KV = 'kv_upgrade';
const dockerUp = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
let PORT = 0;
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function freePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (typeof address === 'string' || address === null) {
reject(new Error('no port'));
return;
}
const port = address.port;
server.close(() => resolve(port));
});
});
}
if (dockerUp) {
beforeAll(async () => {
PORT = await freePort();
execFileSync(
'docker',
[
'run',
'-d',
'--name',
CONTAINER,
'-e',
'POSTGRES_USER=fluxer',
'-e',
'POSTGRES_PASSWORD=fluxer',
'-e',
'POSTGRES_DB=fluxer',
'-p',
`127.0.0.1:${PORT}:5432`,
POSTGRES_IMAGE,
'-c',
'fsync=off',
'-c',
'synchronous_commit=off',
],
{stdio: 'ignore'},
);
for (let attempt = 0; attempt < 180; attempt += 1) {
await sleep(500);
const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
stdio: 'ignore',
});
if (probe.status !== 0) continue;
try {
await initPostgres({url: `postgres://fluxer:[email protected]:${PORT}/fluxer`, maxConnections: 8});
await getDefaultPostgresClient().query('SELECT 1');
return;
} catch {
await shutdownPostgres().catch(() => {});
}
}
throw new Error('postgres not ready');
}, 900_000);
afterAll(async () => {
await shutdownPostgres().catch(() => {});
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
});
}
class TableClient implements IPostgresClient {
constructor(
private readonly inner: IPostgresClient,
private readonly name: string,
) {}
query: IPostgresClient['query'] = (text, values) => this.inner.query(text, values);
async connect() {
await this.inner.connect();
}
async shutdown() {}
isConnected() {
return this.inner.isConnected();
}
async transaction<T>(fn: (c: PostgresQueryable) => Promise<T>) {
return this.inner.transaction(fn);
}
kvTable() {
return this.name;
}
}
function fp(rows: ReadonlyArray<Row>): string {
return rows
.map((r) =>
Object.entries(r)
.map(([k, v]) => `${k}=${String(v)}`)
.join('|'),
)
.join(';');
}
function pq(cql: string, params: Record<string, unknown>) {
return {cql, params} as never;
}
const suite = dockerUp ? describe : describe.skip;
suite('postgres kv upgrade safety', () => {
let raw: IPostgresClient;
let client: TableClient;
let legacy: LegacyPostgresKvQueryExecutor;
let next: PostgresKvQueryExecutor;
const usersScan = () => pq(Users.selectCql(), {});
beforeAll(async () => {
await initPostgres({url: `postgres://fluxer:[email protected]:${PORT}/fluxer`, maxConnections: 8});
raw = getDefaultPostgresClient();
await raw.query(`DROP TABLE IF EXISTS ${KV}`);
client = new TableClient(raw, KV);
await legacyEnsurePostgresKvSchema(client);
legacy = new LegacyPostgresKvQueryExecutor(client);
next = new PostgresKvQueryExecutor(client);
for (let i = 0; i < 40; i += 1) {
await legacy.executeQuery(
Users.upsertAll({user_id: BigInt(1000 + i), username: `u${i}`, discriminator: `${i}`} as never) as never,
);
}
for (let g = 0; g < 3; g += 1) {
for (let u = 0; u < 12; u += 1) {
await legacy.executeQuery(
GuildMembers.upsertAll({guild_id: BigInt(500 + g), user_id: BigInt(1000 + u)} as never) as never,
);
}
}
for (let u = 0; u < 5; u += 1) {
for (let c = 0; c < 4; c += 1) {
await legacy.executeQuery(
ReadStates.upsertAll({user_id: BigInt(1000 + u), channel_id: BigInt(7000 + c)} as never) as never,
);
}
}
}, 300_000);
afterAll(async () => {
await shutdownPostgres().catch(() => {});
});
it('reads data written by the old image identically', async () => {
const shapes = [
{name: 'users scan', q: usersScan()},
{
name: 'guild members by guild',
q: pq(GuildMembers.selectCql({where: GuildMembers.where.eq('guild_id')}), {guild_id: 501n}),
},
{
name: 'read states by user',
q: pq(ReadStates.selectCql({where: ReadStates.where.eq('user_id')}), {user_id: 1002n}),
},
{name: 'user by id', q: pq(Users.selectCql({where: Users.where.eq('user_id')}), {user_id: 1005n})},
];
for (const shape of shapes) {
const a = await legacy.executeQuery(shape.q);
const b = await next.executeQuery(shape.q);
expect(fp(b), shape.name).toBe(fp(a));
expect(a.length, shape.name).toBeGreaterThan(0);
}
});
async function drain(order: Array<'legacy' | 'next'>, pageSize: number) {
const keys: Array<string> = [];
let state: string | null = null;
let pages = 0;
try {
for (;;) {
const who = order[pages % order.length]!;
const exec = who === 'legacy' ? legacy : next;
const page: {rows: Array<Row>; pageState: string | null} = await exec.executePagedQuery(usersScan(), {
pageSize,
pageState: state,
});
pages += 1;
for (const row of page.rows) keys.push(String(row.user_id));
state = page.pageState;
if (!state) break;
if (pages > 200) throw new Error('runaway paging loop');
}
} catch (error) {
return {keys, error: (error as Error).message, pages};
}
return {keys, error: null, pages};
}
it('pages consistently with only the old image running', async () => {
const r = await drain(['legacy'], 7);
expect(r.error).toBeNull();
expect(r.keys.length).toBe(40);
expect(new Set(r.keys).size).toBe(40);
});
it('pages consistently with only the new image running', async () => {
const r = await drain(['next'], 7);
expect(r.error).toBeNull();
expect(r.keys.length).toBe(40);
expect(new Set(r.keys).size).toBe(40);
});
it('ROLLING RESTART: an old replica consumes a token minted by the new one', async () => {
const first = await next.executePagedQuery(usersScan(), {pageSize: 7, pageState: null});
expect(first.pageState).toBeTruthy();
const token = JSON.parse(Buffer.from(first.pageState!, 'base64url').toString('utf8'));
let failure: string | null = null;
let rows = 0;
try {
const second = await legacy.executePagedQuery(usersScan(), {pageSize: 7, pageState: first.pageState});
rows = second.rows.length;
} catch (error) {
failure = (error as Error).message;
}
console.log(`NEW->OLD token=${JSON.stringify(token)} legacy=${failure ?? `${rows} rows`}`);
expect(failure).toBeNull();
});
it('ROLLING RESTART: alternating old and new replicas drain the scan', async () => {
const a = await drain(['next', 'legacy'], 7);
const b = await drain(['legacy', 'next'], 7);
console.log(`new-first err=${a.error} pages=${a.pages} total=${a.keys.length} unique=${new Set(a.keys).size}`);
console.log(`old-first err=${b.error} pages=${b.pages} total=${b.keys.length} unique=${new Set(b.keys).size}`);
expect(a.error).toBeNull();
expect(new Set(a.keys).size).toBe(40);
expect(b.error).toBeNull();
expect(new Set(b.keys).size).toBe(40);
});
it('a dual {offset, after} token would survive both directions', async () => {
const first = await next.executePagedQuery(usersScan(), {pageSize: 7, pageState: null});
const cursor = JSON.parse(Buffer.from(first.pageState!, 'base64url').toString('utf8')) as {after: string};
const dual = Buffer.from(JSON.stringify({offset: first.rows.length, after: cursor.after})).toString('base64url');
const viaOld = await legacy.executePagedQuery(usersScan(), {pageSize: 7, pageState: dual});
const viaNew = await next.executePagedQuery(usersScan(), {pageSize: 7, pageState: dual});
console.log(`dual token -> old replica: ${viaOld.rows.length} rows, new replica: ${viaNew.rows.length} rows`);
expect(viaOld.rows.length).toBe(7);
expect(viaNew.rows.length).toBe(7);
});
it('deletes exactly the same rows as the old image', async () => {
const seed = async (exec: LegacyPostgresKvQueryExecutor | PostgresKvQueryExecutor, base: bigint) => {
for (let c = 0; c < 5; c += 1) {
await exec.executeQuery(ReadStates.upsertAll({user_id: base, channel_id: BigInt(9000 + c)} as never) as never);
}
};
await seed(legacy, 4001n);
await seed(legacy, 4002n);
await legacy.executeQuery(
ReadStates.delete({where: ReadStates.where.eq('user_id')}).bind({user_id: 4001n}) as never,
);
await next.executeQuery(ReadStates.delete({where: ReadStates.where.eq('user_id')}).bind({user_id: 4002n}) as never);
const left = await raw.query<{row_key: string}>(
`SELECT row_key FROM ${KV} WHERE table_name = 'read_states' AND row_key LIKE '%400%' ORDER BY row_key`,
);
console.log(`rows left after legacy delete + new delete: ${left.rows.length}`);
expect(left.rows.length).toBe(0);
});
it('writes byte-identical row_key and partition_key to the old image', async () => {
const OLD = `${KV}_w_old`;
const NEW = `${KV}_w_new`;
await raw.query(`DROP TABLE IF EXISTS ${OLD}`);
await raw.query(`DROP TABLE IF EXISTS ${NEW}`);
const oldClient = new TableClient(raw, OLD);
const newClient = new TableClient(raw, NEW);
await legacyEnsurePostgresKvSchema(oldClient);
await ensurePostgresKvSchema(newClient);
const oldExec = new LegacyPostgresKvQueryExecutor(oldClient);
const newExec = new PostgresKvQueryExecutor(newClient);
for (let i = 0; i < 30; i += 1) {
const row = {user_id: BigInt(9000 + i), channel_id: BigInt(500 + i)} as never;
await oldExec.executeQuery(ReadStates.upsertAll(row) as never);
await newExec.executeQuery(ReadStates.upsertAll(row) as never);
const g = {guild_id: BigInt(70 + i), user_id: BigInt(9000 + i)} as never;
await oldExec.executeQuery(GuildMembers.upsertAll(g) as never);
await newExec.executeQuery(GuildMembers.upsertAll(g) as never);
}
const diff = await raw.query<{n: string}>(`
SELECT count(*) AS n FROM (
(SELECT table_name, partition_key, row_key FROM ${OLD}
EXCEPT SELECT table_name, partition_key, row_key FROM ${NEW})
UNION ALL
(SELECT table_name, partition_key, row_key FROM ${NEW}
EXCEPT SELECT table_name, partition_key, row_key FROM ${OLD})
) d`);
const schemaDiff = await raw.query<{n: string}>(`
SELECT count(*) AS n FROM (
(SELECT replace(indexdef, '${OLD}', 'KV') FROM pg_indexes WHERE tablename = '${OLD}'
EXCEPT SELECT replace(indexdef, '${NEW}', 'KV') FROM pg_indexes WHERE tablename = '${NEW}')
UNION ALL
(SELECT replace(indexdef, '${NEW}', 'KV') FROM pg_indexes WHERE tablename = '${NEW}'
EXCEPT SELECT replace(indexdef, '${OLD}', 'KV') FROM pg_indexes WHERE tablename = '${OLD}')
) d`);
console.log(`key diffs=${diff.rows[0]!.n} index-definition diffs=${schemaDiff.rows[0]!.n}`);
expect(Number(diff.rows[0]!.n)).toBe(0);
expect(Number(schemaDiff.rows[0]!.n)).toBe(0);
});
it('survives three simultaneous boots (two api replicas and a worker)', async () => {
const BOOT = `${KV}_boot`;
await raw.query(`DROP TABLE IF EXISTS ${BOOT}`);
const results = await Promise.allSettled([
ensurePostgresKvSchema(new TableClient(raw, BOOT)),
ensurePostgresKvSchema(new TableClient(raw, BOOT)),
ensurePostgresKvSchema(new TableClient(raw, BOOT)),
]);
const failures = results.filter((r) => r.status === 'rejected');
console.log(`concurrent boots rejected: ${failures.length}`);
for (const f of failures) console.log(` ${(f as PromiseRejectedResult).reason}`);
expect(failures.length).toBe(0);
}, 120_000);
it('backfills the messages partition key once and then skips the update', async () => {
const BACKFILL = 'kv_backfill';
const legacyKey = `"c"${String.fromCharCode(31)}"b"${String.fromCharCode(31)}"m"`;
const backfillClient = new TableClient(raw, BACKFILL);
await raw.query(`DROP TABLE IF EXISTS ${BACKFILL}`);
await ensurePostgresKvSchema(backfillClient);
await raw.query(
`INSERT INTO ${BACKFILL} (table_name, partition_key, row_key, row_data) VALUES ('messages', $1, $1, '{}'::jsonb)`,
[legacyKey],
);
const pendingBefore = await raw.query(
`SELECT 1 FROM ${BACKFILL} WHERE table_name = 'messages' AND partition_key = row_key AND split_part(row_key, chr(31), 3) <> ''`,
);
expect(pendingBefore.rows.length).toBe(1);
await ensurePostgresKvSchema(backfillClient);
const after = await raw.query<{partition_key: string}>(`SELECT partition_key FROM ${BACKFILL}`);
expect(after.rows[0]?.partition_key).toBe(`"c"${String.fromCharCode(31)}"b"`);
const pendingAfter = await raw.query(
`SELECT 1 FROM ${BACKFILL} WHERE table_name = 'messages' AND partition_key = row_key AND split_part(row_key, chr(31), 3) <> ''`,
);
expect(pendingAfter.rows.length).toBe(0);
await ensurePostgresKvSchema(backfillClient);
const stable = await raw.query<{partition_key: string}>(`SELECT partition_key FROM ${BACKFILL}`);
expect(stable.rows[0]?.partition_key).toBe(`"c"${String.fromCharCode(31)}"b"`);
await raw.query(`DROP TABLE IF EXISTS ${BACKFILL}`);
}, 120_000);
it('boot does not repair an invalid C-collation index', async () => {
const q = pq(ReadStates.selectCql({where: ReadStates.where.eq('user_id')}), {user_id: 1002n});
const before = await next.executeQuery(q);
await raw.query(`UPDATE pg_index SET indisvalid = false WHERE indexrelid = '${KV}_row_key_c_idx'::regclass`);
await ensurePostgresKvSchema(client);
const check = await raw.query<{indisvalid: boolean}>(
`SELECT indisvalid FROM pg_index WHERE indexrelid = '${KV}_row_key_c_idx'::regclass`,
);
const after = await next.executeQuery(q);
await raw.query(`UPDATE pg_index SET indisvalid = true WHERE indexrelid = '${KV}_row_key_c_idx'::regclass`);
console.log(`index valid after boot: ${check.rows[0]?.indisvalid}`);
expect(String(after.map((r) => Object.values(r).map(String).join(',')))).toBe(
String(before.map((r) => Object.values(r).map(String).join(','))),
);
expect(check.rows[0]?.indisvalid).toBe(true);
});
describe('multi-range plan shape on a populated upgraded database', () => {
const PERF = 'kv_ranges_perf';
const SEP = String.fromCharCode(31);
const BASE = 1456074443984486400n;
let perf: IPostgresClient;
beforeAll(async () => {
const db = getDefaultPostgresClient();
await db.query(`DROP TABLE IF EXISTS ${PERF}`);
perf = new TableClient(db, PERF);
await ensurePostgresKvSchema(perf);
await db.query(
`INSERT INTO ${PERF} (table_name, partition_key, row_key, row_data)
SELECT 'push_subscriptions',
format('{"__fluxer_type":"bigint","value":"%s"}%s{"__fluxer_type":"bigint","value":"%s"}', u, chr(31), s),
format('{"__fluxer_type":"bigint","value":"%s"}%s{"__fluxer_type":"bigint","value":"%s"}', u, chr(31), s),
jsonb_build_object(
'user_id', jsonb_build_object('__fluxer_type', 'bigint', 'value', u::text),
'subscription_id', jsonb_build_object('__fluxer_type', 'bigint', 'value', s::text),
'endpoint', repeat('e', 300))
FROM generate_series($1::bigint, $1::bigint + 19999) u, generate_series(1, 3) s`,
[BASE.toString()],
);
await db.query(`ANALYZE ${PERF}`);
}, 300_000);
it('emits a multi-range plan that is not slower than the tier-3 scan it replaces', async () => {
const {PushSubscriptions} = await import('../Tables');
const {buildCandidatePlan, planFragments} = await import('./PostgresKvQueryExecutor');
const {getKvMeta} = await import('./CassandraMetaRegistry');
const db = getDefaultPostgresClient();
const explain = async (sql: string, params: Array<unknown>) => {
const r = await db.query<Record<string, string>>(`EXPLAIN (ANALYZE, BUFFERS) ${sql}`, params);
const plan = r.rows.map((row) => row['QUERY PLAN']!).join('\n');
return {plan, ms: Number(/Execution Time: ([\d.]+)/.exec(plan)?.[1] ?? 'NaN')};
};
const cql = PushSubscriptions.selectCql({where: PushSubscriptions.where.in('user_id', 'user_ids')});
const meta = getKvMeta(cql)!;
const scan = await explain(
`SELECT kv.row_key, kv.row_data FROM ${PERF} kv WHERE kv.table_name = $1 AND (kv.expires_at IS NULL OR kv.expires_at > now())`,
['push_subscriptions'],
);
const deltas: Array<string> = [];
for (const size of [5, 10, 25, 50, 100]) {
const params = {user_ids: Array.from({length: size}, (_, i) => BASE + BigInt(i * 3))};
const plan = buildCandidatePlan(meta, params as never);
expect(plan.candidates.kind).toBe('ranges');
const fragments = planFragments(plan.candidates);
expect(fragments.predicate).not.toContain('unnest');
const pushed = await explain(
`SELECT kv.row_key, kv.row_data FROM ${PERF} kv WHERE kv.table_name = $1${fragments.predicate} AND (kv.expires_at IS NULL OR kv.expires_at > now())`,
['push_subscriptions', ...fragments.params],
);
const verdict = `n=${size} pushdown=${pushed.ms.toFixed(2)}ms scan=${scan.ms.toFixed(2)}ms`;
console.log(verdict);
if (pushed.ms > scan.ms) deltas.push(`${verdict}\n${pushed.plan}`);
}
expect(deltas.join('\n')).toBe('');
const overCap = buildCandidatePlan(meta, {
user_ids: Array.from({length: 257}, (_, i) => BASE + BigInt(i * 3)),
} as never);
expect(overCap.candidates.kind).toBe('scan');
expect(SEP).toBe('\u001f');
}, 300_000);
});
});
describe('candidate plan shapes reached by real queries', () => {
it('classifies the push fan-out and webhook lookups', async () => {
const {PushSubscriptions, Webhooks} = await import('../Tables');
const {buildCandidatePlan} = await import('./PostgresKvQueryExecutor');
const {getKvMeta} = await import('./CassandraMetaRegistry');
const cases = [
{
name: 'push_subscriptions IN(user_id) x100',
cql: PushSubscriptions.selectCql({where: PushSubscriptions.where.in('user_id', 'user_ids')}),
params: {user_ids: Array.from({length: 100}, (_, i) => BigInt(1000 + i))},
},
{
name: 'webhooks IN(webhook_id) x100',
cql: Webhooks.selectCql({where: Webhooks.where.in('webhook_id', 'webhook_ids')}),
params: {webhook_ids: Array.from({length: 100}, (_, i) => BigInt(1000 + i))},
},
];
for (const c of cases) {
const meta = getKvMeta(c.cql)!;
const plan = buildCandidatePlan(meta, c.params as never);
const size =
plan.candidates.kind === 'ranges'
? plan.candidates.lowerBounds.length
: plan.candidates.kind === 'rowKeys'
? plan.candidates.rowKeys.length
: 0;
console.log(`${c.name} -> kind=${plan.candidates.kind} n=${size} exact=${plan.exact}`);
}
expect(true).toBe(true);
});
});
@@ -0,0 +1,538 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {type IPostgresClient, type PostgresQueryable, quoteIdentifier} from '@pkgs/postgres/src/Client';
import cassandra from 'cassandra-driver';
import {getKvMeta, getTableMetadata} from '../CassandraMetaRegistry';
import type {CassandraParams, ColumnName, KvQueryMeta, PreparedQuery, WhereExpr} from '../CassandraTypes';
import {matchesWhere} from '../PostgresKvQueryExecutor';
type Row = Record<string, unknown>;
type EqWhereExpr = Extract<WhereExpr<Row>, {kind: 'eq'}>;
interface StoredRow {
row_key: string;
row_data: unknown;
}
interface PageState {
offset: number;
}
const VALUE_SEPARATOR = '\u001f';
const ENCODED_TYPE_KEY = '__fluxer_type';
const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE = 0x46584b56;
const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT = '120s';
function normalizeCql(cql: string): string {
return cql.replace(/\s+/g, ' ').trim();
}
function isPlainObject(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === 'object' && Object.getPrototypeOf(value) === Object.prototype;
}
function encodeValue(value: unknown): unknown {
if (value === null || value === undefined) return null;
if (typeof value === 'bigint') return {[ENCODED_TYPE_KEY]: 'bigint', value: value.toString()};
if (value instanceof Date) return {[ENCODED_TYPE_KEY]: 'date', value: value.toISOString()};
if (Buffer.isBuffer(value)) return {[ENCODED_TYPE_KEY]: 'buffer', value: value.toString('base64')};
if (value instanceof Set) return {[ENCODED_TYPE_KEY]: 'set', value: [...value.values()].map(encodeValue)};
if (value instanceof Map) {
return {
[ENCODED_TYPE_KEY]: 'map',
value: [...value.entries()].map(([key, entry]) => [encodeValue(key), encodeValue(entry)]),
};
}
if (typeof value === 'object' && value.constructor?.name === 'LocalDate') {
return {[ENCODED_TYPE_KEY]: 'local_date', value: value.toString()};
}
if (Array.isArray(value)) return value.map(encodeValue);
if (isPlainObject(value)) {
const encoded: Record<string, unknown> = {};
for (const [key, entry] of Object.entries(value)) {
encoded[key] = encodeValue(entry);
}
return encoded;
}
return value;
}
function decodeValue(value: unknown): unknown {
if (Array.isArray(value)) return value.map(decodeValue);
if (!isPlainObject(value)) return value;
const encodedType = value[ENCODED_TYPE_KEY];
if (encodedType === 'bigint') return BigInt(String(value.value));
if (encodedType === 'date') return new Date(String(value.value));
if (encodedType === 'buffer') return Buffer.from(String(value.value), 'base64');
if (encodedType === 'set') return new Set(((value.value as Array<unknown>) ?? []).map(decodeValue));
if (encodedType === 'map') {
return new Map(
((value.value as Array<[unknown, unknown]>) ?? []).map(([key, entry]) => [decodeValue(key), decodeValue(entry)]),
);
}
if (encodedType === 'local_date') return cassandra.types.LocalDate.fromString(String(value.value));
const decoded: Record<string, unknown> = {};
for (const [key, entry] of Object.entries(value)) {
decoded[key] = decodeValue(entry);
}
return decoded;
}
function encodeRow(row: Row): Record<string, unknown> {
const encoded: Record<string, unknown> = {};
for (const [key, value] of Object.entries(row)) {
encoded[key] = encodeValue(value);
}
return encoded;
}
function decodeRow(value: unknown): Row {
const decoded = decodeValue(value);
if (!isPlainObject(decoded)) {
throw new Error('Postgres KV row payload is not an object');
}
return decoded;
}
function valueKey(value: unknown): string {
return JSON.stringify(encodeValue(value));
}
function keyFromColumns(columns: ReadonlyArray<string>, row: Row): string {
return columns.map((column) => valueKey(row[column])).join(VALUE_SEPARATOR);
}
function paramsRow(params: CassandraParams, columns: ReadonlyArray<string>): Row {
const row: Row = {};
for (const column of columns) {
if (!(column in params)) {
throw new Error(`Missing Postgres KV key parameter: ${column}`);
}
row[column] = params[column];
}
return row;
}
function rowFromParams(meta: KvQueryMeta, params: CassandraParams): Row {
const row: Row = {};
for (const column of meta.table.columns) {
if (column in params) {
row[column] = params[column];
}
}
if (meta.nowColumn) {
row[meta.nowColumn] = new Date();
}
return row;
}
function rowKey(meta: KvQueryMeta, row: Row): string {
return keyFromColumns(meta.table.primaryKey as ReadonlyArray<string>, row);
}
function partitionKey(meta: KvQueryMeta, row: Row): string {
return keyFromColumns(meta.table.partitionKey as ReadonlyArray<string>, row);
}
function rowKeyFromParams(meta: KvQueryMeta, params: CassandraParams): string {
return rowKey(meta, paramsRow(params, (meta.pkColumns ?? meta.table.primaryKey) as ReadonlyArray<string>));
}
function partitionKeyFromParams(meta: KvQueryMeta, params: CassandraParams): string {
return partitionKey(meta, paramsRow(params, meta.table.partitionKey as ReadonlyArray<string>));
}
function compareValues(left: unknown, right: unknown): number {
if (typeof left === 'bigint' || typeof right === 'bigint') {
const l = typeof left === 'bigint' ? left : BigInt(left as number | string);
const r = typeof right === 'bigint' ? right : BigInt(right as number | string);
if (l === r) return 0;
return l < r ? -1 : 1;
}
const l = left instanceof Date ? left.getTime() : left?.constructor?.name === 'LocalDate' ? left.toString() : left;
const r =
right instanceof Date ? right.getTime() : right?.constructor?.name === 'LocalDate' ? right.toString() : right;
if (Buffer.isBuffer(l) && Buffer.isBuffer(r)) return Buffer.compare(l, r);
if (l === r) return 0;
return (l as number | string) < (r as number | string) ? -1 : 1;
}
function projectRow(row: Row, columns: ReadonlyArray<string> | undefined): Row {
if (!columns) return {...row};
const projected: Row = {};
for (const column of columns) {
projected[column] = row[column];
}
return projected;
}
function sortRows(meta: KvQueryMeta, rows: Array<Row>): Array<Row> {
if (meta.orderBy) {
const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
return rows.sort((left, right) => compareValues(left[meta.orderBy!.col], right[meta.orderBy!.col]) * direction);
}
return rows.sort((left, right) => {
for (const column of meta.table.primaryKey) {
const cmp = compareValues(left[column], right[column]);
if (cmp !== 0) return cmp;
}
return 0;
});
}
function equalityParam(where: ReadonlyArray<WhereExpr<Row>> | undefined, column: string): string | null {
const clause = (where ?? []).find((entry) => entry.kind === 'eq' && entry.col === column);
return clause && clause.kind === 'eq' ? clause.param : null;
}
function inParam(where: ReadonlyArray<WhereExpr<Row>> | undefined, column: string): string | null {
const clause = (where ?? []).find((entry) => entry.kind === 'in' && entry.col === column);
return clause && clause.kind === 'in' ? clause.param : null;
}
function fullRowKeysFromWhere(meta: KvQueryMeta, params: CassandraParams): Array<string> | null {
const pk = meta.table.primaryKey as ReadonlyArray<string>;
const eqParams = pk.map((column) => equalityParam(meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, column));
if (eqParams.every((param) => param !== null)) {
const row: Row = {};
for (let i = 0; i < pk.length; i += 1) row[pk[i]!] = params[eqParams[i]!];
return [rowKey(meta, row)];
}
if (pk.length === 1) {
const param = inParam(meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, pk[0]!);
if (param) {
const values = params[param] as ReadonlyArray<unknown> | Set<unknown>;
const haystack = values instanceof Set ? [...values] : values;
return haystack.map((value) => rowKey(meta, {[pk[0]!]: value}));
}
}
return null;
}
function hasFullPartition(meta: KvQueryMeta): boolean {
return meta.table.partitionKey.every((column) =>
equalityParam(meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, column),
);
}
function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null | undefined {
const ttlParam = meta.ttlParamName;
if (!ttlParam) return undefined;
const ttlRaw = params[ttlParam];
if (typeof ttlRaw !== 'number') {
throw new Error(`TTL parameter ${ttlParam} must be a number`);
}
return new Date(Date.now() + ttlRaw * 1000);
}
function encodePageState(pageState: PageState): string {
return Buffer.from(JSON.stringify(pageState)).toString('base64url');
}
function decodePageState(pageState: string | null | undefined): PageState {
if (!pageState) return {offset: 0};
const decoded = JSON.parse(Buffer.from(pageState, 'base64url').toString('utf8')) as PageState;
if (!Number.isInteger(decoded.offset) || decoded.offset < 0) {
throw new Error('Invalid Postgres KV page state');
}
return decoded;
}
function parseRawMeta(cql: string): KvQueryMeta<Row> | null {
const normalized = normalizeCql(cql).replace(/;$/, '');
const update =
/^UPDATE\s+([A-Za-z0-9_]+)(?:\s+USING\s+(?:TIMESTAMP|TTL)\s+:[A-Za-z0-9_]+)?\s+SET\s+(.+?)\s+WHERE\s+(.+)$/iu.exec(
normalized,
);
if (update) {
const table = tableSpec(update[1]!);
const patchKeys = update[2]!.split(',').map((part) => {
const [column, value] = part.trim().split(/\s*=\s*/u);
if (!column || !value?.startsWith(':')) {
throw new Error(`Postgres KV raw UPDATE only supports parameter assignments: ${cql}`);
}
return column;
});
const where = parseEqWhere(update[3]!, cql);
return {
action: 'patch',
table,
where,
patchKeys,
pkColumns: where.map((clause) => clause.col),
};
}
const select =
/^SELECT\s+(.+?)\s+FROM\s+([A-Za-z0-9_]+)(?:\s+WHERE\s+(.+?))?(?:\s+ALLOW\s+FILTERING)?(?:\s+LIMIT\s+(\d+))?$/iu.exec(
normalized,
);
if (select) {
const table = tableSpec(select[2]!);
return {
action: 'select',
table,
columns: select[1]!.split(',').map((part) => part.trim() as ColumnName<Row>),
where: select[3] ? parseEqWhere(select[3], cql) : [],
limit: select[4] ? Number.parseInt(select[4], 10) : undefined,
};
}
return null;
}
function tableSpec(tableName: string): KvQueryMeta<Row>['table'] {
const table = getTableMetadata(tableName);
if (!table) {
throw new Error(`Postgres KV metadata is missing for table: ${tableName}`);
}
return table;
}
function parseEqWhere(whereSql: string, cql: string): ReadonlyArray<EqWhereExpr> {
return whereSql.split(/\s+AND\s+/iu).map((part) => {
const match = /^\s*([A-Za-z0-9_]+)\s*=\s*:([A-Za-z0-9_]+)\s*$/u.exec(part.trim());
if (!match) {
throw new Error(`Postgres KV raw WHERE only supports equality predicates: ${cql}`);
}
return {kind: 'eq', col: match[1]! as ColumnName<Row>, param: match[2]!};
});
}
export async function legacyEnsurePostgresKvSchema(client: IPostgresClient): Promise<void> {
const kvTable = client.kvTable();
const table = quoteIdentifier(kvTable);
await client.transaction(async (db) => {
await db.query("SELECT set_config('statement_timeout', $1, true)", [POSTGRES_KV_SCHEMA_LOCK_TIMEOUT]);
await db.query('SELECT pg_advisory_xact_lock($1, hashtext($2))', [POSTGRES_KV_SCHEMA_LOCK_NAMESPACE, kvTable]);
await db.query("SELECT set_config('statement_timeout', '0', true)");
await db.query(`
CREATE TABLE IF NOT EXISTS ${table} (
table_name text NOT NULL,
partition_key text NOT NULL,
row_key text NOT NULL,
row_data jsonb NOT NULL,
expires_at timestamptz,
updated_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (table_name, row_key)
)`);
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_partition_row_idx`)} ON ${table} (table_name, partition_key, row_key)`,
);
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_row_key_c_idx`)} ON ${table} (table_name, row_key COLLATE "C")`,
);
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_expires_idx`)} ON ${table} (expires_at) WHERE expires_at IS NOT NULL`,
);
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_messages_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'messages'`,
);
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`,
);
await db.query(`
UPDATE ${table}
SET partition_key = split_part(row_key, chr(31), 1) || chr(31) || split_part(row_key, chr(31), 2)
WHERE table_name = 'messages'
AND partition_key = row_key
AND split_part(row_key, chr(31), 3) <> ''`);
await db.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${kvTable}_partition_idx`)}`);
});
}
export class LegacyPostgresKvQueryExecutor {
private readonly table: string;
constructor(private readonly client: IPostgresClient) {
this.table = quoteIdentifier(client.kvTable());
}
async executeQuery<T = Row, P extends CassandraParams = CassandraParams>(
query: PreparedQuery<P>,
db: PostgresQueryable = this.client,
): Promise<Array<T>> {
const meta = this.meta(query);
switch (meta.action) {
case 'select':
return (await this.select(meta, query.params, db)) as Array<T>;
case 'count':
return [{count: (await this.select(meta, query.params, db)).length}] as Array<T>;
case 'upsert':
return (await this.upsert(meta, query.params, db)) as Array<T>;
case 'insert':
return (await this.upsert(meta, query.params, db)) as Array<T>;
case 'patch':
await this.patch(meta, query.params, db);
return [];
case 'delete':
await this.delete(meta, query.params, db);
return [];
case 'batch':
return [];
default: {
const _exhaustive: never = meta.action;
throw new Error(`Unsupported Postgres KV action: ${_exhaustive}`);
}
}
}
async executePagedQuery<T = Row, P extends CassandraParams = CassandraParams>(
query: PreparedQuery<P>,
options: {pageSize: number; pageState?: string | null},
): Promise<{rows: Array<T>; pageState: string | null}> {
const state = decodePageState(options.pageState);
const rows = await this.executeQuery<T, P>(query);
const pageRows = rows.slice(state.offset, state.offset + options.pageSize);
const nextOffset = state.offset + pageRows.length;
return {
rows: pageRows,
pageState: nextOffset < rows.length ? encodePageState({offset: nextOffset}) : null,
};
}
async executeBatch(
queries: Array<{query: string; params: object; meta?: KvQueryMeta}>,
atomic = true,
): Promise<void> {
if (atomic) {
await this.client.transaction(async (db) => {
for (const query of queries) {
await this.executeQuery({cql: query.query, params: query.params as CassandraParams, kvMeta: query.meta}, db);
}
});
return;
}
for (const query of queries) {
await this.executeQuery({cql: query.query, params: query.params as CassandraParams, kvMeta: query.meta});
}
}
private meta(query: PreparedQuery): KvQueryMeta<Row> {
const meta = (query.kvMeta ?? getKvMeta(query.cql) ?? parseRawMeta(query.cql)) as KvQueryMeta<Row> | null;
if (!meta) {
throw new Error(`Postgres KV does not understand query: ${query.cql}`);
}
return meta;
}
private async candidates(
meta: KvQueryMeta,
params: CassandraParams,
db: PostgresQueryable,
): Promise<Array<StoredRow>> {
const rowKeys = fullRowKeysFromWhere(meta, params);
if (rowKeys) {
const result = await db.query<StoredRow>(
`SELECT row_key, row_data FROM ${this.table} WHERE table_name = $1 AND row_key = ANY($2::text[]) AND (expires_at IS NULL OR expires_at > now())`,
[meta.table.name, rowKeys],
);
return result.rows;
}
if (hasFullPartition(meta)) {
const result = await db.query<StoredRow>(
`SELECT row_key, row_data FROM ${this.table} WHERE table_name = $1 AND partition_key = $2 AND (expires_at IS NULL OR expires_at > now())`,
[meta.table.name, partitionKeyFromParams(meta, params)],
);
return result.rows;
}
const result = await db.query<StoredRow>(
`SELECT row_key, row_data FROM ${this.table} WHERE table_name = $1 AND (expires_at IS NULL OR expires_at > now())`,
[meta.table.name],
);
return result.rows;
}
private async select(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<Array<Row>> {
let rows = (await this.candidates(meta, params, db))
.map((stored) => decodeRow(stored.row_data))
.filter((row) => matchesWhere(row, meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, params));
rows = sortRows(meta, rows);
if (typeof meta.limit === 'number') rows = rows.slice(0, meta.limit);
return rows.map((row) => projectRow(row, meta.columns as ReadonlyArray<string> | undefined));
}
private async upsert(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<Array<Row>> {
const incoming = rowFromParams(meta, params);
const key = rowKey(meta, incoming);
const existing = await this.getRow(meta, key, db);
if (meta.ifNotExists && existing) {
return [{'[applied]': false}];
}
if (meta.ifNotExists) {
await db.query(
`DELETE FROM ${this.table} WHERE table_name = $1 AND row_key = $2 AND expires_at IS NOT NULL AND expires_at <= now()`,
[meta.table.name, key],
);
}
const next = {...(existing ?? {}), ...incoming};
const expiresAt = ttlExpiresAt(meta, params) ?? null;
const result = await db.query(
`INSERT INTO ${this.table} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
VALUES ($1, $2, $3, $4::jsonb, $5, now())
ON CONFLICT (table_name, row_key)
DO UPDATE SET partition_key = EXCLUDED.partition_key, row_data = EXCLUDED.row_data, expires_at = EXCLUDED.expires_at, updated_at = now()
WHERE NOT $6`,
[
meta.table.name,
partitionKey(meta, next),
key,
JSON.stringify(encodeRow(next)),
expiresAt,
meta.ifNotExists === true,
],
);
if (meta.ifNotExists) {
return [{'[applied]': result.rowCount === 1}];
}
return [];
}
private async patch(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<void> {
const key = rowKeyFromParams(meta, params);
const stored = await this.getStoredRow(meta, key, db);
const base = stored?.row ?? paramsRow(params, (meta.pkColumns ?? meta.table.primaryKey) as ReadonlyArray<string>);
const next = {...base};
for (const column of meta.patchKeys ?? []) {
next[column] = column in params ? params[column] : null;
}
const ttl = ttlExpiresAt(meta, params);
const expiresAt = ttl === undefined ? (stored?.expiresAt ?? null) : ttl;
await db.query(
`INSERT INTO ${this.table} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
VALUES ($1, $2, $3, $4::jsonb, $5, now())
ON CONFLICT (table_name, row_key)
DO UPDATE SET partition_key = EXCLUDED.partition_key, row_data = EXCLUDED.row_data, expires_at = EXCLUDED.expires_at, updated_at = now()`,
[meta.table.name, partitionKey(meta, next), key, JSON.stringify(encodeRow(next)), expiresAt ?? null],
);
}
private async delete(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<void> {
const rows = await this.candidates(meta, params, db);
const matchingKeys = rows
.filter((stored) =>
matchesWhere(decodeRow(stored.row_data), meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, params),
)
.map((stored) => stored.row_key);
if (matchingKeys.length === 0) return;
await db.query(`DELETE FROM ${this.table} WHERE table_name = $1 AND row_key = ANY($2::text[])`, [
meta.table.name,
matchingKeys,
]);
}
private async getStoredRow(
meta: KvQueryMeta,
key: string,
db: PostgresQueryable,
): Promise<{row: Row; expiresAt: Date | null} | null> {
const result = await db.query<StoredRow & {expires_at: Date | null}>(
`SELECT row_key, row_data, expires_at FROM ${this.table} WHERE table_name = $1 AND row_key = $2 AND (expires_at IS NULL OR expires_at > now()) LIMIT 1`,
[meta.table.name, key],
);
const row = result.rows[0];
return row ? {row: decodeRow(row.row_data), expiresAt: row.expires_at ?? null} : null;
}
private async getRow(meta: KvQueryMeta, key: string, db: PostgresQueryable): Promise<Row | null> {
return (await this.getStoredRow(meta, key, db))?.row ?? null;
}
}
@@ -2,6 +2,7 @@
import type {MemeID, UserID} from '../BrandedTypes';
import {BatchBuilder, fetchMany, fetchOne, upsertOne} from '../database/CassandraQueryExecution';
import {Db} from '../database/CassandraTypes';
import type {FavoriteMemeRow} from '../database/types/UserTypes';
import {FavoriteMeme} from '../models/FavoriteMeme';
import {FavoriteMemes, FavoriteMemesByMemeId} from '../Tables';
@@ -99,6 +100,10 @@ export class FavoriteMemeRepository extends IFavoriteMemeRepository {
return new FavoriteMeme(memeRow);
}
async updatePlaceholder(userId: UserID, memeId: MemeID, placeholder: string): Promise<void> {
await fetchOne(FavoriteMemes.patchByPk({user_id: userId, meme_id: memeId}, {placeholder: Db.set(placeholder)}));
}
async delete(userId: UserID, memeId: MemeID): Promise<void> {
const batch = new BatchBuilder();
batch.addPrepared(FavoriteMemes.deleteByPk({user_id: userId, meme_id: memeId}));
@@ -34,6 +34,8 @@ export abstract class IFavoriteMemeRepository {
abstract update(userId: UserID, memeId: MemeID, data: CreateFavoriteMemeParams): Promise<FavoriteMeme>;
abstract updatePlaceholder(userId: UserID, memeId: MemeID, placeholder: string): Promise<void>;
abstract delete(userId: UserID, memeId: MemeID): Promise<void>;
abstract deleteAllByUserId(userId: UserID): Promise<void>;
@@ -262,7 +262,7 @@ export function GuildMemberController(app: HonoApp) {
const userId = ctx.get('user').id;
const targetId = createUserID(user_id);
const guildId = createGuildID(guild_id);
const {delete_message_days, reason, ban_duration_seconds} = ctx.req.valid('json');
const {delete_message_days, delete_message_seconds, reason, ban_duration_seconds} = ctx.req.valid('json');
const auditLogReason = ctx.get('auditLogReason') ?? null;
const effectiveReason = reason ?? auditLogReason ?? undefined;
await ctx.get('guildService').moderation.banMember(
@@ -271,6 +271,7 @@ export function GuildMemberController(app: HonoApp) {
guildId,
targetId,
deleteMessageDays: delete_message_days,
deleteMessageSeconds: delete_message_seconds,
reason: effectiveReason,
banDurationSeconds: ban_duration_seconds,
},
@@ -30,6 +30,8 @@ import type {IGuildRepositoryAggregate} from '../repositories/IGuildRepositoryAg
import {createGuildMfaEnforcer} from './GuildMfaEnforcement';
import {GuildMemberSearchIndexService} from './member/GuildMemberSearchIndexService';
const SECONDS_PER_DAY = 86_400;
export class GuildModerationService {
private readonly searchIndexService: GuildMemberSearchIndexService;
@@ -64,13 +66,23 @@ export class GuildModerationService {
targetId: UserID;
guildId: GuildID;
deleteMessageDays?: number;
deleteMessageSeconds?: number;
reason?: string | null;
banDurationSeconds?: number;
skipGuildAuditLog?: boolean;
},
auditLogReason?: string | null,
): Promise<void> {
const {userId, guildId, targetId, deleteMessageDays, reason, banDurationSeconds, skipGuildAuditLog} = params;
const {
userId,
guildId,
targetId,
deleteMessageDays,
deleteMessageSeconds,
reason,
banDurationSeconds,
skipGuildAuditLog,
} = params;
await this.checkModerationPermission({guildId, userId, permission: Permissions.BAN_MEMBERS});
if (userId === targetId) throw new UnknownGuildMemberError();
const targetUser = await this.userRepository.findUnique(targetId);
@@ -82,11 +94,13 @@ export class GuildModerationService {
const canManage = await this.gatewayService.checkTargetMember({guildId, userId, targetUserId: targetId});
if (!canManage) throw new MissingPermissionsError();
}
if (deleteMessageDays && deleteMessageDays > 0) {
const effectiveDeleteMessageSeconds =
deleteMessageSeconds ?? (deleteMessageDays !== undefined ? deleteMessageDays * SECONDS_PER_DAY : undefined);
if (effectiveDeleteMessageSeconds && effectiveDeleteMessageSeconds > 0) {
await this.workerService.addJob('deleteUserMessagesInGuildByTime', {
guildId: guildId.toString(),
userId: targetId.toString(),
days: deleteMessageDays,
seconds: effectiveDeleteMessageSeconds,
});
}
const targetIp = isIpBanExempt(targetUser.lastActiveIp) ? null : targetUser.lastActiveIp || null;
@@ -42,7 +42,7 @@ export class GuildMemberRoleService {
requestCache: RequestCache;
}): Promise<void> {
const {userId, targetId, guildId, roleId} = params;
const {guildData, canManageRoles} = await this.authService.getGuildAuthenticated({userId, guildId});
const {guildData, hasPermission, canManageRoles} = await this.authService.getGuildAuthenticated({userId, guildId});
const targetMember = await this.guildRepository.getMember(guildId, targetId);
if (!targetMember) throw new UnknownGuildMemberError();
await this.validationService.validateRoleAssignment({
@@ -51,6 +51,7 @@ export class GuildMemberRoleService {
userId,
targetId,
roleId,
hasPermission,
canManageRoles,
});
if (targetMember.roleIds.has(roleId)) return;
@@ -75,7 +76,7 @@ export class GuildMemberRoleService {
requestCache: RequestCache;
}): Promise<void> {
const {userId, targetId, guildId, roleId} = params;
const {guildData, canManageRoles} = await this.authService.getGuildAuthenticated({userId, guildId});
const {guildData, hasPermission, canManageRoles} = await this.authService.getGuildAuthenticated({userId, guildId});
const targetMember = await this.guildRepository.getMember(guildId, targetId);
if (!targetMember) throw new UnknownGuildMemberError();
await this.validationService.validateRoleAssignment({
@@ -84,6 +85,7 @@ export class GuildMemberRoleService {
userId,
targetId,
roleId,
hasPermission,
canManageRoles,
});
if (!targetMember.roleIds.has(roleId)) return;
@@ -74,9 +74,10 @@ export class GuildMemberValidationService {
userId: UserID;
targetId: UserID;
roleId: RoleID;
hasPermission: (permission: bigint) => Promise<boolean>;
canManageRoles: (targetUserId: UserID, targetRoleId: RoleID) => Promise<boolean>;
}): Promise<void> {
const {guildData, guildId, userId, targetId, roleId, canManageRoles} = params;
const {guildData, guildId, userId, targetId, roleId, hasPermission, canManageRoles} = params;
ensureNotEveryoneRole(roleId, guildId, 'role_id');
if (guildData && guildData.owner_id === userId.toString()) {
const role = await this.guildRepository.getRole(roleId, guildId);
@@ -84,6 +85,9 @@ export class GuildMemberValidationService {
throw new UnknownGuildRoleError();
}
} else {
if (!(await hasPermission(Permissions.MANAGE_ROLES))) {
throw new MissingPermissionsError();
}
if (!(await canManageRoles(targetId, roleId))) {
throw new MissingPermissionsError();
}
@@ -2,14 +2,13 @@
import type {WebhookEvent} from 'livekit-server-sdk';
import {TrackSource, WebhookReceiver} from 'livekit-server-sdk';
import type {ChannelID, GuildID, UserID} from '../BrandedTypes';
import type {ChannelID, GuildID} from '../BrandedTypes';
import {Config} from '../Config';
import {Logger} from '../Logger';
import type {LimitConfigService} from '../limits/LimitConfigService';
import {resolveLimitSafe} from '../limits/LimitConfigUtils';
import {createLimitMatchContext} from '../limits/LimitMatchContextBuilder';
import type {IUserRepository} from '../user/IUserRepository';
import type {VoicePresenceHeartbeatStore} from '../voice/VoicePresenceHeartbeatStore';
import type {VoiceTopology} from '../voice/VoiceTopology';
import type {IGatewayService} from './IGatewayService';
import type {ILiveKitService} from './ILiveKitService';
@@ -39,7 +38,6 @@ export class LiveKitWebhookService {
private liveKitService: ILiveKitService,
private voiceTopology: VoiceTopology,
private limitConfigService: LimitConfigService,
private voicePresenceHeartbeatStore: VoicePresenceHeartbeatStore | null = null,
) {
this.receivers = new Map();
this.serverMap = new Map();
@@ -140,43 +138,6 @@ export class LiveKitWebhookService {
);
}
private async markVoicePresenceHeartbeatEnded(params: {
channelId: ChannelID;
userId: UserID;
connectionId: string;
}): Promise<void> {
if (this.voicePresenceHeartbeatStore === null) {
return;
}
try {
await this.voicePresenceHeartbeatStore.markHeartbeatEnded(params);
} catch (error) {
Logger.warn(
{
error,
channelId: params.channelId.toString(),
userId: params.userId.toString(),
connectionId: params.connectionId,
},
'Failed to mark v2 voice presence heartbeat ended',
);
}
}
private async markChannelVoicePresenceHeartbeatsEnded(channelId: ChannelID): Promise<void> {
if (this.voicePresenceHeartbeatStore === null) {
return;
}
try {
await this.voicePresenceHeartbeatStore.markChannelHeartbeatsEnded(channelId);
} catch (error) {
Logger.warn(
{error, channelId: channelId.toString()},
'Failed to clear active v2 voice presence heartbeats for finished room',
);
}
}
async handleRoomFinished(event: WebhookEvent, apiKey: string): Promise<void> {
if (event.event !== 'room_finished' || !event.room) {
return;
@@ -210,7 +171,6 @@ export class LiveKitWebhookService {
);
return;
}
await this.markChannelVoicePresenceHeartbeatsEnded(context.channelId);
await this.voiceRoomStore.deleteRoomServer(undefined, context.channelId);
Logger.debug({channelId: context.channelId.toString()}, 'Cleared DM voice room server pinning');
} else {
@@ -227,7 +187,6 @@ export class LiveKitWebhookService {
);
return;
}
await this.markChannelVoicePresenceHeartbeatsEnded(context.channelId);
await this.voiceRoomStore.deleteRoomServer(context.guildId, context.channelId);
Logger.debug(
{guildId: context.guildId.toString(), channelId: context.channelId.toString()},
@@ -338,11 +297,6 @@ export class LiveKitWebhookService {
},
'LiveKit participant_joined rejected - disconnecting participant',
);
await this.markVoicePresenceHeartbeatEnded({
channelId: context.channelId,
userId: context.userId,
connectionId: context.connectionId,
});
try {
await this.liveKitService.disconnectParticipant({
guildId,
@@ -479,11 +433,6 @@ export class LiveKitWebhookService {
);
return;
}
await this.markVoicePresenceHeartbeatEnded({
channelId: context.channelId,
userId: context.userId,
connectionId: context.connectionId,
});
const guildId = context.type === 'guild' ? context.guildId : undefined;
Logger.info(
{
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {UserFlags} from '@fluxer/constants/src/UserConstants';
import {CaptchaRequiredError, InvalidCaptchaError} from '@fluxer/errors/src/CaptchaErrors';
import {extractClientIp} from '@fluxer/ip_utils/src/ClientIp';
import {createCaptchaProvider} from '@pkgs/captcha/src/CaptchaProviderFactory';
@@ -53,7 +54,9 @@ export async function verifyCaptchaToken(ctx: Context<HonoEnv>): Promise<void> {
if (!captchaConfig.enabled && !(Config.dev.testModeEnabled && Config.captcha.enabled)) return;
const user = ctx.get('user') as User | undefined;
if (accountPolicyContactHasCapability(user?.email, 'captcha_exempt')) return;
if (userHasCaptchaExemptFlag(user)) return;
if (await requestContactHasCaptchaExemption(ctx.req.raw)) return;
if (await requestUserHasCaptchaExemptFlag(ctx)) return;
const token = ctx.req.header('x-captcha-token');
if (!token) {
throw new CaptchaRequiredError();
@@ -83,6 +86,23 @@ async function requestContactHasCaptchaExemption(request: Request): Promise<bool
}
}
function userHasCaptchaExemptFlag(user: User | null | undefined): boolean {
return user != null && (user.flags & UserFlags.APP_STORE_REVIEWER) !== 0n;
}
async function requestUserHasCaptchaExemptFlag(ctx: Context<HonoEnv>): Promise<boolean> {
try {
const body = (await ctx.req.raw.clone().json()) as unknown;
if (!body || typeof body !== 'object' || Array.isArray(body)) return false;
const email = (body as Record<string, unknown>).email;
if (typeof email !== 'string') return false;
const user = await ctx.get('userRepository').findByEmail(email);
return userHasCaptchaExemptFlag(user);
} catch {
return false;
}
}
export const CaptchaMiddleware = createMiddleware<HonoEnv>(async (ctx, next) => {
await verifyCaptchaToken(ctx);
await next();
@@ -81,7 +81,6 @@ import {UserContentRequestService} from '../user/services/UserContentRequestServ
import {UserRelationshipRequestService} from '../user/services/UserRelationshipRequestService';
import {UserService} from '../user/services/UserService';
import {resolveRequestClientIp} from '../utils/IpUtils';
import {VoicePresenceHeartbeatStore} from '../voice/VoicePresenceHeartbeatStore';
import {VoiceService} from '../voice/VoiceService';
import {WebhookRequestService} from '../webhook/WebhookRequestService';
import {WebhookService} from '../webhook/WebhookService';
@@ -334,7 +333,6 @@ function getLiveKitWebhookService(): LiveKitWebhookService | null {
liveKitService,
voiceTopology,
getLimitConfigService(),
new VoicePresenceHeartbeatStore(getKVClient()),
);
}
}
+8 -647
View File
@@ -5468,410 +5468,6 @@
]
}
},
"/channels/{channel_id}/voice-debug-logging/events": {
"post": {
"operationId": "upload_voice_debug_logging_events",
"summary": "Upload voice debug logging events",
"tags": ["Channels"],
"responses": {
"200": {
"description": "Success",
"content": {
"application/json": {"schema": {"$ref": "#/components/schemas/VoiceDebugLoggingEventsResponse"}}
}
},
"400": {
"description": "Bad Request - The request was malformed or contained invalid data",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"401": {
"description": "Unauthorized - Authentication is required or the token is invalid",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"403": {
"description": "Forbidden - You do not have permission to perform this action",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"429": {
"description": "Too Many Requests - You are being rate limited",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"code": {"type": "string", "enum": ["RATE_LIMITED"]},
"message": {"type": "string"},
"retry_after": {"type": "number", "description": "Seconds to wait before retrying"},
"global": {"type": "boolean", "description": "Whether this is a global rate limit"}
},
"required": ["code", "message", "retry_after"]
}
}
},
"headers": {
"Retry-After": {
"description": "Number of seconds to wait before retrying (only on 429)",
"schema": {"type": "integer"}
},
"X-RateLimit-Limit": {
"description": "The number of requests that can be made in the current window",
"schema": {"type": "integer"}
},
"X-RateLimit-Remaining": {
"description": "The number of remaining requests that can be made",
"schema": {"type": "integer"}
},
"X-RateLimit-Reset": {
"description": "Unix timestamp when the rate limit resets",
"schema": {"type": "integer"}
}
}
},
"500": {
"description": "Internal Server Error - An unexpected error occurred",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
}
},
"x-mint": {"metadata": {"title": "Upload voice debug logging events"}},
"description": "Uploads a small batch of client voice diagnostics events for an active staff-enabled debug logging session.",
"security": [{"sessionToken": []}],
"parameters": [
{
"name": "channel_id",
"in": "path",
"required": true,
"schema": {"$ref": "#/components/schemas/SnowflakeType"},
"description": "The ID of the channel"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {"schema": {"$ref": "#/components/schemas/VoiceDebugLoggingEventsBodySchema"}}
}
}
}
},
"/channels/{channel_id}/voice-debug-logging/session": {
"get": {
"operationId": "get_voice_debug_logging_status",
"summary": "Get voice debug logging status",
"tags": ["Channels"],
"responses": {
"200": {
"description": "Success",
"content": {
"application/json": {"schema": {"$ref": "#/components/schemas/VoiceDebugLoggingStatusResponse"}}
}
},
"400": {
"description": "Bad Request - The request was malformed or contained invalid data",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"401": {
"description": "Unauthorized - Authentication is required or the token is invalid",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"403": {
"description": "Forbidden - You do not have permission to perform this action",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"429": {
"description": "Too Many Requests - You are being rate limited",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"code": {"type": "string", "enum": ["RATE_LIMITED"]},
"message": {"type": "string"},
"retry_after": {"type": "number", "description": "Seconds to wait before retrying"},
"global": {"type": "boolean", "description": "Whether this is a global rate limit"}
},
"required": ["code", "message", "retry_after"]
}
}
},
"headers": {
"Retry-After": {
"description": "Number of seconds to wait before retrying (only on 429)",
"schema": {"type": "integer"}
},
"X-RateLimit-Limit": {
"description": "The number of requests that can be made in the current window",
"schema": {"type": "integer"}
},
"X-RateLimit-Remaining": {
"description": "The number of remaining requests that can be made",
"schema": {"type": "integer"}
},
"X-RateLimit-Reset": {
"description": "Unix timestamp when the rate limit resets",
"schema": {"type": "integer"}
}
}
},
"500": {
"description": "Internal Server Error - An unexpected error occurred",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
}
},
"x-mint": {"metadata": {"title": "Get voice debug logging status"}},
"description": "Returns whether staff-enabled voice debug logging is active for this channel. Clients poll this while connected to decide whether to upload diagnostics.",
"security": [{"sessionToken": []}],
"parameters": [
{
"name": "channel_id",
"in": "path",
"required": true,
"schema": {"$ref": "#/components/schemas/SnowflakeType"},
"description": "The ID of the channel"
}
]
},
"put": {
"operationId": "set_voice_debug_logging_status",
"summary": "Toggle voice debug logging",
"tags": ["Channels"],
"responses": {
"200": {
"description": "Success",
"content": {
"application/json": {"schema": {"$ref": "#/components/schemas/VoiceDebugLoggingStatusResponse"}}
}
},
"400": {
"description": "Bad Request - The request was malformed or contained invalid data",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"401": {
"description": "Unauthorized - Authentication is required or the token is invalid",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"403": {
"description": "Forbidden - You do not have permission to perform this action",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"429": {
"description": "Too Many Requests - You are being rate limited",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"code": {"type": "string", "enum": ["RATE_LIMITED"]},
"message": {"type": "string"},
"retry_after": {"type": "number", "description": "Seconds to wait before retrying"},
"global": {"type": "boolean", "description": "Whether this is a global rate limit"}
},
"required": ["code", "message", "retry_after"]
}
}
},
"headers": {
"Retry-After": {
"description": "Number of seconds to wait before retrying (only on 429)",
"schema": {"type": "integer"}
},
"X-RateLimit-Limit": {
"description": "The number of requests that can be made in the current window",
"schema": {"type": "integer"}
},
"X-RateLimit-Remaining": {
"description": "The number of remaining requests that can be made",
"schema": {"type": "integer"}
},
"X-RateLimit-Reset": {
"description": "Unix timestamp when the rate limit resets",
"schema": {"type": "integer"}
}
}
},
"500": {
"description": "Internal Server Error - An unexpected error occurred",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
}
},
"x-mint": {"metadata": {"title": "Toggle voice debug logging"}},
"description": "Allows staff to start or stop a channel-scoped voice debug logging session. Non-staff users cannot activate or stop sessions.",
"security": [{"sessionToken": []}],
"parameters": [
{
"name": "channel_id",
"in": "path",
"required": true,
"schema": {"$ref": "#/components/schemas/SnowflakeType"},
"description": "The ID of the channel"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {"schema": {"$ref": "#/components/schemas/VoiceDebugLoggingToggleBodySchema"}}
}
}
}
},
"/channels/{channel_id}/voice-presence/heartbeat": {
"post": {
"operationId": "heartbeat_voice_presence",
"summary": "Heartbeat voice presence",
"tags": ["Channels"],
"responses": {
"200": {
"description": "Success",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/VoicePresenceHeartbeatResponse"}}}
},
"400": {
"description": "Bad Request - The request was malformed or contained invalid data",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"401": {
"description": "Unauthorized - Authentication is required or the token is invalid",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"403": {
"description": "Forbidden - You do not have permission to perform this action",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"429": {
"description": "Too Many Requests - You are being rate limited",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"code": {"type": "string", "enum": ["RATE_LIMITED"]},
"message": {"type": "string"},
"retry_after": {"type": "number", "description": "Seconds to wait before retrying"},
"global": {"type": "boolean", "description": "Whether this is a global rate limit"}
},
"required": ["code", "message", "retry_after"]
}
}
},
"headers": {
"Retry-After": {
"description": "Number of seconds to wait before retrying (only on 429)",
"schema": {"type": "integer"}
},
"X-RateLimit-Limit": {
"description": "The number of requests that can be made in the current window",
"schema": {"type": "integer"}
},
"X-RateLimit-Remaining": {
"description": "The number of remaining requests that can be made",
"schema": {"type": "integer"}
},
"X-RateLimit-Reset": {
"description": "Unix timestamp when the rate limit resets",
"schema": {"type": "integer"}
}
}
},
"500": {
"description": "Internal Server Error - An unexpected error occurred",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
}
},
"x-mint": {"metadata": {"title": "Heartbeat voice presence"}},
"description": "Refreshes the current user voice presence marker for v2 voice reconciliation. Clients call this while connected to voice.",
"security": [{"sessionToken": []}],
"parameters": [
{
"name": "channel_id",
"in": "path",
"required": true,
"schema": {"$ref": "#/components/schemas/SnowflakeType"},
"description": "The ID of the channel"
}
],
"requestBody": {
"required": true,
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/VoicePresenceHeartbeatBodySchema"}}}
}
},
"delete": {
"operationId": "end_voice_presence_heartbeat",
"summary": "End voice presence heartbeat",
"tags": ["Channels"],
"responses": {
"200": {
"description": "Success",
"content": {
"application/json": {"schema": {"$ref": "#/components/schemas/VoicePresenceHeartbeatEndResponse"}}
}
},
"400": {
"description": "Bad Request - The request was malformed or contained invalid data",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"401": {
"description": "Unauthorized - Authentication is required or the token is invalid",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"403": {
"description": "Forbidden - You do not have permission to perform this action",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
},
"429": {
"description": "Too Many Requests - You are being rate limited",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"code": {"type": "string", "enum": ["RATE_LIMITED"]},
"message": {"type": "string"},
"retry_after": {"type": "number", "description": "Seconds to wait before retrying"},
"global": {"type": "boolean", "description": "Whether this is a global rate limit"}
},
"required": ["code", "message", "retry_after"]
}
}
},
"headers": {
"Retry-After": {
"description": "Number of seconds to wait before retrying (only on 429)",
"schema": {"type": "integer"}
},
"X-RateLimit-Limit": {
"description": "The number of requests that can be made in the current window",
"schema": {"type": "integer"}
},
"X-RateLimit-Remaining": {
"description": "The number of remaining requests that can be made",
"schema": {"type": "integer"}
},
"X-RateLimit-Reset": {
"description": "Unix timestamp when the rate limit resets",
"schema": {"type": "integer"}
}
}
},
"500": {
"description": "Internal Server Error - An unexpected error occurred",
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
}
},
"x-mint": {"metadata": {"title": "End voice presence heartbeat"}},
"description": "Clears the current user active v2 voice presence marker for a voice connection while preserving v2 enrollment for fast reconciliation.",
"security": [{"sessionToken": []}],
"parameters": [
{
"name": "channel_id",
"in": "path",
"required": true,
"schema": {"$ref": "#/components/schemas/SnowflakeType"},
"description": "The ID of the channel"
}
],
"requestBody": {
"required": true,
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/VoicePresenceHeartbeatBodySchema"}}}
}
}
},
"/channels/{channel_id}/webhooks": {
"get": {
"operationId": "list_channel_webhooks",
@@ -29516,248 +29112,6 @@
},
"required": ["rate_limit_per_user", "retry_after_ms", "next_send_allowed_at", "can_bypass"]
},
"VoiceDebugLoggingEventsResponse": {
"type": "object",
"properties": {
"accepted": {"type": "boolean", "description": "Whether the telemetry batch was accepted for storage"},
"active": {
"type": "boolean",
"description": "Whether the server still considers this logging session active"
},
"stored_event_count": {
"type": "integer",
"minimum": 0,
"maximum": 2147483647,
"format": "int32",
"description": "Number of events written to diagnostics storage"
}
},
"required": ["accepted", "active", "stored_event_count"]
},
"VoiceDebugLoggingEventsBodySchema": {
"type": "object",
"properties": {
"session_id": {"type": "string", "description": "Active voice debug logging session id"},
"connection_id": {"type": "string", "description": "Client voice connection id"},
"participant_identity": {"type": "string", "description": "LiveKit participant identity"},
"events": {
"type": "array",
"items": {"$ref": "#/components/schemas/VoiceDebugLoggingEventSchema"},
"minItems": 1,
"maxItems": 200,
"description": "NDJSON batch events to store"
}
},
"required": ["session_id", "events"]
},
"VoiceDebugLoggingEventSchema": {
"type": "object",
"properties": {
"type": {"type": "string", "description": "Client-side diagnostic event type"},
"timestamp_ns": {
"type": "string",
"pattern": "^[0-9]{1,32}$",
"description": "Client wall-clock Unix timestamp in nanoseconds"
},
"monotonic_ns": {
"type": "string",
"pattern": "^[0-9]{1,32}$",
"description": "Client monotonic timestamp in nanoseconds"
},
"data": {
"type": "object",
"additionalProperties": {
"anyOf": [
{"type": "string"},
{"type": "number"},
{"type": "boolean"},
{
"type": "object",
"additionalProperties": {
"anyOf": [
{"type": "string"},
{"type": "number"},
{"type": "boolean"},
{
"type": "object",
"additionalProperties": {
"anyOf": [
{"type": "string"},
{"type": "number"},
{"type": "boolean"},
{"type": "object", "additionalProperties": true},
{"type": "null"}
]
}
},
{
"type": "array",
"items": {
"anyOf": [
{"type": "string"},
{"type": "number"},
{"type": "boolean"},
{"type": "object", "additionalProperties": true},
{"type": "null"}
]
}
},
{"type": "null"}
]
}
},
{
"type": "array",
"items": {
"anyOf": [
{"type": "string"},
{"type": "number"},
{"type": "boolean"},
{
"type": "object",
"additionalProperties": {
"anyOf": [
{"type": "string"},
{"type": "number"},
{"type": "boolean"},
{"type": "object", "additionalProperties": true},
{"type": "null"}
]
}
},
{
"type": "array",
"items": {
"anyOf": [
{"type": "string"},
{"type": "number"},
{"type": "boolean"},
{"type": "object", "additionalProperties": true},
{"type": "null"}
]
}
},
{"type": "null"}
]
}
},
{"type": "null"}
]
},
"description": "Event-specific diagnostic payload"
}
},
"required": ["type", "timestamp_ns"]
},
"VoiceDebugLoggingStatusResponse": {
"type": "object",
"properties": {
"active": {
"type": "boolean",
"description": "Whether clients in this channel should currently send voice diagnostics"
},
"session_id": {
"anyOf": [{"type": "string"}, {"type": "null"}],
"description": "Current debug logging session id, if active"
},
"activated_by_user_id": {
"anyOf": [{"$ref": "#/components/schemas/SnowflakeType"}, {"type": "null"}],
"description": "Staff user that activated the session, if active"
},
"started_at_ms": {
"anyOf": [
{"type": "integer", "minimum": 0, "maximum": 9007199254740991, "format": "int53"},
{"type": "null"}
],
"description": "Session start Unix timestamp in milliseconds"
},
"expires_at_ms": {
"anyOf": [
{"type": "integer", "minimum": 0, "maximum": 9007199254740991, "format": "int53"},
{"type": "null"}
],
"description": "Session expiration Unix timestamp in milliseconds"
},
"poll_interval_ms": {
"type": "integer",
"minimum": 0,
"maximum": 2147483647,
"format": "int32",
"description": "Recommended client polling interval in milliseconds"
},
"upload_interval_ms": {
"type": "integer",
"minimum": 0,
"maximum": 2147483647,
"format": "int32",
"description": "Recommended client telemetry batch upload interval in milliseconds"
}
},
"required": [
"active",
"session_id",
"activated_by_user_id",
"started_at_ms",
"expires_at_ms",
"poll_interval_ms",
"upload_interval_ms"
]
},
"VoiceDebugLoggingToggleBodySchema": {
"type": "object",
"properties": {
"enabled": {
"type": "boolean",
"description": "Whether voice debug logging should be active for this channel"
},
"duration_ms": {
"type": "integer",
"minimum": 60000,
"maximum": 14400000,
"format": "int32",
"description": "Optional activation duration in milliseconds. Defaults to one hour and is capped at four hours."
}
},
"required": ["enabled"]
},
"VoicePresenceHeartbeatResponse": {
"type": "object",
"properties": {
"ok": {"type": "boolean", "description": "Whether the heartbeat was accepted"},
"heartbeat_interval_ms": {
"type": "integer",
"minimum": 0,
"maximum": 2147483647,
"format": "int32",
"description": "Recommended client heartbeat interval in milliseconds"
},
"heartbeat_ttl_ms": {
"type": "integer",
"minimum": 0,
"maximum": 2147483647,
"format": "int32",
"description": "Server-side heartbeat expiration window in milliseconds"
},
"expires_at_ms": {
"type": "integer",
"minimum": 0,
"maximum": 9007199254740991,
"format": "int53",
"description": "Unix timestamp in milliseconds when this heartbeat expires"
}
},
"required": ["ok", "heartbeat_interval_ms", "heartbeat_ttl_ms", "expires_at_ms"]
},
"VoicePresenceHeartbeatBodySchema": {
"type": "object",
"properties": {"connection_id": {"type": "string", "description": "Client voice connection id"}},
"required": ["connection_id"]
},
"VoicePresenceHeartbeatEndResponse": {
"type": "object",
"properties": {"ok": {"type": "boolean", "description": "Whether the heartbeat was ended"}},
"required": ["ok"]
},
"WebhookResponse": {
"type": "object",
"properties": {
@@ -30823,7 +30177,14 @@
"minimum": 0,
"maximum": 7,
"format": "int32",
"description": "Number of days of messages to delete from the banned user (0-7)"
"description": "Number of days of messages to delete from the banned user (0-7). Deprecated in favor of delete_message_seconds."
},
"delete_message_seconds": {
"type": "integer",
"minimum": 0,
"maximum": 604800,
"format": "int32",
"description": "Number of seconds of messages to delete for the banned user (0-604800, default 0)"
},
"reason": {
"anyOf": [{"type": "string"}, {"type": "null"}],
+9 -14
View File
@@ -143,7 +143,7 @@ function buildApnsHeaders(params: {
'apns-topic': params.topic,
'apns-push-type': isClear ? APNS_PUSH_TYPE_BACKGROUND : APNS_PUSH_TYPE_ALERT,
'apns-priority': isClear ? '5' : '10',
'apns-expiration': String(Math.floor(Date.now() / 1000) + (isClear ? 300 : 86400)),
'apns-expiration': String(Math.floor(Date.now() / 1000) + (isClear ? 3600 : 86400)),
'content-type': 'application/json',
};
if (collapseId && Buffer.byteLength(collapseId) <= 64) {
@@ -155,18 +155,11 @@ function buildApnsHeaders(params: {
function buildApnsPayload(payload: Record<string, unknown>): Record<string, unknown> {
if (isClearNotificationPayload(payload)) {
const data = isRecord(payload.data) ? payload.data : {};
const badge = normalizeBadgeCount(data.badge_count);
const aps: Record<string, unknown> = {
'content-available': 1,
};
if (badge !== undefined) {
aps.badge = badge;
}
return {
...data,
type: 'notification_clear',
action: 'clear_channel',
aps,
aps: {'content-available': 1},
};
}
const data = isRecord(payload.data) ? payload.data : {};
@@ -177,16 +170,18 @@ function buildApnsPayload(payload: Record<string, unknown>): Record<string, unkn
const channelId = optionalString(data.channel_id);
const threadId =
optionalString(data.notification_tag) ?? (channelId ? `channel:${channelId}` : undefined) ?? 'fluxer-message';
const imageUrl = firstString([payload.image_url, notification.image, notification.icon, payload.icon]);
const imageUrl = firstString([payload.image_url, notification.image]);
const aps: Record<string, unknown> = {
alert: {title, body},
sound: APNS_DEFAULT_SOUND,
badge,
'thread-id': threadId,
category: APNS_CATEGORY_MESSAGE,
'interruption-level': 'active',
'relevance-score': 0.5,
};
if (badge !== undefined) {
aps.badge = badge;
}
if (imageUrl) {
aps['mutable-content'] = 1;
}
@@ -292,13 +287,13 @@ function isPermanentApnsFailure(statusCode: number, reason: string): boolean {
return reason === 'Unregistered';
}
function normalizeBadgeCount(value: unknown): number {
function normalizeBadgeCount(value: unknown): number | undefined {
if (typeof value === 'number' && Number.isFinite(value)) return Math.max(0, Math.floor(value));
if (typeof value === 'string') {
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) ? Math.max(0, parsed) : 0;
return Number.isFinite(parsed) ? Math.max(0, parsed) : undefined;
}
return 0;
return undefined;
}
function firstString(values: Array<unknown>): string | undefined {
@@ -38,7 +38,7 @@ describe('ApnsPushService', () => {
},
});
});
it('builds silent APNs clear payloads with badge when badge_count is present', () => {
it('builds silent APNs clear payloads as pure background pushes', () => {
const payload = ApnsPushServiceTestHooks.buildApnsPayload({
type: 'notification_clear',
action: 'clear_channel',
@@ -56,11 +56,11 @@ describe('ApnsPushService', () => {
badge_count: 0,
aps: {
'content-available': 1,
badge: 0,
},
});
expect(payload.aps).not.toHaveProperty('alert');
expect(payload.aps).not.toHaveProperty('sound');
expect(payload.aps).not.toHaveProperty('badge');
});
it('uses APNs push-type and priority headers that match alert versus background delivery', () => {
const alertHeaders = ApnsPushServiceTestHooks.buildApnsHeaders({
@@ -94,6 +94,43 @@ describe('ApnsPushService', () => {
'apns-priority': '5',
'apns-collapse-id': 'channel:123',
});
const now = Math.floor(Date.now() / 1000);
expect(Number(clearHeaders['apns-expiration']) - now).toBeGreaterThan(1800);
});
it('omits the badge entirely when no usable badge count is supplied', () => {
const payload = ApnsPushServiceTestHooks.buildApnsPayload({
tag: 'channel:123:456',
data: {channel_id: '123', message_id: '456'},
notification: {title: 'Alice', body: 'Hello'},
});
expect(payload.aps).not.toHaveProperty('badge');
const unparseable = ApnsPushServiceTestHooks.buildApnsPayload({
tag: 'channel:123:456',
data: {channel_id: '123', message_id: '456', badge_count: 'not-a-number'},
notification: {title: 'Alice', body: 'Hello'},
});
expect(unparseable.aps).not.toHaveProperty('badge');
});
it('keeps a genuine zero badge so reading the last message clears the app icon', () => {
const payload = ApnsPushServiceTestHooks.buildApnsPayload({
tag: 'channel:123:456',
data: {channel_id: '123', message_id: '456', badge_count: 0},
notification: {title: 'Alice', body: 'Hello'},
});
expect((payload.aps as Record<string, unknown>).badge).toBe(0);
});
it('does not use the sender avatar as the notification media image', () => {
const payload = ApnsPushServiceTestHooks.buildApnsPayload({
tag: 'channel:123:456',
data: {channel_id: '123', message_id: '456', author_avatar_url: 'https://cdn.example/avatar.png'},
notification: {title: 'Alice', body: 'Hello', icon: 'https://cdn.example/avatar.png'},
});
expect(payload.image_url).toBeUndefined();
expect(payload.aps).not.toHaveProperty('mutable-content');
expect(payload.author_avatar_url).toBe('https://cdn.example/avatar.png');
});
it('marks only permanent APNs token failures as subscription deletion signals', () => {
expect(ApnsPushServiceTestHooks.isPermanentApnsFailure(410, 'Unregistered')).toBe(true);
@@ -96,22 +96,6 @@ export const ChannelRateLimitConfigs = {
bucket: 'channel:call:stop_ringing::channel_id',
config: {limit: 20, windowMs: ms('10 seconds')},
} as RouteRateLimitConfig,
CHANNEL_VOICE_DEBUG_LOGGING_STATUS: {
bucket: 'channel:voice_debug_logging:status::channel_id',
config: {limit: 60, windowMs: ms('10 seconds')},
} as RouteRateLimitConfig,
CHANNEL_VOICE_DEBUG_LOGGING_TOGGLE: {
bucket: 'channel:voice_debug_logging:toggle::channel_id',
config: {limit: 10, windowMs: ms('1 minute')},
} as RouteRateLimitConfig,
CHANNEL_VOICE_DEBUG_LOGGING_EVENTS: {
bucket: 'channel:voice_debug_logging:events::channel_id::user_id',
config: {limit: 60, windowMs: ms('10 seconds')},
} as RouteRateLimitConfig,
CHANNEL_VOICE_PRESENCE_HEARTBEAT: {
bucket: 'channel:voice_presence:heartbeat::channel_id::user_id',
config: {limit: 20, windowMs: ms('10 seconds')},
} as RouteRateLimitConfig,
CHANNEL_STREAM_UPDATE: {
bucket: 'channel:stream:update::stream_key',
config: {limit: 20, windowMs: ms('10 seconds')},
@@ -37,6 +37,7 @@ export class TestMediaService extends IMediaService {
height: 128,
animated: stored ? this.isAnimatedImage(stored, format) : format === 'gif',
nsfw: false,
placeholder: this.fakePlaceholder(request.key),
};
}
if (request.type === 'upload') {
@@ -55,6 +56,7 @@ export class TestMediaService extends IMediaService {
height: 128,
animated: format === 'gif',
nsfw: false,
placeholder: this.fakePlaceholder(request.upload_filename),
};
}
if (request.type === 'external') {
@@ -67,11 +69,16 @@ export class TestMediaService extends IMediaService {
height: 128,
animated: false,
nsfw: false,
placeholder: this.fakePlaceholder(request.url),
};
}
return null;
}
private fakePlaceholder(seed: string): string {
return crypto.createHash('sha256').update(seed).digest('base64').slice(0, 24);
}
getExternalMediaProxyURL(): string {
return 'https://media-proxy.test';
}
@@ -3,12 +3,15 @@
import {MessageAttachmentFlags} from '@fluxer/constants/src/ChannelConstants';
import {afterEach, beforeEach, describe, expect, test} from 'vitest';
import {AttachmentDecayRepository} from '../../attachment/AttachmentDecayRepository';
import {createAttachmentID} from '../../BrandedTypes';
import {createAttachmentID, createMemeID, createUserID} from '../../BrandedTypes';
import {
createTestAccountForAttachmentTests,
sendMessageWithAttachments,
setupTestGuildAndChannel,
} from '../../channel/tests/AttachmentTestUtils';
import {fetchOne} from '../../database/CassandraQueryExecution';
import {Db} from '../../database/CassandraTypes';
import {FavoriteMemes} from '../../Tables';
import {type ApiTestHarness, createApiTestHarness} from '../../test/ApiTestHarness';
import {HTTP_STATUS} from '../../test/TestConstants';
import {createBuilder} from '../../test/TestRequestBuilder';
@@ -42,6 +45,24 @@ interface MessageWithDecayAttachment {
}>;
}
interface MessageWithPlaceholderAttachment {
id: string;
attachments: Array<{
id: string;
filename: string;
placeholder?: string | null;
}>;
}
async function clearFavoriteMemePlaceholder(userId: string, memeId: string) {
await fetchOne(
FavoriteMemes.patchByPk(
{user_id: createUserID(BigInt(userId)), meme_id: createMemeID(BigInt(memeId))},
{placeholder: Db.set(null)},
),
);
}
async function fetchDecayRow(attachmentId: string) {
return new AttachmentDecayRepository().fetchById(createAttachmentID(BigInt(attachmentId)));
}
@@ -111,6 +132,43 @@ describe('Favorite Meme Operations', () => {
expect(sent.attachments[0].filename).toBe(filename);
expect(sent.attachments[0].flags & MessageAttachmentFlags.IS_ANIMATED).toBe(MessageAttachmentFlags.IS_ANIMATED);
});
test('should carry the saved placeholder onto the sent attachment', async () => {
const account = await createTestAccountForAttachmentTests(harness);
const {channel} = await setupTestGuildAndChannel(harness, account);
const message = await createMessageWithImageAttachment(harness, account.token, channel.id);
const meme = await createFavoriteMemeFromMessage(harness, account.token, channel.id, message.id, {
attachment_id: message.attachments[0].id,
name: 'Placeholder Meme',
});
expect(meme.placeholder).toBeTruthy();
const sent = await createBuilder<MessageWithPlaceholderAttachment>(harness, account.token)
.post(`/channels/${channel.id}/messages`)
.body({favorite_meme_id: meme.id})
.expect(HTTP_STATUS.OK)
.execute();
expect(sent.attachments[0].placeholder).toBe(meme.placeholder);
});
test('should read-repair a missing placeholder when sending a favorite meme', async () => {
const account = await createTestAccountForAttachmentTests(harness);
const {channel} = await setupTestGuildAndChannel(harness, account);
const message = await createMessageWithImageAttachment(harness, account.token, channel.id);
const meme = await createFavoriteMemeFromMessage(harness, account.token, channel.id, message.id, {
attachment_id: message.attachments[0].id,
name: 'Repair Meme',
});
expect(meme.placeholder).toBeTruthy();
await clearFavoriteMemePlaceholder(account.userId, meme.id);
const stripped = await getFavoriteMeme(harness, account.token, meme.id);
expect(stripped.placeholder).toBeNull();
const sent = await createBuilder<MessageWithPlaceholderAttachment>(harness, account.token)
.post(`/channels/${channel.id}/messages`)
.body({favorite_meme_id: meme.id})
.expect(HTTP_STATUS.OK)
.execute();
expect(sent.attachments[0].placeholder).toBeTruthy();
const repaired = await getFavoriteMeme(harness, account.token, meme.id);
expect(repaired.placeholder).toBe(sent.attachments[0].placeholder);
});
test('should create decay metadata when sending favorite meme', async () => {
const account = await createTestAccountForAttachmentTests(harness);
const {channel} = await setupTestGuildAndChannel(harness, account);
@@ -1,109 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createHash} from 'node:crypto';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import type {ChannelID, UserID} from '../BrandedTypes';
export type VoicePresenceHeartbeatState = 'active' | 'expired' | 'legacy';
interface VoicePresenceHeartbeatParams {
channelId: ChannelID;
userId: UserID;
connectionId: string;
}
interface VoicePresenceHeartbeatResult {
heartbeatIntervalMs: number;
heartbeatTtlMs: number;
expiresAtMs: number;
}
const VOICE_PRESENCE_HEARTBEAT_INTERVAL_MS = 15000;
const VOICE_PRESENCE_HEARTBEAT_TTL_SECONDS = 45;
const VOICE_PRESENCE_ENROLLMENT_TTL_SECONDS = 3600;
const VOICE_PRESENCE_HEARTBEAT_ACTIVE_PREFIX = 'voice:presence:v2:active:';
const VOICE_PRESENCE_HEARTBEAT_ENROLLED_PREFIX = 'voice:presence:v2:enrolled:';
const VOICE_PRESENCE_HEARTBEAT_CHANNEL_PREFIX = 'voice:presence:v2:channel:';
export class VoicePresenceHeartbeatStore {
constructor(private readonly kvClient: IKVProvider) {}
async recordHeartbeat(params: VoicePresenceHeartbeatParams): Promise<VoicePresenceHeartbeatResult> {
const now = Date.now();
const payload = JSON.stringify({
version: 2,
channelId: params.channelId.toString(),
userId: params.userId.toString(),
connectionId: params.connectionId,
heartbeatAtMs: now,
});
await Promise.all([
this.kvClient.setex(voicePresenceHeartbeatActiveKey(params), VOICE_PRESENCE_HEARTBEAT_TTL_SECONDS, payload),
this.kvClient.setex(voicePresenceHeartbeatEnrollmentKey(params), VOICE_PRESENCE_ENROLLMENT_TTL_SECONDS, payload),
this.kvClient.sadd(voicePresenceHeartbeatChannelKey(params.channelId), voicePresenceHeartbeatKeySuffix(params)),
this.kvClient.expire(voicePresenceHeartbeatChannelKey(params.channelId), VOICE_PRESENCE_ENROLLMENT_TTL_SECONDS),
]);
return {
heartbeatIntervalMs: VOICE_PRESENCE_HEARTBEAT_INTERVAL_MS,
heartbeatTtlMs: VOICE_PRESENCE_HEARTBEAT_TTL_SECONDS * 1000,
expiresAtMs: now + VOICE_PRESENCE_HEARTBEAT_TTL_SECONDS * 1000,
};
}
async getHeartbeatState(params: VoicePresenceHeartbeatParams): Promise<VoicePresenceHeartbeatState> {
const active = await this.kvClient.get(voicePresenceHeartbeatActiveKey(params));
if (active !== null) {
return 'active';
}
const enrolled = await this.kvClient.get(voicePresenceHeartbeatEnrollmentKey(params));
return enrolled !== null ? 'expired' : 'legacy';
}
async markHeartbeatEnded(params: VoicePresenceHeartbeatParams): Promise<void> {
const enrollmentKey = voicePresenceHeartbeatEnrollmentKey(params);
const enrolled = await this.kvClient.get(enrollmentKey);
const payload = JSON.stringify({
version: 2,
channelId: params.channelId.toString(),
userId: params.userId.toString(),
connectionId: params.connectionId,
endedAtMs: Date.now(),
});
await Promise.all([
this.kvClient.del(voicePresenceHeartbeatActiveKey(params)),
this.kvClient.srem(voicePresenceHeartbeatChannelKey(params.channelId), voicePresenceHeartbeatKeySuffix(params)),
enrolled === null
? Promise.resolve()
: this.kvClient.setex(enrollmentKey, VOICE_PRESENCE_ENROLLMENT_TTL_SECONDS, payload),
]);
}
async markChannelHeartbeatsEnded(channelId: ChannelID): Promise<void> {
const channelKey = voicePresenceHeartbeatChannelKey(channelId);
const suffixes = await this.kvClient.smembers(channelKey);
const activeKeys = suffixes.map((suffix) => `${VOICE_PRESENCE_HEARTBEAT_ACTIVE_PREFIX}${suffix}`);
if (activeKeys.length > 0) {
await this.kvClient.del(...activeKeys, channelKey);
return;
}
await this.kvClient.del(channelKey);
}
}
function voicePresenceHeartbeatActiveKey(params: VoicePresenceHeartbeatParams): string {
return `${VOICE_PRESENCE_HEARTBEAT_ACTIVE_PREFIX}${voicePresenceHeartbeatKeySuffix(params)}`;
}
function voicePresenceHeartbeatEnrollmentKey(params: VoicePresenceHeartbeatParams): string {
return `${VOICE_PRESENCE_HEARTBEAT_ENROLLED_PREFIX}${voicePresenceHeartbeatKeySuffix(params)}`;
}
function voicePresenceHeartbeatChannelKey(channelId: ChannelID): string {
return `${VOICE_PRESENCE_HEARTBEAT_CHANNEL_PREFIX}${channelId.toString()}`;
}
function voicePresenceHeartbeatKeySuffix(params: VoicePresenceHeartbeatParams): string {
const connectionHash = createHash('sha256').update(params.connectionId).digest('base64url');
return `channel:${params.channelId.toString()}:user:${params.userId.toString()}:connection:${connectionHash}`;
}
@@ -12,7 +12,6 @@ import type {GatewayVoiceStateEntry, IGatewayService} from '../infrastructure/IG
import type {ILiveKitService, LiveKitRoomLocation} from '../infrastructure/ILiveKitService';
import type {IVoiceRoomStore} from '../infrastructure/IVoiceRoomStore';
import {parseParticipantIdentity, parseRoomName} from '../infrastructure/VoiceRoomContext';
import {type VoicePresenceHeartbeatState, VoicePresenceHeartbeatStore} from './VoicePresenceHeartbeatStore';
interface GatewayPendingJoinEntry {
readonly connectionId: string;
@@ -27,7 +26,6 @@ interface VoiceReconciliationWorkerOptions {
voiceRoomStore: IVoiceRoomStore;
kvClient: IKVProvider;
logger: ILogger;
voicePresenceHeartbeatStore?: VoicePresenceHeartbeatStore;
intervalMs?: number;
staggerDelayMs?: number;
lockTtlSeconds?: number;
@@ -122,7 +120,6 @@ export class VoiceReconciliationWorker {
private readonly liveKitService: ILiveKitService;
private readonly voiceRoomStore: IVoiceRoomStore;
private readonly kvClient: IKVProvider;
private readonly voicePresenceHeartbeatStore: VoicePresenceHeartbeatStore;
private readonly logger: ILogger;
private readonly intervalMs: number;
private readonly staggerDelayMs: number;
@@ -140,8 +137,6 @@ export class VoiceReconciliationWorker {
this.liveKitService = options.liveKitService;
this.voiceRoomStore = options.voiceRoomStore;
this.kvClient = options.kvClient;
this.voicePresenceHeartbeatStore =
options.voicePresenceHeartbeatStore ?? new VoicePresenceHeartbeatStore(this.kvClient);
this.logger = options.logger.child({worker: 'VoiceReconciliationWorker'});
this.intervalMs = options.intervalMs ?? DEFAULT_INTERVAL_MS;
this.staggerDelayMs = options.staggerDelayMs ?? DEFAULT_STAGGER_DELAY_MS;
@@ -575,34 +570,6 @@ export class VoiceReconciliationWorker {
livekitOnlyDeferred++;
continue;
}
const heartbeatState = await this.getVoicePresenceHeartbeatState(room.channelId, participant);
if (heartbeatState === 'active') {
await this.clearLiveKitOnlyCandidate(room.guildId, room.channelId, participant);
this.logger.debug(
{
roomName: room.roomName,
userId: participant.userId.toString(),
connectionId: participant.connectionId,
},
'Deferring LiveKit-only participant because v2 voice presence heartbeat is active',
);
livekitOnlyDeferred++;
continue;
}
if (heartbeatState === 'expired') {
await this.clearLiveKitOnlyCandidate(room.guildId, room.channelId, participant);
this.logger.warn(
{
roomName: room.roomName,
userId: participant.userId.toString(),
connectionId: participant.connectionId,
},
'Disconnecting LiveKit-only participant because v2 voice presence heartbeat expired',
);
await this.disconnectLiveKitOnlyParticipant(room, participant);
livekitOnlyDisconnected++;
continue;
}
if (!liveKitSnapshot.completed) {
this.logger.warn(
{
@@ -649,7 +616,7 @@ export class VoiceReconciliationWorker {
gatewayOnlySkipped++;
continue;
}
const shouldRemove = await this.shouldRemoveGatewayOnlyState(room, voiceState);
const shouldRemove = await this.confirmGatewayOnlyCandidate(room.guildId, room.channelId, voiceState);
if (!shouldRemove) {
gatewayOnlyDeferred++;
continue;
@@ -954,67 +921,6 @@ export class VoiceReconciliationWorker {
}
}
private async shouldRemoveGatewayOnlyState(
room: DiscoveredRoom,
voiceState: GatewayVoiceStateEntry,
): Promise<boolean> {
let userId: UserID;
try {
userId = createUserID(BigInt(voiceState.userId));
} catch (error) {
this.logger.warn(
{error, roomName: room.roomName, userId: voiceState.userId, connectionId: voiceState.connectionId},
'Falling back to legacy gateway-only reconciliation because voice state user id was invalid',
);
return this.confirmGatewayOnlyCandidate(room.guildId, room.channelId, voiceState);
}
const heartbeatState = await this.getVoicePresenceHeartbeatState(room.channelId, {
userId,
connectionId: voiceState.connectionId,
});
if (heartbeatState === 'active') {
await this.clearGatewayOnlyCandidate(room.guildId, room.channelId, voiceState.connectionId);
this.logger.debug(
{roomName: room.roomName, userId: voiceState.userId, connectionId: voiceState.connectionId},
'Deferring gateway-only state because v2 voice presence heartbeat is active',
);
return false;
}
if (heartbeatState === 'expired') {
await this.clearGatewayOnlyCandidate(room.guildId, room.channelId, voiceState.connectionId);
this.logger.warn(
{roomName: room.roomName, userId: voiceState.userId, connectionId: voiceState.connectionId},
'Removing gateway-only state because v2 voice presence heartbeat expired',
);
return true;
}
return this.confirmGatewayOnlyCandidate(room.guildId, room.channelId, voiceState);
}
private async getVoicePresenceHeartbeatState(
channelId: ChannelID,
connection: {userId: UserID; connectionId: string},
): Promise<VoicePresenceHeartbeatState> {
try {
return await this.voicePresenceHeartbeatStore.getHeartbeatState({
channelId,
userId: connection.userId,
connectionId: connection.connectionId,
});
} catch (error) {
this.logger.warn(
{
error,
channelId: channelId.toString(),
userId: connection.userId.toString(),
connectionId: connection.connectionId,
},
'Falling back to legacy reconciliation because v2 voice presence heartbeat lookup failed',
);
return 'legacy';
}
}
private async confirmGatewayOnlyCandidate(
guildId: GuildID | undefined,
channelId: ChannelID,
@@ -9,28 +9,28 @@ import {getWorkerDependencies} from '../WorkerContext';
const PayloadSchema = z.object({
guildId: z.string(),
userId: z.string(),
days: z.number().min(0).max(7),
seconds: z.number().min(0).max(604800),
});
const deleteUserMessagesInGuildByTime: WorkerTaskHandler = async (payload, helpers) => {
const validated = PayloadSchema.parse(payload);
helpers.logger.debug({payload: validated}, 'Processing deleteUserMessagesInGuildByTime task');
const guildId = createGuildID(BigInt(validated.guildId));
const userId = createUserID(BigInt(validated.userId));
const {days} = validated;
const {seconds} = validated;
Logger.debug(
{guildId: guildId.toString(), userId: userId.toString(), days},
{guildId: guildId.toString(), userId: userId.toString(), seconds},
'Starting time-based message deletion for guild ban',
);
try {
const {channelService} = getWorkerDependencies();
await channelService.messages.deletion.deleteUserMessagesInGuild({guildId, userId, days});
await channelService.messages.deletion.deleteUserMessagesInGuild({guildId, userId, seconds});
Logger.debug(
{guildId: guildId.toString(), userId: userId.toString(), days},
{guildId: guildId.toString(), userId: userId.toString(), seconds},
'Time-based message deletion completed successfully',
);
} catch (error) {
Logger.error(
{guildId: guildId.toString(), userId: userId.toString(), days, error},
{guildId: guildId.toString(), userId: userId.toString(), seconds, error},
'Failed to delete user messages in guild',
);
throw error;
@@ -21,6 +21,7 @@ import {
} from '../../SearchFactory';
import type {IGuildMemberSearchService} from '../../search/IGuildMemberSearchService';
import type {IMessageSearchService} from '../../search/IMessageSearchService';
import {deleteChannelMessageSearchDocuments} from '../../search/MessageSearchIndexCleanup';
import {getWorkerDependencies} from '../WorkerContext';
const INDEX_TYPES = [
@@ -190,11 +191,16 @@ const refreshChannelMessages: IndexHandler = async (payload, helpers, kvClient,
const {channelRepository} = getWorkerDependencies();
const guildId = createGuildID(BigInt(payload.guild_id!));
const searchService = requireSearchService<IMessageSearchService>(getMessageSearchService());
await searchService.deleteGuildMessages(guildId);
const channels = await channelRepository.listGuildChannels(guildId);
if (channels.length === 0) {
return 0;
}
for (const channel of channels) {
await deleteChannelMessageSearchDocuments(channel.id, {
searchService,
context: {source: 'bulk_reindex', guildId: guildId.toString()},
});
}
const completionKey = `bulk_reindex:${payload.job_id}:remaining`;
await kvClient.del(completionKey);
for (const channel of channels) {
+89
View File
@@ -0,0 +1,89 @@
/* SPDX-License-Identifier: AGPL-3.0-or-later */
:root {
--font-fallback-sans:
'Apple Color Emoji', 'Segoe UI Emoji', 'Noto Color Emoji',
'Fluxer Fallback', 'Fluxer Symbols', 'Fluxer Math',
'Noto Sans', 'Nirmala UI',
'Noto Sans Bengali', 'Noto Sans Tamil', 'Noto Sans Telugu', 'Noto Sans Kannada', 'Noto Sans Malayalam',
'Noto Sans Gujarati', 'Noto Sans Gurmukhi', 'Noto Sans Oriya', 'Noto Sans Sinhala',
'Noto Sans Khmer', 'Noto Sans Lao', 'Noto Sans Myanmar', 'Noto Sans Tibetan',
'Noto Sans Armenian', 'Noto Sans Georgian', 'Noto Sans Ethiopic', 'Noto Sans Cherokee',
'Noto Sans Canadian Aboriginal', 'Noto Sans Mongolian', 'Noto Sans Vai', 'Noto Sans NKo',
'Noto Sans Adlam', 'Noto Sans Yi', 'Noto Sans Osage', 'Noto Sans Thaana',
'Noto Sans CJK SC', 'Noto Sans CJK TC', 'Noto Sans CJK JP', 'Noto Sans CJK KR',
'Segoe UI Symbol', 'Segoe UI Historic', 'Apple Symbols',
'Noto Sans Symbols', 'Noto Sans Symbols 2', 'Noto Sans Math', 'Cambria Math';
--font-fallback-mono:
'Noto Sans Mono',
'Fluxer Sans SC', 'Fluxer Sans TC', 'Fluxer Sans JP', 'Fluxer Sans KR',
var(--font-fallback-sans);
--font-sans:
'Fluxer Sans', 'Fluxer Sans Arabic', 'Fluxer Sans Hebrew', 'Fluxer Sans Devanagari', 'Fluxer Sans Thai Looped',
'Fluxer Sans SC', 'Fluxer Sans TC', 'Fluxer Sans JP', 'Fluxer Sans KR', system-ui, -apple-system,
BlinkMacSystemFont, 'Segoe UI', 'Roboto', var(--font-fallback-sans), sans-serif;
--font-mono: 'Fluxer Mono', 'Menlo', 'Monaco', 'Courier New', var(--font-fallback-mono), monospace;
}
:root:lang(ja) {
--font-sans:
'Fluxer Sans', 'Fluxer Sans JP', 'Fluxer Sans Arabic', 'Fluxer Sans Hebrew', 'Fluxer Sans Devanagari',
'Fluxer Sans Thai Looped', 'Fluxer Sans SC', 'Fluxer Sans TC', 'Fluxer Sans KR', system-ui, -apple-system,
BlinkMacSystemFont, 'Hiragino Sans', 'Hiragino Kaku Gothic ProN', 'Yu Gothic', 'Meiryo',
var(--font-fallback-sans), sans-serif;
}
:root:lang(ko) {
--font-sans:
'Fluxer Sans', 'Fluxer Sans KR', 'Fluxer Sans Arabic', 'Fluxer Sans Hebrew', 'Fluxer Sans Devanagari',
'Fluxer Sans Thai Looped', 'Fluxer Sans SC', 'Fluxer Sans TC', 'Fluxer Sans JP', system-ui, -apple-system,
BlinkMacSystemFont, 'Malgun Gothic', 'Apple SD Gothic Neo', var(--font-fallback-sans), sans-serif;
}
:root:lang(zh-CN),
:root:lang(zh-Hans) {
--font-sans:
'Fluxer Sans', 'Fluxer Sans SC', 'Fluxer Sans Arabic', 'Fluxer Sans Hebrew', 'Fluxer Sans Devanagari',
'Fluxer Sans Thai Looped', 'Fluxer Sans TC', 'Fluxer Sans JP', 'Fluxer Sans KR', system-ui, -apple-system,
BlinkMacSystemFont, 'PingFang SC', 'Microsoft YaHei', var(--font-fallback-sans), sans-serif;
}
:root:lang(zh-Hant-TW),
:root:lang(zh-TW) {
--font-sans:
'Fluxer Sans', 'Fluxer Sans TC', 'Fluxer Sans Arabic', 'Fluxer Sans Hebrew', 'Fluxer Sans Devanagari',
'Fluxer Sans Thai Looped', 'Fluxer Sans SC', 'Fluxer Sans JP', 'Fluxer Sans KR', system-ui, -apple-system,
BlinkMacSystemFont, 'PingFang TC', 'Microsoft JhengHei', var(--font-fallback-sans), sans-serif;
}
:root:lang(zh-HK),
:root:lang(zh-Hant-HK) {
--font-sans:
'Fluxer Sans', 'Fluxer Sans TC', 'Fluxer Sans Arabic', 'Fluxer Sans Hebrew', 'Fluxer Sans Devanagari',
'Fluxer Sans Thai Looped', 'Fluxer Sans SC', 'Fluxer Sans JP', 'Fluxer Sans KR', system-ui, -apple-system,
BlinkMacSystemFont, 'PingFang HK', 'Microsoft JhengHei', var(--font-fallback-sans), sans-serif;
}
:root:lang(ar) {
--font-sans:
'Fluxer Sans', 'Fluxer Sans Arabic', 'Fluxer Sans Hebrew', 'Fluxer Sans Devanagari', 'Fluxer Sans Thai Looped',
'Fluxer Sans SC', 'Fluxer Sans TC', 'Fluxer Sans JP', 'Fluxer Sans KR', system-ui, -apple-system,
BlinkMacSystemFont, 'Segoe UI', 'Geeza Pro', 'Arabic Typesetting', var(--font-fallback-sans), sans-serif;
}
:root:lang(he) {
--font-sans:
'Fluxer Sans', 'Fluxer Sans Hebrew', 'Fluxer Sans Arabic', 'Fluxer Sans Devanagari', 'Fluxer Sans Thai Looped',
'Fluxer Sans SC', 'Fluxer Sans TC', 'Fluxer Sans JP', 'Fluxer Sans KR', system-ui, -apple-system,
BlinkMacSystemFont, 'Segoe UI', 'Arial Hebrew', var(--font-fallback-sans), sans-serif;
}
:root:lang(hi),
:root:lang(mr),
:root:lang(ne),
:root:lang(sa) {
--font-sans:
'Fluxer Sans', 'Fluxer Sans Devanagari', 'Fluxer Sans Arabic', 'Fluxer Sans Hebrew', 'Fluxer Sans Thai Looped',
'Fluxer Sans SC', 'Fluxer Sans TC', 'Fluxer Sans JP', 'Fluxer Sans KR', system-ui, -apple-system,
BlinkMacSystemFont, 'Nirmala UI', 'Noto Sans Devanagari', var(--font-fallback-sans), sans-serif;
}
:root:lang(th) {
--font-sans:
'Fluxer Sans', 'Fluxer Sans Thai Looped', 'Fluxer Sans Arabic', 'Fluxer Sans Hebrew', 'Fluxer Sans Devanagari',
'Fluxer Sans SC', 'Fluxer Sans TC', 'Fluxer Sans JP', 'Fluxer Sans KR', system-ui, -apple-system,
BlinkMacSystemFont, 'Leelawadee UI', 'Thonburi', var(--font-fallback-sans), sans-serif;
}
@@ -0,0 +1,96 @@
Copyright 2012-2025 The Noto Project Authors (https://github.com/notofonts)
This Font Software is a subset of Google Noto Sans faces, self-hosted for
guaranteed script coverage. The unmodified license follows.
This Font Software is licensed under the SIL Open Font License, Version 1.1.
This license is copied below, and is also available with a FAQ at:
https://openfontlicense.org
-----------------------------------------------------------
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
-----------------------------------------------------------
PREAMBLE
The goals of the Open Font License (OFL) are to stimulate worldwide
development of collaborative font projects, to support the font creation
efforts of academic and linguistic communities, and to provide a free and
open framework in which fonts may be shared and improved in partnership
with others.
The OFL allows the licensed fonts to be used, studied, modified and
redistributed freely as long as they are not sold by themselves. The
fonts, including any derivative works, can be bundled, embedded,
redistributed and/or sold with any software provided that any reserved
names are not used by derivative works. The fonts and derivatives,
however, cannot be released under any other type of license. The
requirement for fonts to remain under this license does not apply
to any document created using the fonts or their derivatives.
DEFINITIONS
"Font Software" refers to the set of files released by the Copyright
Holder(s) under this license and clearly marked as such. This may
include source files, build scripts and documentation.
"Reserved Font Name" refers to any names specified as such after the
copyright statement(s).
"Original Version" refers to the collection of Font Software components as
distributed by the Copyright Holder(s).
"Modified Version" refers to any derivative made by adding to, deleting,
or substituting -- in part or in whole -- any of the components of the
Original Version, by changing formats or by porting the Font Software to a
new environment.
"Author" refers to any designer, engineer, programmer, technical
writer or other person who contributed to the Font Software.
PERMISSION & CONDITIONS
Permission is hereby granted, free of charge, to any person obtaining
a copy of the Font Software, to use, study, copy, merge, embed, modify,
redistribute, and sell modified and unmodified copies of the Font
Software, subject to the following conditions:
1) Neither the Font Software nor any of its individual components,
in Original or Modified Versions, may be sold by itself.
2) Original or Modified Versions of the Font Software may be bundled,
redistributed and/or sold with any software, provided that each copy
contains the above copyright notice and this license. These can be
included either as stand-alone text files, human-readable headers or
in the appropriate machine-readable metadata fields within text or
binary files as long as those fields can be easily viewed by the user.
3) No Modified Version of the Font Software may use the Reserved Font
Name(s) unless explicit written permission is granted by the corresponding
Copyright Holder. This restriction only applies to the primary font name as
presented to the users.
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
Software shall not be used to promote, endorse or advertise any
Modified Version, except to acknowledge the contribution(s) of the
Copyright Holder(s) and the Author(s) or with their explicit written
permission.
5) The Font Software, modified or unmodified, in part or in whole,
must be distributed entirely under this license, and must not be
distributed under any other license. The requirement for fonts to
remain under this license does not apply to any document created
using the fonts.
TERMINATION
This license becomes null and void if any of the above conditions are
not met.
DISCLAIMER
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
OTHER DEALINGS IN THE FONT SOFTWARE.
@@ -0,0 +1,218 @@
/* SPDX-License-Identifier: OFL-1.1 */
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansBengali-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+0980-0983, U+0985-098C, U+098F-0990, U+0993-09A8, U+09AA-09B0, U+09B2, U+09B6-09B9, U+09BC-09C4, U+09C7-09C8, U+09CB-09CE, U+09D7, U+09DC-09DD, U+09DF-09E3, U+09E6-09FE;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansTamil-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+0B82-0B83, U+0B85-0B8A, U+0B8E-0B90, U+0B92-0B95, U+0B99-0B9A, U+0B9C, U+0B9E-0B9F, U+0BA3-0BA4, U+0BA8-0BAA, U+0BAE-0BB9, U+0BBE-0BC2, U+0BC6-0BC8, U+0BCA-0BCD, U+0BD0, U+0BD7, U+0BE6-0BFA;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansTelugu-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+0C00-0C0C, U+0C0E-0C10, U+0C12-0C28, U+0C2A-0C39, U+0C3C-0C44, U+0C46-0C48, U+0C4A-0C4D, U+0C55-0C56, U+0C58-0C5A, U+0C5D, U+0C60-0C63, U+0C66-0C6F, U+0C77-0C7F;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansKannada-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+0C80-0C8C, U+0C8E-0C90, U+0C92-0CA8, U+0CAA-0CB3, U+0CB5-0CB9, U+0CBC-0CC4, U+0CC6-0CC8, U+0CCA-0CCD, U+0CD5-0CD6, U+0CDD-0CDE, U+0CE0-0CE3, U+0CE6-0CEF, U+0CF1-0CF3;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansMalayalam-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+0D00-0D0C, U+0D0E-0D10, U+0D12-0D44, U+0D46-0D48, U+0D4A-0D4F, U+0D54-0D63, U+0D66-0D7F;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansGujarati-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+0A81-0A83, U+0A85-0A8D, U+0A8F-0A91, U+0A93-0AA8, U+0AAA-0AB0, U+0AB2-0AB3, U+0AB5-0AB9, U+0ABC-0AC5, U+0AC7-0AC9, U+0ACB-0ACD, U+0AD0, U+0AE0-0AE3, U+0AE6-0AF1, U+0AF9-0AFF;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansGurmukhi-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+0A01-0A03, U+0A05-0A0A, U+0A0F-0A10, U+0A13-0A28, U+0A2A-0A30, U+0A32-0A33, U+0A35-0A36, U+0A38-0A39, U+0A3C, U+0A3E-0A42, U+0A47-0A48, U+0A4B-0A4D, U+0A51, U+0A59-0A5C, U+0A5E, U+0A66-0A76;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansOriya-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+0B01-0B03, U+0B05-0B0C, U+0B0F-0B10, U+0B13-0B28, U+0B2A-0B30, U+0B32-0B33, U+0B35-0B39, U+0B3C-0B44, U+0B47-0B48, U+0B4B-0B4D, U+0B55-0B57, U+0B5C-0B5D, U+0B5F-0B63, U+0B66-0B77;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansSinhala-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+0D81-0D83, U+0D85-0D96, U+0D9A-0DB1, U+0DB3-0DBB, U+0DBD, U+0DC0-0DC6, U+0DCA, U+0DCF-0DD4, U+0DD6, U+0DD8-0DDF, U+0DE6-0DEF, U+0DF2-0DF4, U+111E1-111F4;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansKhmer-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+1780-17DD, U+17E0-17E9, U+17F0-17F9, U+19E0-19FF;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansLao-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+0E81-0E82, U+0E84, U+0E86-0E8A, U+0E8C-0EA3, U+0EA5, U+0EA7-0EBD, U+0EC0-0EC4, U+0EC6, U+0EC8-0ECE, U+0ED0-0ED9, U+0EDC-0EDF;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansMyanmar-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+1000-109F, U+A9E0-A9FE, U+AA60-AA7F;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansArmenian-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+0531-0556, U+0559-058A, U+058D-058F, U+FB13-FB17;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansGeorgian-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+10A0-10C5, U+10C7, U+10CD, U+10D0-10FF, U+1C90-1CBA, U+1CBD-1CBF, U+2D00-2D25, U+2D27, U+2D2D;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansEthiopic-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+1200-1248, U+124A-124D, U+1250-1256, U+1258, U+125A-125D, U+1260-1288, U+128A-128D, U+1290-12B0, U+12B2-12B5, U+12B8-12BE, U+12C0, U+12C2-12C5, U+12C8-12D6, U+12D8-1310, U+1312-1315, U+1318-135A, U+135D-137C, U+1380-1399, U+2D80-2D96, U+2DA0-2DA6, U+2DA8-2DAE, U+2DB0-2DB6, U+2DB8-2DBE, U+2DC0-2DC6, U+2DC8-2DCE, U+2DD0-2DD6, U+2DD8-2DDE, U+AB01-AB06, U+AB09-AB0E, U+AB11-AB16, U+AB20-AB26, U+AB28-AB2E, U+1E7E0-1E7E6, U+1E7E8-1E7EB, U+1E7ED-1E7EE, U+1E7F0-1E7FE;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansThaana-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+0780-07B1;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansCherokee-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+13A0-13F5, U+13F8-13FD, U+AB70-ABBF;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansCanadianAboriginal-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+1400-167F, U+18B0-18F5, U+11AB0-11ABF;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansAdlam-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+1E900-1E94B, U+1E950-1E959, U+1E95E-1E95F;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansNKo-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+07C0-07FA, U+07FD-07FF;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansVai-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+A500-A62B;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansOsage-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+104B0-104D3, U+104D8-104FB;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansMongolian-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+1800-1819, U+1820-1878, U+1880-18AA, U+11660-1166C;
}
@font-face {
font-family: 'Fluxer Fallback';
src: url('./NotoSansYi-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+A000-A48C, U+A490-A4C6;
}
@font-face {
font-family: 'Fluxer Symbols';
src: url('./NotoSansSymbols-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+2190-2199, U+2300-230F, U+2311-2315, U+2317, U+231C-231F, U+2322-2323, U+2329-232A, U+232C-2335, U+237C, U+2380-2394, U+2396-239A, U+23AF, U+23BE-23CD, U+23D0-23DB, U+23E2-23E8, U+2460-24FF, U+25CC, U+260A-260D, U+2613, U+2624-262F, U+2638-263B, U+263D-2653, U+2669-267E, U+2690-269D, U+26A2-26A9, U+26AD-26BC, U+26CE, U+26E2-26FF, U+271D-2721, U+2776-2793, U+1F100-1F10C, U+1F110-1F16C, U+1F170-1F190, U+1F19B-1F1AC, U+1F546-1F549, U+1F54F, U+1F610, U+1F700-1F773;
}
@font-face {
font-family: 'Fluxer Symbols';
src: url('./NotoSansSymbols2-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+21AF, U+21E6-21F0, U+21F3, U+2316, U+2318, U+231A-231B, U+2324-2328, U+232B, U+237B, U+237D-237F, U+23CE-23CF, U+23E9-23EA, U+23ED-23EF, U+23F1-2426, U+2440-244A, U+25A0-25CB, U+25CD-2609, U+260E-2612, U+2614-2623, U+2630-2637, U+263C, U+2654-2668, U+267F-268F, U+269E-26A1, U+26AA-26AC, U+26BD-26CD, U+26CF-26E1, U+2700-2704, U+2706-2709, U+270B-271C, U+2722-2727, U+2729-274B, U+274D, U+274F-2753, U+2756-2775, U+2794, U+2798-27AF, U+27B1-27BE, U+2800-28FF, U+2B00-2B0D, U+2B12-2B2F, U+2B4D-2B73, U+2B76-2B95, U+2B97-2BFD, U+2BFF, U+1F000-1F02B, U+1F030-1F093, U+1F0A0-1F0AE, U+1F0B1-1F0BF, U+1F0C1-1F0CF, U+1F0D1-1F0F5, U+1F10D-1F10F, U+1F16D-1F16F, U+1F1AD, U+1F30D-1F30F, U+1F315, U+1F31C, U+1F321-1F32C, U+1F336, U+1F378, U+1F37D, U+1F393-1F39F, U+1F3A7, U+1F3AC-1F3AE, U+1F3C2, U+1F3C4, U+1F3C6, U+1F3CA-1F3CE, U+1F3D4-1F3E0, U+1F3ED, U+1F3F1-1F3F3, U+1F3F5-1F3F7, U+1F408, U+1F415, U+1F41F, U+1F426, U+1F43F, U+1F441-1F442, U+1F446-1F449, U+1F44C-1F44E, U+1F453, U+1F46A, U+1F47D, U+1F4A3, U+1F4B0, U+1F4B3, U+1F4B9, U+1F4BB, U+1F4BF, U+1F4C8-1F4CB, U+1F4DA, U+1F4DF, U+1F4E4-1F4E6, U+1F4EA-1F4ED, U+1F4F7, U+1F4F9-1F4FB, U+1F4FD-1F4FE, U+1F503, U+1F507-1F50A, U+1F50D, U+1F512-1F513, U+1F53E-1F545, U+1F54A, U+1F550-1F579, U+1F57B-1F594, U+1F597-1F5A3, U+1F5A5-1F5FA, U+1F650-1F67F, U+1F687, U+1F68D, U+1F691, U+1F694, U+1F698, U+1F6AD, U+1F6B2, U+1F6B9-1F6BA, U+1F6BC, U+1F6C6-1F6CB, U+1F6CD-1F6CF, U+1F6D3-1F6D7, U+1F6E0-1F6EA, U+1F6F0-1F6F3, U+1F6F7-1F6FC, U+1F774-1F776, U+1F77B-1F7D9, U+1F7E0-1F7EB, U+1F800-1F80B, U+1F810-1F847, U+1F850-1F859, U+1F860-1F887, U+1F890-1F8AD, U+1F8B0-1F8B1, U+1F900-1F90B, U+1F93B, U+1F946, U+1FA00-1FA53, U+1FA60-1FA6D, U+1FA70-1FA74, U+1FA78-1FA7A, U+1FA80-1FA86, U+1FA90-1FAA8, U+1FAB0-1FAB6, U+1FAC0-1FAC2, U+1FAD0-1FAD6, U+1FB00-1FB92, U+1FB94-1FBCA, U+1FBF0-1FBF9;
}
@font-face {
font-family: 'Fluxer Math';
src: url('./NotoSansMath-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
font-display: swap;
unicode-range: U+2102, U+210A-2113, U+2115, U+2119-211D, U+2124, U+2128, U+212C-212D, U+212F-2131, U+2133-2138, U+213C-2140, U+2145-2149, U+2200-22FF, U+27C0-27FF, U+2900-2AFF, U+1D400-1D454, U+1D456-1D49C, U+1D49E-1D49F, U+1D4A2, U+1D4A5-1D4A6, U+1D4A9-1D4AC, U+1D4AE-1D4B9, U+1D4BB, U+1D4BD-1D4C3, U+1D4C5-1D505, U+1D507-1D50A, U+1D50D-1D514, U+1D516-1D51C, U+1D51E-1D539, U+1D53B-1D53E, U+1D540-1D544, U+1D546, U+1D54A-1D550, U+1D552-1D6A5, U+1D6A8-1D7CB, U+1D7CE-1D7FF;
}
@@ -8,6 +8,7 @@ import {PreloadableUserPopout} from '@app/features/channel/components/Preloadabl
import type {VoiceState} from '@app/features/gateway/types/GatewayVoiceTypes';
import {isKeyboardActivationKey} from '@app/features/input/utils/KeyboardUtils';
import * as NavigationCommands from '@app/features/navigation/commands/NavigationCommands';
import type {VoiceParticipantMenuSource} from '@app/features/ui/action_menu/items/VoiceParticipantMenuTypes';
import {VoiceParticipantContextMenu} from '@app/features/ui/action_menu/VoiceParticipantContextMenu';
import {AvatarWithPresence} from '@app/features/ui/avatars/AvatarWithPresence';
import * as ContextMenuCommands from '@app/features/ui/commands/ContextMenuCommands';
@@ -70,6 +71,19 @@ export const GroupedVoiceParticipant = observer(function GroupedVoiceParticipant
const isContextMenuOpen = useContextMenuHoverState(rowRef);
const toggleExpanded = useCallback(() => setIsExpanded((prev) => !prev), []);
const connectionCount = voiceStates.length;
const containsLocalConnection =
currentConnectionId !== null && voiceStates.some((state) => state.connection_id === currentConnectionId);
const participantMenuSource = useMemo<VoiceParticipantMenuSource>(() => {
const hasCamera =
voiceStates.some((state) => state.self_video === true) ||
(isCurrentUser && containsLocalConnection && localSelfVideo);
if (hasCamera) return {kind: 'participant', focusSource: 'camera'};
const hasScreenShare =
voiceStates.some((state) => state.self_stream === true) ||
(isCurrentUser && containsLocalConnection && localSelfStream);
if (hasScreenShare) return {kind: 'participant', focusSource: 'screen-share'};
return {kind: 'participant'};
}, [containsLocalConnection, isCurrentUser, localSelfStream, localSelfVideo, voiceStates]);
const handleContextMenu = useCallback(
(event: React.MouseEvent) => {
event.preventDefault();
@@ -80,13 +94,16 @@ export const GroupedVoiceParticipant = observer(function GroupedVoiceParticipant
participantName={NicknameUtils.getNickname(user, guildId)}
onClose={onClose}
guildId={guildId}
surface="participant-list"
source={participantMenuSource}
isGroupedItem={true}
isParentGroupedItem={true}
groupContainsLocalConnection={containsLocalConnection}
data-flx="app.grouped-voice-participant.handle-context-menu.voice-participant-context-menu"
/>
));
},
[user, guildId],
[user, guildId, participantMenuSource, containsLocalConnection],
);
const stateAgg = useMemo(() => {
let anySpeaking = propAnySpeaking ?? false;
@@ -128,7 +145,7 @@ export const GroupedVoiceParticipant = observer(function GroupedVoiceParticipant
allSelfMuted = allSelfMuted && displayState.selfMute;
allSelfDeaf = allSelfDeaf && displayState.selfDeaf;
}
if (isCurrentUser) {
if (isCurrentUser && containsLocalConnection) {
anyCameraOn = anyCameraOn || localSelfVideo;
anyLive = anyLive || localSelfStream;
}
@@ -138,6 +155,7 @@ export const GroupedVoiceParticipant = observer(function GroupedVoiceParticipant
user.id,
guildId,
isCurrentUser,
containsLocalConnection,
currentConnectionId,
localSelfMute,
localSelfDeaf,
@@ -154,15 +172,19 @@ export const GroupedVoiceParticipant = observer(function GroupedVoiceParticipant
const live = state.self_stream === true || (participant ? participant.isScreenShareEnabled : false);
if (live) return state;
}
if (isCurrentUser && localSelfStream) {
return (
voiceStates.find((state) => state.connection_id === currentConnectionId) ??
voiceStates.find((state) => Boolean(state.connection_id)) ??
null
);
if (isCurrentUser && containsLocalConnection && localSelfStream) {
return voiceStates.find((state) => state.connection_id === currentConnectionId) ?? null;
}
return null;
}, [voiceStates, user.id, isCurrentUser, localSelfStream, currentConnectionId, mediaEngineVersion]);
}, [
voiceStates,
user.id,
isCurrentUser,
containsLocalConnection,
localSelfStream,
currentConnectionId,
mediaEngineVersion,
]);
const streamKey = activeStreamState?.connection_id
? getStreamKey(guildId, activeStreamState.channel_id ?? null, activeStreamState.connection_id)
: '';
@@ -10,6 +10,7 @@ import {PreloadableUserPopout} from '@app/features/channel/components/Preloadabl
import type {VoiceState} from '@app/features/gateway/types/GatewayVoiceTypes';
import * as NavigationCommands from '@app/features/navigation/commands/NavigationCommands';
import Permission from '@app/features/permissions/state/Permission';
import type {VoiceParticipantMenuSource} from '@app/features/ui/action_menu/items/VoiceParticipantMenuTypes';
import {VoiceParticipantContextMenu} from '@app/features/ui/action_menu/VoiceParticipantContextMenu';
import {AvatarWithPresence} from '@app/features/ui/avatars/AvatarWithPresence';
import * as ContextMenuCommands from '@app/features/ui/commands/ContextMenuCommands';
@@ -126,6 +127,11 @@ export const VoiceParticipantItem = observer(function VoiceParticipantItem({
const isActuallySpeaking = displayState.speaking;
const displayCameraOn = displayState.cameraOn;
const displayLive = displayState.streaming;
const participantMenuSource = useMemo<VoiceParticipantMenuSource>(() => {
if (displayCameraOn) return {kind: 'participant', focusSource: 'camera'};
if (displayLive) return {kind: 'participant', focusSource: 'screen-share'};
return {kind: 'participant'};
}, [displayCameraOn, displayLive]);
const streamKey = useMemo(
() => getStreamKey(guildId, currentChannelId, connectionId),
[guildId, currentChannelId, connectionId],
@@ -166,12 +172,14 @@ export const VoiceParticipantItem = observer(function VoiceParticipantItem({
onClose={onClose}
guildId={guildId}
connectionId={connectionId}
surface="participant-list"
source={participantMenuSource}
isGroupedItem={isGroupedItem}
data-flx="app.voice-participant-item.handle-context-menu.voice-participant-context-menu"
/>
));
},
[user, guildId, connectionId, currentChannelId],
[user, guildId, connectionId, currentChannelId, isGroupedItem, participantMenuSource],
);
const handleProfilePopoutOpen = useCallback(() => {
setIsProfilePopoutOpen(true);
@@ -318,6 +326,8 @@ export const VoiceParticipantItem = observer(function VoiceParticipantItem({
participant={participant}
guildId={guildId}
connectionId={connectionId}
surface="participant-list"
source={participantMenuSource}
isConnectionItem={isGroupedItem}
data-flx="app.voice-participant-item.voice-participant-bottom-sheet"
/>
@@ -83,9 +83,6 @@ export const Endpoints = {
CHANNEL_CALL: (channelId: string) => `/channels/${channelId}/call`,
CHANNEL_CALL_RING: (channelId: string) => `/channels/${channelId}/call/ring`,
CHANNEL_CALL_STOP_RINGING: (channelId: string) => `/channels/${channelId}/call/stop-ringing`,
CHANNEL_VOICE_DEBUG_LOGGING_SESSION: (channelId: string) => `/channels/${channelId}/voice-debug-logging/session`,
CHANNEL_VOICE_DEBUG_LOGGING_EVENTS: (channelId: string) => `/channels/${channelId}/voice-debug-logging/events`,
CHANNEL_VOICE_PRESENCE_HEARTBEAT: (channelId: string) => `/channels/${channelId}/voice-presence/heartbeat`,
GUILDS: '/guilds',
GUILD: (guildId: string) => `/guilds/${guildId}`,
GUILD_CHANNELS: (guildId: string) => `/guilds/${guildId}/channels`,
@@ -381,6 +381,12 @@ class KeybindManager {
private async checkInputMonitoringPermission(): Promise<boolean> {
if (!isNativeMacOS()) return true;
if (this.inputMonitoringHookStatus === 'granted') return true;
const electronApi = getElectronAPI();
if (await electronApi?.checkInputMonitoringAccess?.()) {
NativePermission.setInputMonitoringStatus('granted');
this.inputMonitoringHookStatus = 'granted';
return true;
}
const result = await ensureMacPermission('input-monitoring', {behavior: 'passive'});
switch (result) {
case 'granted':
@@ -81,6 +81,8 @@ export const CallParticipantsRow = observer(
onClose={onClose}
guildId={channel.guildId ?? undefined}
connectionId={participantEntry.connectionId}
surface="call-avatar"
source={{kind: 'participant'}}
data-flx="channel.channel-view.dm-channel-view.handle-context-menu.voice-participant-context-menu"
/>
) : (
@@ -408,7 +408,7 @@ export const EmbedGifv: FC<
},
[visibilityRef],
);
const {dimensions, style} = mediaCalculator.calculate({width: naturalWidth, height: naturalHeight});
const {dimensions} = mediaCalculator.calculate({width: naturalWidth, height: naturalHeight});
const canPlayInline = isInlinePlayableVideoSize({width: naturalWidth, height: naturalHeight});
const posterSource = thumbnailProxyURL && thumbnailProxyURL.length > 0 ? thumbnailProxyURL : videoProxyURL;
const posterURL = useMemo(() => {
@@ -534,8 +534,11 @@ export const EmbedGifv: FC<
safePause(video);
}, [shouldPlay]);
if (shouldBlur) {
const {width: _width, height: _height, ...styleWithoutDimensions} = style;
const blurContainerStyle = {...styleWithoutDimensions, maxWidth: '100%', width: '100%'};
const blurContainerStyle: React.CSSProperties = {
maxWidth: '100%',
width: remFromPx(dimensions.width),
...aspectRatioStyle,
};
return (
<div
ref={visibilityRef}
@@ -719,7 +722,7 @@ export const EmbedGif: FC<
},
[visibilityRef],
);
const {dimensions, style} = mediaCalculator.calculate({width: naturalWidth, height: naturalHeight});
const {dimensions} = mediaCalculator.calculate({width: naturalWidth, height: naturalHeight});
const {width: displayWidth, height: displayHeight} = dimensions;
const gifAutoPlay = useShouldAnimate({kind: 'gif'});
const animationPolicyAllowed = useShouldAnimate({
@@ -856,8 +859,11 @@ export const EmbedGif: FC<
],
);
if (shouldBlur) {
const {width: _width, height: _height, ...styleWithoutDimensions} = style;
const blurContainerStyle = {...styleWithoutDimensions, maxWidth: '100%', width: '100%'};
const blurContainerStyle: React.CSSProperties = {
maxWidth: '100%',
width: remFromPx(dimensions.width),
...aspectRatioStyle,
};
return (
<div
ref={visibilityRef}
@@ -174,7 +174,8 @@
cursor: pointer;
background: transparent;
}
.overwriteItem:not(.overwriteItemSelected):hover {
.overwriteItem:not(.overwriteItemSelected):hover,
.overwriteItemContextMenuOpen:not(.overwriteItemSelected) {
background: var(--background-modifier-hover);
}
.overwriteItemSelected {
@@ -519,27 +519,34 @@ const ChannelPermissionsTab: React.FC<{channelId: string}> = observer(({channelI
});
setSelectedOverwriteId(id);
}, []);
const handleDeleteOverride = useCallback(() => {
if (!selectedOverwrite || !guild || selectedOverwrite.id === guild.id) return;
const futureOverwrites = overwritesWithUpdates.filter((o) => o.id !== selectedOverwrite.id);
const currentIndex = overwritesWithUpdates.findIndex((o) => o.id === selectedOverwrite.id);
const nextOverwrite = futureOverwrites[currentIndex] ?? futureOverwrites[0];
if (newOverwriteIds.has(selectedOverwrite.id)) {
setNewOverwriteIds((prev) => {
const newSet = new Set(prev);
newSet.delete(selectedOverwrite.id);
return newSet;
const removeOverwrite = useCallback(
(overwriteId: string) => {
if (!guild || overwriteId === guild.id) return;
const futureOverwrites = overwritesWithUpdates.filter((o) => o.id !== overwriteId);
const currentIndex = overwritesWithUpdates.findIndex((o) => o.id === overwriteId);
const nextOverwrite = futureOverwrites[currentIndex] ?? futureOverwrites[0];
if (newOverwriteIds.has(overwriteId)) {
setNewOverwriteIds((prev) => {
const newSet = new Set(prev);
newSet.delete(overwriteId);
return newSet;
});
} else {
setDeletedOverwriteIds((prev) => new Set(prev).add(overwriteId));
}
setOverwriteUpdates((prev) => {
const newMap = new Map(prev);
newMap.delete(overwriteId);
return newMap;
});
} else {
setDeletedOverwriteIds((prev) => new Set(prev).add(selectedOverwrite.id));
}
setOverwriteUpdates((prev) => {
const newMap = new Map(prev);
newMap.delete(selectedOverwrite.id);
return newMap;
});
setSelectedOverwriteId(nextOverwrite?.id ?? null);
}, [selectedOverwrite, guild, overwritesWithUpdates, newOverwriteIds]);
setSelectedOverwriteId((prev) => (prev === overwriteId ? (nextOverwrite?.id ?? null) : prev));
},
[guild, overwritesWithUpdates, newOverwriteIds],
);
const handleDeleteOverride = useCallback(() => {
if (!selectedOverwrite) return;
removeOverwrite(selectedOverwrite.id);
}, [selectedOverwrite, removeOverwrite]);
const getOverwriteName = useCallback(
(overwrite: PermissionOverwrite): string => {
if (!guild) return '';
@@ -585,6 +592,8 @@ const ChannelPermissionsTab: React.FC<{channelId: string}> = observer(({channelI
selectedOverwriteId={selectedOverwriteId}
canManageChannels={canManageChannels}
canManageRoles={canManageRoles}
canManageOverwrites={canManageChannels && canManageRoles}
onDeleteOverwrite={removeOverwrite}
isAddOverrideOpen={isAddOverrideOpen}
setIsAddOverrideOpen={setIsAddOverrideOpen}
existingOverwriteIds={existingOverwriteIds}
@@ -615,6 +624,7 @@ const ChannelPermissionsTab: React.FC<{channelId: string}> = observer(({channelI
overwritesWithUpdates,
canManageChannels,
canManageRoles,
removeOverwrite,
isAddOverrideOpen,
addOverrideContext,
addOverrideFloatingStyles,
@@ -17,6 +17,8 @@ interface ChannelPermissionsSidebarProps {
selectedOverwriteId: string | null;
canManageChannels: boolean;
canManageRoles: boolean;
canManageOverwrites: boolean;
onDeleteOverwrite: (overwriteId: string) => void;
isAddOverrideOpen: boolean;
setIsAddOverrideOpen: (open: boolean) => void;
existingOverwriteIds: Set<string>;
@@ -40,6 +42,8 @@ export const ChannelPermissionsSidebar: React.FC<ChannelPermissionsSidebarProps>
selectedOverwriteId,
canManageChannels,
canManageRoles,
canManageOverwrites,
onDeleteOverwrite,
isAddOverrideOpen,
setIsAddOverrideOpen,
existingOverwriteIds,
@@ -136,6 +140,8 @@ export const ChannelPermissionsSidebar: React.FC<ChannelPermissionsSidebarProps>
roleId={roleId}
isSelected={isSelected}
isEveryone={isEveryone}
canDelete={canManageOverwrites && !isEveryone}
onDelete={onDeleteOverwrite}
onClick={() => onSelectOverwrite(overwrite.id)}
guildId={guildId}
data-flx="channel.channel-tabs.channel-permissions-tab.sidebar-content.overwrite-item.set-selected-overwrite-id"
@@ -0,0 +1,71 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {PermissionOverwrite} from '@app/features/channel/components/modals/channel_tabs/channel_permissions_tab/shared';
import {DeleteIcon} from '@app/features/ui/action_menu/ContextMenuIcons';
import {CopyRoleIdMenuItem, CopyUserIdMenuItem} from '@app/features/ui/action_menu/items/CopyMenuItems';
import {MenuGroup} from '@app/features/ui/action_menu/MenuGroup';
import {MenuItem} from '@app/features/ui/action_menu/MenuItem';
import type {User} from '@app/features/user/models/User';
import {msg} from '@lingui/core/macro';
import {useLingui} from '@lingui/react/macro';
import {observer} from 'mobx-react-lite';
import type React from 'react';
const DELETE_ROLE_DESCRIPTOR = msg({
message: 'Delete role',
comment:
'Danger action in the channel permissions overwrite context menu that removes a role permission override from the channel.',
});
const DELETE_USER_DESCRIPTOR = msg({
message: 'Delete user',
comment:
'Danger action in the channel permissions overwrite context menu that removes a member permission override from the channel.',
});
interface OverwriteContextMenuProps {
overwrite: PermissionOverwrite;
roleId: string | null;
user: User | null;
canDelete: boolean;
onDelete: () => void;
onClose: () => void;
}
export const OverwriteContextMenu: React.FC<OverwriteContextMenuProps> = observer(
({overwrite, roleId, user, canDelete, onDelete, onClose}) => {
const {i18n} = useLingui();
return (
<>
<MenuGroup data-flx="channel.channel-tabs.channel-permissions-tab.overwrite-context-menu.menu-group">
{overwrite.type === 0 && roleId ? (
<CopyRoleIdMenuItem
roleId={roleId}
onClose={onClose}
data-flx="channel.channel-tabs.channel-permissions-tab.overwrite-context-menu.copy-role-id-menu-item"
/>
) : user ? (
<CopyUserIdMenuItem
user={user}
onClose={onClose}
data-flx="channel.channel-tabs.channel-permissions-tab.overwrite-context-menu.copy-user-id-menu-item"
/>
) : null}
</MenuGroup>
{canDelete && (
<MenuGroup data-flx="channel.channel-tabs.channel-permissions-tab.overwrite-context-menu.menu-group--2">
<MenuItem
icon={
<DeleteIcon data-flx="channel.channel-tabs.channel-permissions-tab.overwrite-context-menu.delete-icon" />
}
danger
onClick={onDelete}
data-flx="channel.channel-tabs.channel-permissions-tab.overwrite-context-menu.menu-item.delete"
>
{overwrite.type === 0 ? i18n._(DELETE_ROLE_DESCRIPTOR) : i18n._(DELETE_USER_DESCRIPTOR)}
</MenuItem>
</MenuGroup>
)}
</>
);
},
);
@@ -1,16 +1,18 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {DEFAULT_ROLE_COLOR_HEX, getRoleColor} from '@app/features/app/components/dialogs/shared/PermissionComponents';
import {useContextMenuHoverState} from '@app/features/app/hooks/useContextMenuHoverState';
import styles from '@app/features/channel/components/modals/channel_tabs/ChannelPermissionsTab.module.css';
import {OverwriteContextMenu} from '@app/features/channel/components/modals/channel_tabs/channel_permissions_tab/OverwriteContextMenu';
import type {PermissionOverwrite} from '@app/features/channel/components/modals/channel_tabs/channel_permissions_tab/shared';
import {openRoleContextMenu} from '@app/features/ui/action_menu/RoleContextMenu';
import * as ContextMenuCommands from '@app/features/ui/commands/ContextMenuCommands';
import {Avatar} from '@app/features/ui/components/Avatar';
import type {User} from '@app/features/user/models/User';
import {UsersIcon} from '@phosphor-icons/react';
import {clsx} from 'clsx';
import {observer} from 'mobx-react-lite';
import type React from 'react';
import {useCallback} from 'react';
import {useCallback, useRef} from 'react';
interface OverwriteItemProps {
overwrite: PermissionOverwrite;
@@ -20,26 +22,43 @@ interface OverwriteItemProps {
roleId?: string | null;
isSelected: boolean;
isEveryone: boolean;
canDelete: boolean;
onDelete: (overwriteId: string) => void;
onClick: () => void;
guildId: string;
}
export const OverwriteItem: React.FC<OverwriteItemProps> = observer(
({overwrite, name, color, user, roleId, isSelected, isEveryone, onClick, guildId}) => {
({overwrite, name, color, user, roleId, isSelected, isEveryone, canDelete, onDelete, onClick, guildId}) => {
const buttonRef = useRef<HTMLButtonElement>(null);
const contextMenuOpen = useContextMenuHoverState(buttonRef);
const handleContextMenu = useCallback(
(event: React.MouseEvent<HTMLButtonElement>) => {
if (!roleId) return;
openRoleContextMenu(event, roleId);
ContextMenuCommands.openFromEvent(event, ({onClose}) => (
<OverwriteContextMenu
overwrite={overwrite}
roleId={roleId ?? null}
user={user ?? null}
canDelete={canDelete}
onDelete={() => onDelete(overwrite.id)}
onClose={onClose}
data-flx="channel.channel-tabs.channel-permissions-tab.overwrite-item.handle-context-menu.overwrite-context-menu"
/>
));
},
[roleId],
[overwrite, roleId, user, canDelete, onDelete],
);
return (
<button
ref={buttonRef}
type="button"
aria-pressed={isSelected}
className={clsx(styles.overwriteItem, {[styles.overwriteItemSelected]: isSelected})}
className={clsx(styles.overwriteItem, {
[styles.overwriteItemSelected]: isSelected,
[styles.overwriteItemContextMenuOpen]: contextMenuOpen && !isSelected,
})}
onClick={onClick}
onContextMenu={roleId ? handleContextMenu : undefined}
onContextMenu={handleContextMenu}
data-flx="channel.channel-tabs.channel-permissions-tab.overwrite-item.overwrite-item.click.button"
>
{overwrite.type === 0 && !isEveryone ? (
@@ -21,7 +21,8 @@
}
.button:hover:not(:disabled),
.button.contextMenuHover:not(:disabled) {
.button.contextMenuHover:not(:disabled),
.button.selected:not(:disabled) {
color: var(--text-primary);
background-color: var(--background-modifier-hover);
}
@@ -31,10 +32,6 @@
opacity: 0.7;
}
.button.selected {
color: var(--text-primary);
}
.icon {
width: var(--textarea-button-icon-size);
height: var(--textarea-button-icon-size);
@@ -1,5 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import * as AccessibilityCommands from '@app/features/accessibility/commands/AccessibilityCommands';
import Accessibility from '@app/features/accessibility/state/Accessibility';
import {PRODUCT_NAME} from '@app/features/app/config/I18nDisplayConstants';
import Spellcheck from '@app/features/messaging/state/Spellcheck';
import {isEditableTextInput, replaceSelectedText} from '@app/features/messaging/utils/TextInputEditUtils';
@@ -97,6 +99,11 @@ const SPELLCHECK_SETTINGS_DESCRIPTOR = msg({
'Item in the textarea spellcheck submenu that opens the full settings page. Trailing horizontal ellipsis (…) indicates the action opens settings.',
});
const SHOW_SEND_BUTTON_DESCRIPTOR = msg({
message: 'Show send button',
comment: 'Short label in the channel and chat textarea plus menu. Keep it concise.',
});
export interface TextareaContextMenuEditFlags {
canUndo: boolean;
canRedo: boolean;
@@ -111,12 +118,21 @@ interface TextareaContextMenuProps {
suggestions?: Array<string>;
editFlags?: TextareaContextMenuEditFlags;
targetElement?: HTMLElement | null;
showSendButtonToggle?: boolean;
onClose: () => void;
}
export const TextareaContextMenu = observer(
({misspelledWord, suggestions = [], editFlags, targetElement, onClose}: TextareaContextMenuProps) => {
({
misspelledWord,
suggestions = [],
editFlags,
targetElement,
showSendButtonToggle = false,
onClose,
}: TextareaContextMenuProps) => {
const {i18n} = useLingui();
const showMessageSendButton = Accessibility.showMessageSendButton;
const electronAPI = isElectron() ? getElectronAPI() : null;
const focusTargetElement = () => {
if (!targetElement?.isConnected) {
@@ -301,6 +317,18 @@ export const TextareaContextMenu = observer(
data-flx="channel.textarea.textarea-context-menu.menu-item-submenu"
/>
</MenuGroup>
{showSendButtonToggle && (
<MenuGroup data-flx="channel.textarea.textarea-context-menu.menu-group--6">
<CheckboxItem
checked={showMessageSendButton}
onCheckedChange={(checked) => AccessibilityCommands.update({showMessageSendButton: checked})}
closeOnChange={false}
data-flx="channel.textarea.textarea-context-menu.checkbox-item.show-send-button"
>
{i18n._(SHOW_SEND_BUTTON_DESCRIPTOR)}
</CheckboxItem>
</MenuGroup>
)}
</>
);
},
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {BAN_DELETE_MESSAGE_OPTIONS} from '@app/features/moderation/constants/BanDeleteMessageOptions';
import {Permissions} from '@fluxer/constants/src/ChannelConstants';
import {msg} from '@lingui/core/macro';
import {useLingui} from '@lingui/react/macro';
@@ -83,39 +84,6 @@ const COMMAND_DELETE_MESSAGES_OPTION_DESCRIPTOR = msg({
message: "How much of the member's recent message history to delete.",
comment: 'Description for the /ban delete_messages option.',
});
const DELETE_MESSAGES_NONE_DESCRIPTOR = msg({
message: "Don't delete any",
comment: 'Choice label for retaining all messages when banning a member.',
});
const DELETE_MESSAGES_ONE_DAY_DESCRIPTOR = msg({
message: 'Previous 24 hours',
comment: 'Choice label for deleting one day of messages when banning a member.',
});
const DELETE_MESSAGES_TWO_DAYS_DESCRIPTOR = msg({
message: 'Previous 2 days',
comment: 'Choice label for deleting two days of messages when banning a member.',
});
const DELETE_MESSAGES_THREE_DAYS_DESCRIPTOR = msg({
message: 'Previous 3 days',
comment: 'Choice label for deleting three days of messages when banning a member.',
});
const DELETE_MESSAGES_FOUR_DAYS_DESCRIPTOR = msg({
message: 'Previous 4 days',
comment: 'Choice label for deleting four days of messages when banning a member.',
});
const DELETE_MESSAGES_FIVE_DAYS_DESCRIPTOR = msg({
message: 'Previous 5 days',
comment: 'Choice label for deleting five days of messages when banning a member.',
});
const DELETE_MESSAGES_SIX_DAYS_DESCRIPTOR = msg({
message: 'Previous 6 days',
comment: 'Choice label for deleting six days of messages when banning a member.',
});
const DELETE_MESSAGES_SEVEN_DAYS_DESCRIPTOR = msg({
message: 'Previous 7 days',
comment: 'Choice label for deleting seven days of messages when banning a member.',
});
interface SimpleCommand {
type: 'simple';
name: string;
@@ -271,16 +239,10 @@ export function useCommands(): Array<Command> {
type: 'choice',
required: true,
allowEmpty: false,
choices: [
{name: i18n._(DELETE_MESSAGES_NONE_DESCRIPTOR), value: '0'},
{name: i18n._(DELETE_MESSAGES_ONE_DAY_DESCRIPTOR), value: '1'},
{name: i18n._(DELETE_MESSAGES_TWO_DAYS_DESCRIPTOR), value: '2'},
{name: i18n._(DELETE_MESSAGES_THREE_DAYS_DESCRIPTOR), value: '3'},
{name: i18n._(DELETE_MESSAGES_FOUR_DAYS_DESCRIPTOR), value: '4'},
{name: i18n._(DELETE_MESSAGES_FIVE_DAYS_DESCRIPTOR), value: '5'},
{name: i18n._(DELETE_MESSAGES_SIX_DAYS_DESCRIPTOR), value: '6'},
{name: i18n._(DELETE_MESSAGES_SEVEN_DAYS_DESCRIPTOR), value: '7'},
],
choices: BAN_DELETE_MESSAGE_OPTIONS.map((option) => ({
name: i18n._(option.label),
value: String(option.seconds),
})),
},
{
name: 'reason',
@@ -7,6 +7,10 @@ import * as GuildMemberCommands from '@app/features/member/commands/GuildMemberC
import GuildMembers from '@app/features/member/state/GuildMembers';
import * as MessageCommands from '@app/features/messaging/commands/MessageCommands';
import {Message} from '@app/features/messaging/models/MessagingMessage';
import {
BAN_DELETE_MESSAGE_SECONDS_CHOICE_VALUES,
DEFAULT_BAN_DELETE_MESSAGE_SECONDS,
} from '@app/features/moderation/constants/BanDeleteMessageOptions';
import {Logger} from '@app/features/platform/utils/AppLogger';
import {User} from '@app/features/user/models/User';
import Users from '@app/features/user/state/Users';
@@ -43,7 +47,7 @@ export type ParsedCommand =
| {
type: 'ban';
userId: string;
deleteMessageDays: number;
deleteMessageSeconds: number;
duration: number;
reason?: string;
}
@@ -101,11 +105,11 @@ export function parseCommand(content: string): ParsedCommand {
const userId = userMatch[1];
const afterMention = rest.slice(userMatch[0].length).trim();
const parts = afterMention.length === 0 ? [] : afterMention.split(/\s+/);
let deleteMessageDays = 1;
let deleteMessageSeconds = DEFAULT_BAN_DELETE_MESSAGE_SECONDS;
let reasonStart = 0;
const firstPart = parts[0];
if (firstPart !== undefined && /^[0-7]$/.test(firstPart)) {
deleteMessageDays = Number(firstPart);
if (firstPart !== undefined && BAN_DELETE_MESSAGE_SECONDS_CHOICE_VALUES.has(firstPart)) {
deleteMessageSeconds = Number(firstPart);
reasonStart = 1;
} else if (firstPart !== undefined && /^\d+$/.test(firstPart)) {
return {type: 'unknown'};
@@ -114,7 +118,7 @@ export function parseCommand(content: string): ParsedCommand {
const reasonParts = parts.slice(reasonStart);
const reasonText = reasonParts.join(' ').trim();
const reason = reasonText || undefined;
return {type: 'ban', userId, deleteMessageDays, duration, reason};
return {type: 'ban', userId, deleteMessageSeconds, duration, reason};
}
if (trimmed.startsWith('/msg ')) {
const rest = trimmed.slice(5).trim();
@@ -310,7 +314,7 @@ export async function executeCommand(
await GuildCommands.banMember(
guildId,
command.userId,
command.deleteMessageDays,
command.deleteMessageSeconds,
command.reason,
command.duration,
);
@@ -23,10 +23,6 @@ import {MagnifyingGlassIcon} from '@phosphor-icons/react';
import {observer} from 'mobx-react-lite';
import {useCallback, useState} from 'react';
const I_CAN_T_REDEEM_THIS_BECAUSE_I_ALREADY_DESCRIPTOR = msg({
message: "I can't redeem this because I already have lifetime {premiumProductName}, so this gift is all yours!",
comment: 'Gift redemption pre-filled DM message for a sender who already owns lifetime premium.',
});
const YOU_ALREADY_HAVE_LIFETIME_DESCRIPTOR = msg({
message: 'You already have lifetime {premiumProductName}',
comment: 'Status label shown to a user who already owns the lifetime premium tier.',
@@ -62,7 +58,7 @@ export const GiftSendToFriendModal = observer(function GiftSendToFriendModal({co
? item.channelId
: await PrivateChannelCommands.ensureDMChannel(item.user.id);
const result = await MessageCommands.send(targetChannelId, {
content: `${i18n._(I_CAN_T_REDEEM_THIS_BECAUSE_I_ALREADY_DESCRIPTOR, {premiumProductName: PREMIUM_PRODUCT_NAME})}\n${giftUrl}`,
content: giftUrl,
nonce: SnowflakeUtils.fromTimestamp(Date.now()),
});
if (result) {
@@ -79,7 +75,7 @@ export const GiftSendToFriendModal = observer(function GiftSendToFriendModal({co
});
}
},
[giftUrl, i18n],
[giftUrl],
);
return (
<Modal.Root size="small" centered data-flx="expressions.gift-send-to-friend-modal.modal-root">
@@ -51,7 +51,7 @@ interface GuildTemplateCreateParams {
}
interface BanMemberRequest {
delete_message_days: number;
delete_message_seconds: number;
reason: string | null;
ban_duration_seconds?: number;
}
@@ -124,9 +124,13 @@ function transferOwnershipRequest(newOwnerId: string): {new_owner_id: string} {
return {new_owner_id: newOwnerId};
}
function banMemberRequest(deleteMessageDays?: number, reason?: string, banDurationSeconds?: number): BanMemberRequest {
function banMemberRequest(
deleteMessageSeconds?: number,
reason?: string,
banDurationSeconds?: number,
): BanMemberRequest {
return {
delete_message_days: deleteMessageDays ?? 0,
delete_message_seconds: deleteMessageSeconds ?? 0,
reason: reason ?? null,
ban_duration_seconds: banDurationSeconds,
};
@@ -250,10 +254,18 @@ export async function updateVanityURL(guildId: string, code: string | null): Pro
}
}
export async function createRole(guildId: string, name: string): Promise<void> {
export async function createRole(
guildId: string,
name: string,
options?: {color?: number; permissions?: bigint},
): Promise<GuildRole> {
try {
await http.post(Endpoints.GUILD_ROLES(guildId), {body: {name}});
const body: {name: string; color?: number; permissions?: string} = {name};
if (options?.color !== undefined) body.color = options.color;
if (options?.permissions !== undefined) body.permissions = options.permissions.toString();
const response = await http.post<GuildRole>(Endpoints.GUILD_ROLES(guildId), {body});
logger.debug(`Created role "${name}" in guild ${guildId}`);
return response.body;
} catch (error) {
logger.error(`Failed to create role in guild ${guildId}:`, error);
throw error;
@@ -373,13 +385,13 @@ export async function transferOwnership(guildId: string, newOwnerId: string): Pr
export async function banMember(
guildId: string,
userId: string,
deleteMessageDays?: number,
deleteMessageSeconds?: number,
reason?: string,
banDurationSeconds?: number,
): Promise<void> {
try {
await http.put(Endpoints.GUILD_BAN(guildId, userId), {
body: banMemberRequest(deleteMessageDays, reason, banDurationSeconds),
body: banMemberRequest(deleteMessageSeconds, reason, banDurationSeconds),
});
logger.debug(`Banned user ${userId} from guild ${guildId}`);
} catch (error) {
@@ -262,7 +262,8 @@
background: transparent;
}
.overwriteItem:not(.overwriteItemSelected):hover {
.overwriteItem:not(.overwriteItemSelected):hover,
.overwriteItem:not(.overwriteItemSelected).overwriteItemHovered {
background: var(--background-modifier-hover);
}
@@ -28,6 +28,7 @@ import {RoleUpdateFailedModal} from '@app/features/moderation/components/alerts/
import Permission from '@app/features/permissions/state/Permission';
import PermissionLayout from '@app/features/permissions/state/PermissionLayout';
import * as PermissionUtils from '@app/features/permissions/utils/PermissionUtils';
import {Logger} from '@app/features/platform/utils/AppLogger';
import * as ModalCommands from '@app/features/ui/commands/ModalCommands';
import {modal} from '@app/features/ui/commands/ModalCommands';
import * as ToastCommands from '@app/features/ui/commands/ToastCommands';
@@ -43,10 +44,16 @@ import {observer} from 'mobx-react-lite';
import type React from 'react';
import {useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState} from 'react';
const logger = new Logger('GuildRolesTab');
const NEW_ROLE_DESCRIPTOR = msg({
message: 'New role',
comment: 'Default name for a newly created role in the community roles settings tab. Short label.',
});
const ROLE_COPY_NAME_DESCRIPTOR = msg({
message: '{name} copy',
comment:
'Default name for a role created by duplicating another role in the community roles settings tab. {name} is the source role name; translate the whole label so "copy" can move or inflect as the language needs.',
});
const DELETE_ROLE_DESCRIPTOR = msg({
message: 'Delete role',
comment:
@@ -85,6 +92,7 @@ const GuildRolesTab: React.FC<{guildId: string}> = observer(({guildId}) => {
const [mobileShowEditor, setMobileShowEditor] = useState(false);
const [roleUpdates, setRoleUpdates] = useState<Map<string, RoleUpdate>>(new Map());
const [pendingRoleOrder, setPendingRoleOrder] = useState<Array<string> | null>(null);
const [optimisticRoleOrder, setOptimisticRoleOrder] = useState<Array<string> | null>(null);
const [hoistOrderMode, setHoistOrderMode] = useState(false);
const [pendingHoistOrder, setPendingHoistOrder] = useState<Array<string> | null>(null);
const [permissionSearchQuery, setPermissionSearchQuery] = useState('');
@@ -98,13 +106,16 @@ const GuildRolesTab: React.FC<{guildId: string}> = observer(({guildId}) => {
if (!guild) return [];
const rolesList = Object.values(guild.roles);
const sorted = sortRolesByPosition(rolesList);
if (pendingRoleOrder) {
return pendingRoleOrder
const order = pendingRoleOrder ?? optimisticRoleOrder;
if (order) {
const mapped = order
.map((id: string) => sorted.find((r: GuildRole) => r.id === id))
.filter((r): r is GuildRole => r != null);
const missing = sorted.filter((r: GuildRole) => !order.includes(r.id));
return [...mapped, ...missing];
}
return sorted;
}, [guild, pendingRoleOrder]);
}, [guild, pendingRoleOrder, optimisticRoleOrder]);
useEffect(() => {
if (!pendingRoleOrder || !guild) return;
const sortedIds = sortRolesByPosition(Object.values(guild.roles)).map((role: GuildRole) => role.id);
@@ -112,6 +123,13 @@ const GuildRolesTab: React.FC<{guildId: string}> = observer(({guildId}) => {
const matches = pendingRoleOrder.every((id, index) => id === sortedIds[index]);
if (matches) setPendingRoleOrder(null);
}, [pendingRoleOrder, guild]);
useEffect(() => {
if (!optimisticRoleOrder || !guild) return;
const sortedIds = sortRolesByPosition(Object.values(guild.roles)).map((role: GuildRole) => role.id);
if (optimisticRoleOrder.length !== sortedIds.length) return;
const matches = optimisticRoleOrder.every((id, index) => id === sortedIds[index]);
if (matches) setOptimisticRoleOrder(null);
}, [optimisticRoleOrder, guild]);
const hoistedRoles = useMemo(() => {
if (!guild) return [];
const rolesList = Object.values(guild.roles).filter((role) => role.hoist && !role.isEveryone);
@@ -341,6 +359,59 @@ const GuildRolesTab: React.FC<{guildId: string}> = observer(({guildId}) => {
);
}
}, [guild]);
const handleDuplicateRole = useCallback(
async (sourceRoleId: string) => {
if (!guild || !canManageRoles) return;
const sourceRole = guild.roles[sourceRoleId];
if (!sourceRole) return;
const copyName = i18n._(ROLE_COPY_NAME_DESCRIPTOR, {name: sourceRole.name}).slice(0, 100);
const grantablePermissions = sourceRole.permissions & currentUserPermissions;
pendingRoleCreationRef.current = true;
try {
const createdRole = await GuildCommands.createRole(guild.id, copyName, {
color: sourceRole.color,
permissions: grantablePermissions,
});
const orderedRoleIds = roles.map((role: GuildRole) => role.id).filter((id) => id !== createdRole.id);
const sourceIndex = orderedRoleIds.indexOf(sourceRoleId);
const insertIndex = sourceIndex === -1 ? orderedRoleIds.length : sourceIndex + 1;
orderedRoleIds.splice(insertIndex, 0, createdRole.id);
setOptimisticRoleOrder(orderedRoleIds);
await GuildCommands.updateRole(guild.id, createdRole.id, {
hoist: sourceRole.hoist,
mentionable: sourceRole.mentionable,
});
try {
const submittableRoleOrder = createSubmittableRoleOrderIds({
guildId: guild.id,
orderedRoleIds,
isRoleLocked: (roleId) => {
if (roleId === createdRole.id) return false;
const role = guild.roles[roleId];
if (!role) return true;
return isRoleLocked(role);
},
});
if (submittableRoleOrder.length > 0) {
await GuildCommands.setRoleOrder(guild.id, submittableRoleOrder);
}
} catch (error) {
logger.error(`Failed to position duplicated role in guild ${guild.id}:`, error);
setOptimisticRoleOrder(null);
}
ToastCommands.createToast({type: 'success', children: <Trans>Role created successfully</Trans>});
} catch (_error) {
pendingRoleCreationRef.current = false;
setOptimisticRoleOrder(null);
ModalCommands.push(
modal(() => (
<RoleCreateFailedModal data-flx="guild.guild-tabs.guild-roles-tab.handle-duplicate-role.role-create-failed-modal" />
)),
);
}
},
[guild, canManageRoles, roles, isRoleLocked, currentUserPermissions, i18n],
);
const handleDeleteRole = useCallback(() => {
if (!selectedRole || selectedRole.isEveryone || !guild) return;
const currentIndex = roles.findIndex((r: GuildRole) => r.id === selectedRole.id);
@@ -386,6 +457,30 @@ const GuildRolesTab: React.FC<{guildId: string}> = observer(({guildId}) => {
)),
);
}, [selectedRole, guild, roles]);
const handleContextMenuDeleteRole = useCallback(
async (roleId: string) => {
if (!guild) return;
const role = guild.roles[roleId];
if (!role || role.isEveryone || isRoleLocked(role)) return;
const currentIndex = roles.findIndex((r: GuildRole) => r.id === roleId);
const nextRole = roles[currentIndex + 1] ?? roles[0];
try {
await GuildCommands.deleteRole(guild.id, roleId);
ToastCommands.createToast({type: 'success', children: <Trans>Role deleted successfully</Trans>});
setSelectedRoleId((current) => (current === roleId ? (nextRole?.id ?? null) : current));
} catch (_error) {
ModalCommands.push(
modal(() => (
<RoleDeleteFailedModal
roleName={role.name}
data-flx="guild.guild-tabs.guild-roles-tab.handle-context-menu-delete-role.role-delete-failed-modal"
/>
)),
);
}
},
[guild, roles, isRoleLocked],
);
const evaluateRoleMove = useCallback(
(draggedRoleId: string, targetRoleId: string | null, position: 'before' | 'after') => {
if (!guild) return null;
@@ -500,6 +595,8 @@ const GuildRolesTab: React.FC<{guildId: string}> = observer(({guildId}) => {
isRoleLocked={isRoleLocked}
onSelectRole={setSelectedRoleId}
onCreateRole={handleCreateRole}
onDuplicateRole={handleDuplicateRole}
onDeleteRole={handleContextMenuDeleteRole}
onEnterHoistOrderMode={handleEnterHoistOrderMode}
onExitHoistOrderMode={handleExitHoistOrderMode}
onResetHoistOrder={handleResetHoistOrder}
@@ -519,6 +616,8 @@ const GuildRolesTab: React.FC<{guildId: string}> = observer(({guildId}) => {
isGuildOwner,
canManageRoles,
handleCreateRole,
handleDuplicateRole,
handleContextMenuDeleteRole,
evaluateRoleMove,
handleRoleDrop,
evaluateHoistMove,
@@ -2,6 +2,7 @@
import {DEFAULT_ROLE_COLOR_HEX, getRoleColor} from '@app/features/app/components/dialogs/shared/PermissionComponents';
import {DropIndicator} from '@app/features/app/components/layout/DropIndicator';
import {useContextMenuHoverState} from '@app/features/app/hooks/useContextMenuHoverState';
import {useMergeRefs} from '@app/features/app/hooks/useMergeRefs';
import styles from '@app/features/guild/components/modals/guild_tabs/GuildRolesTab.module.css';
import {
@@ -17,6 +18,7 @@ import {
} from '@app/features/guild/components/modals/guild_tabs/RoleReorderStateMachine';
import type {GuildRole} from '@app/features/guild/models/GuildRole';
import {openRoleContextMenu} from '@app/features/ui/action_menu/RoleContextMenu';
import MobileLayout from '@app/features/ui/state/MobileLayout';
import {Tooltip} from '@app/features/ui/tooltip/Tooltip';
import {msg} from '@lingui/core/macro';
import {useLingui} from '@lingui/react/macro';
@@ -43,6 +45,8 @@ interface RoleItemProps {
isTerminal: boolean;
canManageRoles: boolean;
onClick: () => void;
onDuplicate: (roleId: string) => void;
onDelete: (roleId: string) => void;
onEvaluateMove: (
draggedRoleId: string,
targetRoleId: string | null,
@@ -52,15 +56,34 @@ interface RoleItemProps {
}
export const RoleItem: React.FC<RoleItemProps> = observer(
({role, isSelected, isLocked, isGuildOwner, isTerminal, canManageRoles, onClick, onEvaluateMove, onCommitMove}) => {
({
role,
isSelected,
isLocked,
isGuildOwner,
isTerminal,
canManageRoles,
onClick,
onDuplicate,
onDelete,
onEvaluateMove,
onCommitMove,
}) => {
const {i18n} = useLingui();
const elementRef = useRef<HTMLButtonElement | null>(null);
const [dropIndicator, setDropIndicator] = useState<{position: 'top' | 'bottom'; isValid: boolean} | null>(null);
const contextMenuOpen = useContextMenuHoverState(elementRef, !MobileLayout.enabled);
const canDelete = canManageRoles && !role.isEveryone && !isLocked;
const handleContextMenu = useCallback(
(event: React.MouseEvent<HTMLButtonElement>) => {
openRoleContextMenu(event, role.id);
openRoleContextMenu(event, role.id, {
canDuplicate: canManageRoles,
onDuplicate: () => onDuplicate(role.id),
canDelete,
onDelete: () => onDelete(role.id),
});
},
[role.id],
[role.id, canManageRoles, canDelete, onDuplicate, onDelete],
);
const dragItem = useMemo<RoleDragItem>(
() => ({type: ROLE_DND_TYPE, id: role.id, isEveryone: role.isEveryone, isLocked}),
@@ -204,6 +227,7 @@ export const RoleItem: React.FC<RoleItemProps> = observer(
styles.overwriteItem,
styles.roleButton,
{[styles.overwriteItemSelected]: isSelected},
contextMenuOpen && styles.overwriteItemHovered,
isDragging && styles.dragging,
!canDrag && styles.noDrag,
)}
@@ -44,6 +44,8 @@ interface RoleSidebarProps {
isRoleLocked: (role: GuildRole) => boolean;
onSelectRole: (roleId: string) => void;
onCreateRole: () => void;
onDuplicateRole: (roleId: string) => void;
onDeleteRole: (roleId: string) => void;
onEnterHoistOrderMode: () => void;
onExitHoistOrderMode: () => void;
onResetHoistOrder: () => void;
@@ -74,6 +76,8 @@ export const RoleSidebar: React.FC<RoleSidebarProps> = observer(
isRoleLocked,
onSelectRole,
onCreateRole,
onDuplicateRole,
onDeleteRole,
onEnterHoistOrderMode,
onExitHoistOrderMode,
onResetHoistOrder,
@@ -155,6 +159,8 @@ export const RoleSidebar: React.FC<RoleSidebarProps> = observer(
isTerminal={index === hoistedRoles.length - 1}
canManageRoles={canManageRoles}
onClick={() => onSelectRole(role.id)}
onDuplicate={onDuplicateRole}
onDelete={onDeleteRole}
onEvaluateMove={onEvaluateHoistMove}
onCommitMove={onHoistDrop}
data-flx="guild.guild-tabs.guild-roles-tab.sidebar-content.role-item.set-selected-role-id"
@@ -251,6 +257,8 @@ export const RoleSidebar: React.FC<RoleSidebarProps> = observer(
isTerminal={index === roles.length - 1}
canManageRoles={canManageRoles}
onClick={() => onSelectRole(role.id)}
onDuplicate={onDuplicateRole}
onDelete={onDeleteRole}
onEvaluateMove={onEvaluateRoleMove}
onCommitMove={onRoleDrop}
data-flx="guild.guild-tabs.guild-roles-tab.sidebar-content.role-item.set-selected-role-id--2"
File diff suppressed because it is too large Load Diff
@@ -1347,6 +1347,9 @@
{
"msgid": "Decide how premium limits apply to people on this instance."
},
{
"msgid": "Delete user"
},
{
"msgid": "Denied"
},
@@ -1356,6 +1359,9 @@
{
"msgid": "Disable direct messages and friend requests"
},
{
"msgid": "Duplicate role"
},
{
"msgid": "Emulate decode codec (testing)"
},
@@ -1410,6 +1416,9 @@
{
"msgid": "Give everyone on this instance the highest premium limits."
},
{
"msgid": "Global shortcut"
},
{
"msgid": "Grant"
},
@@ -1773,6 +1782,9 @@
{
"msgid": "Use video and camera previews."
},
{
"msgid": "Use your operating system's spellchecker when available. Otherwise, use {productName}'s in-app dictionaries."
},
{
"msgid": "Volume sliders"
},
@@ -1830,6 +1842,9 @@
{
"msgid": "{count, plural, one {# minute} other {# minutes}}"
},
{
"msgid": "{name} copy"
},
{
"msgid": "{productName} application logo"
},
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More