mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-17 13:05:32 +09:00
Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8bda151596 | ||
|
|
ab19c0d1ac | ||
|
|
68cb2a4bbb | ||
|
|
17746fcdc4 | ||
|
|
c890956e4f | ||
|
|
8a5d4a8bdb | ||
|
|
0ba15ee46a | ||
|
|
85d9ef7386 | ||
|
|
f8d85f0e5e |
No files matched your search
@@ -157,8 +157,8 @@ MEILI_MASTER_KEY=CHANGE_ME
|
||||
#FLUXER_GEOIP_DB_PATH=
|
||||
|
||||
# The client address. The edge sets X-Forwarded-For on every hop, so keep the
|
||||
# trust on and the default header. Turning the trust off makes the api refuse
|
||||
# every request outside its exempt routes with a 403.
|
||||
# trust on and the default header. Turning the trust off makes the api ignore
|
||||
# the header and use the connection's peer address, which here is the edge.
|
||||
#FLUXER_CLIENT_IP_HEADER_NAME=x-forwarded-for
|
||||
#FLUXER_TRUST_CLIENT_IP_HEADER=true
|
||||
|
||||
|
||||
Generated
+1
-1
@@ -3074,7 +3074,7 @@
|
||||
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
|
||||
}
|
||||
},
|
||||
"description": "Permanently deletes a guild. Deletes all channels, messages, and settings. Irreversible operation with no recovery window. Logged to audit log. Requires GUILD_DELETE permission.",
|
||||
"description": "Permanently deletes a guild. The guild becomes unavailable at once, and a background job deletes all channels, messages, and settings. Irreversible operation with no recovery window. Logged to audit log. Requires GUILD_DELETE permission.",
|
||||
"security": [{"adminApiKey": []}],
|
||||
"parameters": [
|
||||
{
|
||||
|
||||
@@ -62,7 +62,9 @@ describe('ICacheService.getOrSet', () => {
|
||||
expect(factory).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('resolves the ttl from the produced value', async () => {
|
||||
it('resolves the ttl from the produced value', async ({onTestFinished}) => {
|
||||
const now = vi.spyOn(Date, 'now').mockReturnValue(1_000);
|
||||
onTestFinished(() => now.mockRestore());
|
||||
const client = new MockKVProvider();
|
||||
const provider = new KVCacheProvider({client});
|
||||
const resolver = (value: number | null) => (value === null ? 5 : 30);
|
||||
|
||||
@@ -2,18 +2,11 @@
|
||||
|
||||
import {AGE_BLOCKED_GEOS, AGE_RESTRICTED_GEOS} from '@fluxer/instance_bootstrap/src/AgeGeos';
|
||||
import type {GeolocationResponse} from '@fluxer/instance_bootstrap/src/Types';
|
||||
import {extractClientIp} from '@fluxer/ip_utils/src/ClientIp';
|
||||
import {lookupGeoipByIp} from '@pkgs/geoip/src/GeoipLookup';
|
||||
|
||||
interface ResolveClientGeoipOptions {
|
||||
maxmindDbPath: string | undefined;
|
||||
trustClientIpHeader: boolean;
|
||||
clientIpHeaderName: string;
|
||||
}
|
||||
|
||||
export async function resolveClientGeoip(
|
||||
req: Request,
|
||||
options: ResolveClientGeoipOptions,
|
||||
ip: string | null,
|
||||
maxmindDbPath: string | undefined,
|
||||
): Promise<GeolocationResponse> {
|
||||
const response: GeolocationResponse = {
|
||||
countryCode: null,
|
||||
@@ -23,14 +16,10 @@ export async function resolveClientGeoip(
|
||||
ageRestrictedGeos: AGE_RESTRICTED_GEOS,
|
||||
ageBlockedGeos: AGE_BLOCKED_GEOS,
|
||||
};
|
||||
const ip = extractClientIp(req, {
|
||||
trustClientIpHeader: options.trustClientIpHeader,
|
||||
clientIpHeaderName: options.clientIpHeaderName,
|
||||
});
|
||||
if (!ip) {
|
||||
return response;
|
||||
}
|
||||
const geoip = await lookupGeoipByIp(ip, options.maxmindDbPath);
|
||||
const geoip = await lookupGeoipByIp(ip, maxmindDbPath);
|
||||
response.countryCode = geoip.countryCode;
|
||||
response.regionCode = geoip.regionCode ?? null;
|
||||
response.latitude = geoip.latitude != null ? geoip.latitude.toString() : null;
|
||||
|
||||
@@ -253,7 +253,7 @@ export function GuildAdminController(app: HonoApp) {
|
||||
operationId: 'delete_admin_guild',
|
||||
summary: 'Delete guild',
|
||||
description:
|
||||
'Permanently deletes a guild. Deletes all channels, messages, and settings. Irreversible operation with no recovery window. Logged to audit log. Requires GUILD_DELETE permission.',
|
||||
'Permanently deletes a guild. The guild becomes unavailable at once, and a background job deletes all channels, messages, and settings. Irreversible operation with no recovery window. Logged to audit log. Requires GUILD_DELETE permission.',
|
||||
responseSchema: SuccessResponse,
|
||||
statusCode: 200,
|
||||
security: 'adminApiKey',
|
||||
|
||||
@@ -57,6 +57,7 @@ export class AdminGuildService {
|
||||
entityAssetService: deps.entityAssetService,
|
||||
auditService: deps.auditService,
|
||||
updatePropagator: this.updatePropagator,
|
||||
guildService: deps.guildService,
|
||||
});
|
||||
this.vanityService = new AdminGuildVanityService({
|
||||
guildRepository: deps.guildRepository,
|
||||
|
||||
@@ -6,6 +6,9 @@ import type {AdminGuildUpdatePropagator} from '@app/api/admin/services/guild/Adm
|
||||
import {createGuildID, createUserID, type GuildID, type UserID} from '@app/api/BrandedTypes';
|
||||
import type {GuildRow} from '@app/api/database/types/GuildTypes';
|
||||
import type {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
|
||||
import {staffEditTouchesInvitePause} from '@app/api/guild/services/GuildInvitePauseApplier';
|
||||
import type {GuildService} from '@app/api/guild/services/GuildService';
|
||||
import {emitGuildFeaturesChanged} from '@app/api/infrastructure/activity/GuildEvents';
|
||||
import type {EntityAssetService, PreparedAssetUpload} from '@app/api/infrastructure/EntityAssetService';
|
||||
import {Logger} from '@app/api/Logger';
|
||||
import {UnknownGuildError} from '@fluxer/errors/src/domains/guild/UnknownGuildError';
|
||||
@@ -21,6 +24,7 @@ interface AdminGuildUpdateServiceDeps {
|
||||
entityAssetService: EntityAssetService;
|
||||
auditService: AdminAuditService;
|
||||
updatePropagator: AdminGuildUpdatePropagator;
|
||||
guildService: Pick<GuildService, 'withUpdateLock'>;
|
||||
}
|
||||
|
||||
export class AdminGuildUpdateService {
|
||||
@@ -39,23 +43,36 @@ export class AdminGuildUpdateService {
|
||||
adminUserId: UserID;
|
||||
auditLogReason: string | null;
|
||||
}) {
|
||||
const {guildRepository, auditService, updatePropagator} = this.deps;
|
||||
const guild = await guildRepository.findUnique(guildId);
|
||||
if (!guild) {
|
||||
throw new UnknownGuildError();
|
||||
}
|
||||
const newFeatures = new Set(guild.features);
|
||||
for (const feature of addFeatures) {
|
||||
newFeatures.add(feature);
|
||||
}
|
||||
for (const feature of removeFeatures) {
|
||||
newFeatures.delete(feature);
|
||||
}
|
||||
const updatedGuild = await guildRepository.upsertPartial(guildId, {features: newFeatures}, guild.toRow());
|
||||
const {guildRepository, auditService, updatePropagator, guildService} = this.deps;
|
||||
const {guild, newFeatures, updatedGuild} = await guildService.withUpdateLock(guildId, async () => {
|
||||
const guild = await guildRepository.findUnique(guildId);
|
||||
if (!guild) {
|
||||
throw new UnknownGuildError();
|
||||
}
|
||||
const newFeatures = new Set(guild.features);
|
||||
for (const feature of addFeatures) {
|
||||
newFeatures.add(feature);
|
||||
}
|
||||
for (const feature of removeFeatures) {
|
||||
newFeatures.delete(feature);
|
||||
}
|
||||
const patch: Partial<GuildRow> = {features: newFeatures};
|
||||
if (staffEditTouchesInvitePause(guild, [...addFeatures, ...removeFeatures])) {
|
||||
patch.invites_paused_until = null;
|
||||
}
|
||||
const updatedGuild = await guildRepository.upsertPartial(guildId, patch, guild.toRow());
|
||||
return {guild, newFeatures, updatedGuild};
|
||||
});
|
||||
await updatePropagator.dispatchGuildUpdate(guildId, updatedGuild, {
|
||||
adminUserId,
|
||||
reconcileDiscoveryFeature: true,
|
||||
});
|
||||
await emitGuildFeaturesChanged({
|
||||
previous: guild,
|
||||
updated: updatedGuild,
|
||||
actorId: adminUserId,
|
||||
source: 'staff',
|
||||
});
|
||||
await auditService.createAuditLog({
|
||||
adminUserId,
|
||||
targetType: 'guild',
|
||||
|
||||
@@ -15,6 +15,7 @@ import {ServiceMiddleware} from '@app/api/middleware/ServiceMiddleware';
|
||||
import {TrustedClientIpHeaderMiddleware} from '@app/api/middleware/TrustedClientIpHeaderMiddleware';
|
||||
import {UserMiddleware} from '@app/api/middleware/UserMiddleware';
|
||||
import type {HonoApp} from '@app/api/types/HonoEnv';
|
||||
import {PeerAddressMiddleware} from '@app/api/utils/RequestClientIp';
|
||||
import {Headers as HttpHeaders} from '@fluxer/constants/src/Headers';
|
||||
import {InvalidApiOriginError} from '@fluxer/errors/src/domains/core/InvalidApiOriginError';
|
||||
import {cors} from '@fluxer/hono/src/middleware/Cors';
|
||||
@@ -35,6 +36,7 @@ interface MiddlewarePipelineOptions {
|
||||
export function configureMiddleware(routes: HonoApp, options: MiddlewarePipelineOptions): void {
|
||||
const {logger, nodeEnv, corsOrigins, trustClientIpHeader, clientIpHeaderName, maxInflightRequests} = options;
|
||||
const resolvedHeader = resolveClientIpHeaderName(clientIpHeaderName);
|
||||
routes.use(PeerAddressMiddleware);
|
||||
routes.use('/webhooks/:webhook_id/:token', cors({origins: '*'}));
|
||||
routes.use('/webhooks/:webhook_id/:token/messages/:message_id', cors({origins: '*'}));
|
||||
routes.use(
|
||||
|
||||
@@ -2,31 +2,34 @@
|
||||
|
||||
import {configureMiddleware} from '@app/api/app/MiddlewarePipeline';
|
||||
import {Config} from '@app/api/Config';
|
||||
import {InstanceController} from '@app/api/instance/InstanceController';
|
||||
import {setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';
|
||||
import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
|
||||
import {NoopWorkerService} from '@app/api/test/NoopWorkerService';
|
||||
import type {HonoEnv} from '@app/api/types/HonoEnv';
|
||||
import {getRequestClientIp} from '@app/api/utils/RequestClientIp';
|
||||
import {AppErrorHandler, AppNotFoundHandler} from '@fluxer/errors/src/domains/core/ErrorHandlers';
|
||||
import {Hono} from 'hono';
|
||||
import {afterEach, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
const CLIENT_IP_HEADER_NAME = 'x-real-ip';
|
||||
|
||||
function createProductionApp(): Hono<HonoEnv> {
|
||||
function createProductionApp(trustClientIpHeader = true): Hono<HonoEnv> {
|
||||
const routes = new Hono<HonoEnv>({strict: true});
|
||||
configureMiddleware(routes, {
|
||||
logger: new NoopLogger(),
|
||||
nodeEnv: 'production',
|
||||
corsOrigins: ['https://web.fluxer.app'],
|
||||
trustClientIpHeader: true,
|
||||
trustClientIpHeader,
|
||||
clientIpHeaderName: CLIENT_IP_HEADER_NAME,
|
||||
maxInflightRequests: 100,
|
||||
});
|
||||
routes.onError(AppErrorHandler);
|
||||
routes.notFound(AppNotFoundHandler);
|
||||
InstanceController(routes);
|
||||
routes.post('/internal/rpc', (ctx) => ctx.json({ok: true}));
|
||||
routes.get('/connections/bluesky/jwks.json', (ctx) => ctx.json({keys: []}));
|
||||
routes.get('/users/@me', (ctx) => ctx.json({ok: true}));
|
||||
routes.get('/users/@me', (ctx) => ctx.json({ok: true, ip: getRequestClientIp(ctx)}));
|
||||
const app = new Hono<HonoEnv>({strict: true});
|
||||
app.route('/v1', routes);
|
||||
app.route('/', routes);
|
||||
@@ -91,4 +94,52 @@ describe('client ip requirements across the production middleware pipeline', ()
|
||||
expect(response.status).toBe(403);
|
||||
expect(await response.json()).toMatchObject({code: 'FORBIDDEN'});
|
||||
});
|
||||
|
||||
it('resolves the custom client ip header while it is trusted', async () => {
|
||||
const app = createProductionApp();
|
||||
const response = await app.request(
|
||||
'http://api:8080/v1/users/@me',
|
||||
{headers: {[CLIENT_IP_HEADER_NAME]: '203.0.113.10', 'x-forwarded-for': '198.51.100.1'}},
|
||||
{incoming: {socket: {remoteAddress: '10.0.0.5'}}},
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toMatchObject({ip: '203.0.113.10'});
|
||||
});
|
||||
|
||||
describe('with the client ip header untrusted', () => {
|
||||
beforeEach(() => {
|
||||
Config.proxy.trust_client_ip_header = false;
|
||||
});
|
||||
|
||||
it('keys the request on the peer address and ignores a spoofed header', async () => {
|
||||
const app = createProductionApp(false);
|
||||
const response = await app.request(
|
||||
'http://api:8080/v1/users/@me',
|
||||
{headers: {[CLIENT_IP_HEADER_NAME]: '203.0.113.10', 'x-forwarded-for': '203.0.113.11'}},
|
||||
{incoming: {socket: {remoteAddress: '::ffff:198.51.100.20'}}},
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toMatchObject({ip: '198.51.100.20'});
|
||||
});
|
||||
|
||||
it('serves the discovery document', async () => {
|
||||
const app = createProductionApp(false);
|
||||
const response = await app.request(
|
||||
'http://api:8080/.well-known/fluxer',
|
||||
{headers: {'x-forwarded-for': '203.0.113.10'}},
|
||||
{incoming: {socket: {remoteAddress: '198.51.100.20'}}},
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
});
|
||||
|
||||
it('serves the internal rpc route', async () => {
|
||||
const app = createProductionApp(false);
|
||||
const response = await app.request('http://api:8080/internal/rpc', {
|
||||
method: 'POST',
|
||||
headers: {'content-type': 'application/json'},
|
||||
body: '{}',
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -325,6 +325,7 @@ async function registerAccount(
|
||||
timezone: null,
|
||||
invite_code: data.invite_code?.trim() || null,
|
||||
flags: user.flags.toString(),
|
||||
date_of_birth: user.dateOfBirth,
|
||||
},
|
||||
null,
|
||||
user.id.toString(),
|
||||
|
||||
@@ -564,6 +564,7 @@ export class SsoService {
|
||||
timezone: null,
|
||||
invite_code: null,
|
||||
flags: user.flags.toString(),
|
||||
date_of_birth: user.dateOfBirth,
|
||||
},
|
||||
null,
|
||||
user.id.toString(),
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
sendMessage,
|
||||
setupFanoutWorld,
|
||||
} from '@app/api/channel/tests/CrosspostWorkerTestUtils';
|
||||
import {processRecordedGuildDeletions} from '@app/api/guild/tests/GuildDeletionTestUtils';
|
||||
import {setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {NoopGatewayService} from '@app/api/test/NoopGatewayService';
|
||||
@@ -315,6 +316,7 @@ describe('Announcement channel conversion', () => {
|
||||
.body({password: owner.password})
|
||||
.expect(HTTP_STATUS.NO_CONTENT)
|
||||
.executeWithResponse();
|
||||
await processRecordedGuildDeletions(worker);
|
||||
const removals = worker.byTask('removeChannelFollowers');
|
||||
expect(removals.map((job) => job.payload.sourceChannelId).sort()).toEqual([first.id, second.id].sort());
|
||||
for (const job of removals) {
|
||||
@@ -333,6 +335,7 @@ describe('Announcement channel conversion', () => {
|
||||
.body(null)
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
await processRecordedGuildDeletions(worker);
|
||||
const removals = worker.byTask('removeChannelFollowers');
|
||||
expect(removals).toHaveLength(1);
|
||||
expect(removals[0]?.payload).toEqual({
|
||||
|
||||
@@ -11,7 +11,7 @@ import {
|
||||
createPermissionOverwrite,
|
||||
createRole,
|
||||
} from '@app/api/channel/tests/ChannelTestUtils';
|
||||
import {createTestChannelService} from '@app/api/channel/tests/CrosspostTestUtils';
|
||||
import {createTestGuildStack} from '@app/api/channel/tests/CrosspostTestUtils';
|
||||
import {getGatewayService, getSnowflakeService, setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';
|
||||
import {
|
||||
getAvatarService,
|
||||
@@ -31,6 +31,7 @@ import {createBuilder} from '@app/api/test/TestRequestBuilder';
|
||||
import {WebhookRepository} from '@app/api/webhook/WebhookRepository';
|
||||
import crosspostMessage from '@app/api/worker/tasks/CrosspostMessage';
|
||||
import crosspostMessageChunk from '@app/api/worker/tasks/CrosspostMessageChunk';
|
||||
import guildProcessDeletion from '@app/api/worker/tasks/GuildProcessDeletion';
|
||||
import removeChannelFollowers from '@app/api/worker/tasks/RemoveChannelFollowers';
|
||||
import syncCrosspostCopies from '@app/api/worker/tasks/SyncCrosspostCopies';
|
||||
import syncCrosspostedMessage from '@app/api/worker/tasks/SyncCrosspostedMessage';
|
||||
@@ -201,6 +202,7 @@ export const crosspostTaskHandlers: Record<string, WorkerTaskHandler> = {
|
||||
syncCrosspostedMessage,
|
||||
syncCrosspostCopies,
|
||||
removeChannelFollowers,
|
||||
guildProcessDeletion,
|
||||
};
|
||||
|
||||
export function enableCrosspostWorker(
|
||||
@@ -210,6 +212,7 @@ export function enableCrosspostWorker(
|
||||
const worker = new SyncTaskWorkerService({...crosspostTaskHandlers, ...extraHandlers}, {deferred: true});
|
||||
setInjectedWorkerService(worker);
|
||||
resetApiServicesForTesting();
|
||||
const guildStack = createTestGuildStack();
|
||||
setWorkerDependenciesForTest({
|
||||
channelRepository: getChannelRepository(),
|
||||
webhookRepository: getWebhookRepository(),
|
||||
@@ -222,7 +225,8 @@ export function enableCrosspostWorker(
|
||||
snowflakeService: getSnowflakeService(),
|
||||
cacheService: getCacheService(),
|
||||
limitConfigService: getLimitConfigService(),
|
||||
channelService: createTestChannelService(),
|
||||
channelService: guildStack.channelService,
|
||||
guildService: guildStack.guildService,
|
||||
workerService: worker,
|
||||
...extraDependencies,
|
||||
});
|
||||
|
||||
@@ -22,7 +22,7 @@ import {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
|
||||
import {DisabledLiveKitService} from '@app/api/infrastructure/DisabledLiveKitService';
|
||||
import {InMemoryVoiceRoomStore} from '@app/api/infrastructure/InMemoryVoiceRoomStore';
|
||||
import {ensureSessionStarted} from '@app/api/message/tests/MessageTestUtils';
|
||||
import {createGuildStackServices} from '@app/api/middleware/GuildStackServiceFactory';
|
||||
import {createGuildStackServices, type GuildStackServices} from '@app/api/middleware/GuildStackServiceFactory';
|
||||
import {getVoiceAvailabilityService} from '@app/api/middleware/ServiceRegistry';
|
||||
import {
|
||||
getAssetDeletionQueue,
|
||||
@@ -190,7 +190,7 @@ export function createStallGate(): StallGate & {hit: () => Promise<void>} {
|
||||
};
|
||||
}
|
||||
|
||||
export function createTestChannelService(): ChannelService {
|
||||
export function createTestGuildStack(): GuildStackServices {
|
||||
return createGuildStackServices({
|
||||
apiContext: createApiContext(),
|
||||
channelRepository: getChannelRepository(),
|
||||
@@ -214,7 +214,11 @@ export function createTestChannelService(): ChannelService {
|
||||
voiceRoomStore: new InMemoryVoiceRoomStore(),
|
||||
liveKitService: new DisabledLiveKitService(),
|
||||
voiceAvailabilityService: getVoiceAvailabilityService(),
|
||||
}).channelService;
|
||||
});
|
||||
}
|
||||
|
||||
export function createTestChannelService(): ChannelService {
|
||||
return createTestGuildStack().channelService;
|
||||
}
|
||||
|
||||
export {sendChannelMessage as postMessage} from '@app/api/channel/tests/ChannelTestUtils';
|
||||
@@ -9,6 +9,7 @@ import {acceptInvite, createChannel, createChannelInvite, createGuild} from '@ap
|
||||
import {threadsRequest} from '@app/api/channel/tests/ThreadTestUtils';
|
||||
import {upsertOne} from '@app/api/database/CassandraQueryExecution';
|
||||
import {clearChannelThreadsTaintCacheForTesting} from '@app/api/experiment/ChannelThreadsGate';
|
||||
import {processRecordedGuildDeletions, recordGuildDeletions} from '@app/api/guild/tests/GuildDeletionTestUtils';
|
||||
import {ensureSessionStarted, sendMessage} from '@app/api/message/tests/MessageTestUtils';
|
||||
import {getKVThreadAutoArchiveQueue} from '@app/api/middleware/ServiceSingletons';
|
||||
import {ThreadState as ThreadStateTable} from '@app/api/Tables';
|
||||
@@ -481,11 +482,13 @@ describe('thread message path', () => {
|
||||
const queue = getKVThreadAutoArchiveQueue();
|
||||
const guildId = createGuildID(BigInt(s.guildId));
|
||||
expect(await queue.listGuilds()).toContain(guildId);
|
||||
const worker = recordGuildDeletions();
|
||||
await createBuilder(harness, s.owner.token)
|
||||
.post(`/guilds/${s.guildId}/delete`)
|
||||
.body({password: s.owner.password})
|
||||
.expect(204)
|
||||
.execute();
|
||||
await processRecordedGuildDeletions(worker);
|
||||
expect(await queue.listGuilds()).not.toContain(guildId);
|
||||
expect(await queue.getDue(guildId, Number.MAX_SAFE_INTEGER, 10)).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -49,6 +49,7 @@ export interface GuildRow {
|
||||
audit_logs_indexed_at: Date | null;
|
||||
members_indexed_at: Date | null;
|
||||
message_history_cutoff: Date | null;
|
||||
invites_paused_until: Date | null;
|
||||
version: number;
|
||||
}
|
||||
|
||||
@@ -87,6 +88,7 @@ export const GUILD_COLUMNS = [
|
||||
'audit_logs_indexed_at',
|
||||
'members_indexed_at',
|
||||
'message_history_cutoff',
|
||||
'invites_paused_until',
|
||||
'version',
|
||||
] as const satisfies ReadonlyArray<keyof GuildRow>;
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware';
|
||||
import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware';
|
||||
import {RateLimitConfigs} from '@app/api/RateLimitConfig';
|
||||
import type {HonoEnv} from '@app/api/types/HonoEnv';
|
||||
import {getRequestClientIp} from '@app/api/utils/RequestClientIp';
|
||||
import {GeolocationResponse} from '@fluxer/schema/src/domains/geolocation/GeolocationSchemas';
|
||||
import {resolveClientGeoip} from '@pkgs/geoip/src/ResolveClientGeoip';
|
||||
import type {Hono} from 'hono';
|
||||
@@ -27,11 +28,7 @@ export function GeolocationController(app: Hono<HonoEnv>): void {
|
||||
ctx.header('Access-Control-Allow-Origin', '*');
|
||||
ctx.header('Access-Control-Allow-Headers', 'Content-Type');
|
||||
ctx.header('Access-Control-Allow-Methods', 'GET, OPTIONS');
|
||||
const response = await resolveClientGeoip(ctx.req.raw, {
|
||||
maxmindDbPath: Config.geoip.maxmindDbPath,
|
||||
trustClientIpHeader: Config.proxy.trust_client_ip_header,
|
||||
clientIpHeaderName: Config.proxy.client_ip_header,
|
||||
});
|
||||
const response = await resolveClientGeoip(getRequestClientIp(ctx), Config.geoip.maxmindDbPath);
|
||||
return ctx.json(response);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {GuildID} from '@app/api/BrandedTypes';
|
||||
import {ResourceLockedError} from '@fluxer/errors/src/domains/core/ResourceLockedError';
|
||||
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
|
||||
|
||||
const GUILD_UPDATE_LOCK_TTL_SECONDS = 10;
|
||||
const GUILD_UPDATE_LOCK_RETRY_DELAY_MS = 50;
|
||||
const GUILD_UPDATE_LOCK_MAX_WAIT_MS = 5000;
|
||||
|
||||
export type GuildUpdateLocks = Pick<ICacheService, 'acquireLock' | 'releaseLock'>;
|
||||
|
||||
async function acquireGuildUpdateLock(locks: GuildUpdateLocks, lockKey: string): Promise<string | null> {
|
||||
const startTime = Date.now();
|
||||
while (Date.now() - startTime < GUILD_UPDATE_LOCK_MAX_WAIT_MS) {
|
||||
const token = await locks.acquireLock(lockKey, GUILD_UPDATE_LOCK_TTL_SECONDS);
|
||||
if (token) {
|
||||
return token;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, GUILD_UPDATE_LOCK_RETRY_DELAY_MS));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function withGuildUpdateLock<T>(
|
||||
locks: GuildUpdateLocks,
|
||||
guildId: GuildID,
|
||||
fn: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const lockKey = `guild:${guildId}:update`;
|
||||
const lockToken = await acquireGuildUpdateLock(locks, lockKey);
|
||||
if (!lockToken) {
|
||||
throw new ResourceLockedError();
|
||||
}
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
await locks.releaseLock(lockKey, lockToken);
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import {createChannelID, createGuildID} from '@app/api/BrandedTypes';
|
||||
import {Config} from '@app/api/Config';
|
||||
import type {GuildDiscoveryRow} from '@app/api/database/types/GuildDiscoveryTypes';
|
||||
import {viewerFromCtx} from '@app/api/experiment/ChannelThreadsGate';
|
||||
import {emitGuildJoinRefused} from '@app/api/infrastructure/activity/GuildEvents';
|
||||
import {DefaultUserOnly, LoginRequired} from '@app/api/middleware/AuthMiddleware';
|
||||
import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware';
|
||||
import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware';
|
||||
@@ -153,6 +154,9 @@ export function GuildDiscoveryController(app: HonoApp) {
|
||||
}
|
||||
const guild = await ctx.get('guildService').data.operations.getGuildSystem(guildId);
|
||||
if (guild.features.has(GuildFeatures.INVITES_DISABLED)) {
|
||||
if (!(await ctx.get('gatewayService').hasGuildMember({guildId, userId: user.id}))) {
|
||||
await emitGuildJoinRefused({guildId, userId: user.id, joinSource: 'discovery'});
|
||||
}
|
||||
throw new InvitesDisabledError();
|
||||
}
|
||||
await ctx.get('guildService').members.addUserToGuild({
|
||||
|
||||
@@ -16,6 +16,8 @@ import type {InviteRepository} from '@app/api/invite/InviteRepository';
|
||||
import type {LimitConfigService} from '@app/api/limits/LimitConfigService';
|
||||
import type {UserRepository} from '@app/api/user/repositories/UserRepository';
|
||||
import type {WebhookRepository} from '@app/api/webhook/WebhookRepository';
|
||||
import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
|
||||
import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
|
||||
|
||||
export class GuildDataService {
|
||||
private readonly helpers: GuildDataHelpers;
|
||||
@@ -35,6 +37,7 @@ export class GuildDataService {
|
||||
private readonly webhookRepository: WebhookRepository,
|
||||
private readonly guildAuditLogService: GuildAuditLogService,
|
||||
private readonly limitConfigService: LimitConfigService,
|
||||
private readonly workerService: IWorkerService<WorkerTaskName>,
|
||||
) {
|
||||
this.helpers = new GuildDataHelpers(this.gatewayService, this.guildAuditLogService, this.userRepository);
|
||||
this.operations = new GuildOperationsService(
|
||||
@@ -50,6 +53,7 @@ export class GuildDataService {
|
||||
this.helpers,
|
||||
this.limitConfigService,
|
||||
new GuildDiscoveryRepository(),
|
||||
this.workerService,
|
||||
);
|
||||
this.vanity = new GuildVanityService(this.guildRepository, this.inviteRepository, this.helpers);
|
||||
this.ownership = new GuildOwnershipService(this.guildRepository, this.userRepository, this.helpers);
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createGuildID} from '@app/api/BrandedTypes';
|
||||
import {SYSTEM_USER_ID} from '@app/api/constants/Core';
|
||||
import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService';
|
||||
import {mapGuildToGuildResponse} from '@app/api/guild/GuildModel';
|
||||
import {type GuildUpdateLocks, withGuildUpdateLock} from '@app/api/guild/GuildUpdateLock';
|
||||
import type {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
|
||||
import {workerMeta} from '@app/api/infrastructure/activity/ActivityMeta';
|
||||
import {
|
||||
type ActionOutcome,
|
||||
type FeatureSource,
|
||||
INVITES_ALREADY_PAUSED,
|
||||
} from '@app/api/infrastructure/activity/Contract.generated';
|
||||
import {emitGuildFeaturesChanged} from '@app/api/infrastructure/activity/GuildEvents';
|
||||
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
|
||||
import {Logger} from '@app/api/Logger';
|
||||
import type {Guild} from '@app/api/models/Guild';
|
||||
import {type ActionOf, outcomeOf} from '@app/api/user/services/AccountStateApplier';
|
||||
import {serializeGuildForAudit} from '@app/api/utils/AuditSerializationUtils';
|
||||
import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType';
|
||||
import {GuildFeatures} from '@fluxer/constants/src/GuildConstants';
|
||||
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
|
||||
|
||||
export const GUILD_INVITE_PAUSE_SCHEDULE_KEY = 'guild:invite_pause:lifts';
|
||||
const INVITE_PAUSE_FEATURES = [GuildFeatures.INVITES_DISABLED, GuildFeatures.RAID_DETECTED] as const;
|
||||
|
||||
export interface GuildInvitePauseDeps {
|
||||
guilds: Pick<GuildRepository, 'findUnique' | 'upsertPartial'>;
|
||||
locks: GuildUpdateLocks;
|
||||
schedule: Pick<IKVProvider, 'zadd' | 'zrem' | 'zrangebyscore'>;
|
||||
gateway: Pick<IGatewayService, 'dispatchGuild'>;
|
||||
auditLog: Pick<GuildAuditLogService, 'record' | 'computeChanges'>;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
export function memberEndsInvitePause(guild: Guild, nextFeatures: ReadonlySet<string>): boolean {
|
||||
return (
|
||||
guild.invitesPausedUntil !== null &&
|
||||
guild.features.has(GuildFeatures.INVITES_DISABLED) &&
|
||||
!nextFeatures.has(GuildFeatures.INVITES_DISABLED)
|
||||
);
|
||||
}
|
||||
|
||||
export function staffEditTouchesInvitePause(guild: Guild, changed: Iterable<string>): boolean {
|
||||
if (guild.invitesPausedUntil === null) return false;
|
||||
for (const feature of changed) {
|
||||
if ((INVITE_PAUSE_FEATURES as ReadonlyArray<string>).includes(feature)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function writePause(
|
||||
deps: GuildInvitePauseDeps,
|
||||
guild: Guild,
|
||||
features: Set<string>,
|
||||
until: Date | null,
|
||||
source: FeatureSource,
|
||||
): Promise<void> {
|
||||
const updated = await deps.guilds.upsertPartial(guild.id, {features, invites_paused_until: until}, guild.toRow());
|
||||
try {
|
||||
await deps.gateway.dispatchGuild({
|
||||
guildId: guild.id,
|
||||
event: 'GUILD_UPDATE',
|
||||
data: mapGuildToGuildResponse(updated),
|
||||
});
|
||||
} catch (error) {
|
||||
Logger.error({error, guildId: guild.id}, 'Failed to dispatch guild update after invite pause change');
|
||||
}
|
||||
const changes = deps.auditLog
|
||||
.computeChanges(serializeGuildForAudit(guild), serializeGuildForAudit(updated))
|
||||
.filter((change) => change.key === 'features');
|
||||
if (changes.length > 0) {
|
||||
try {
|
||||
await deps.auditLog.record({
|
||||
guildId: guild.id,
|
||||
userId: SYSTEM_USER_ID,
|
||||
action: AuditLogActionType.GUILD_UPDATE,
|
||||
targetId: guild.id,
|
||||
changes,
|
||||
});
|
||||
} catch (error) {
|
||||
Logger.error({error, guildId: guild.id}, 'Failed to record invite pause change in the guild audit log');
|
||||
}
|
||||
}
|
||||
await emitGuildFeaturesChanged({
|
||||
previous: guild,
|
||||
updated,
|
||||
actorId: null,
|
||||
source,
|
||||
meta: workerMeta(),
|
||||
});
|
||||
}
|
||||
|
||||
async function liftPause(deps: GuildInvitePauseDeps, guild: Guild, source: FeatureSource): Promise<void> {
|
||||
const features = new Set(guild.features);
|
||||
for (const feature of INVITE_PAUSE_FEATURES) features.delete(feature);
|
||||
await writePause(deps, guild, features, null, source);
|
||||
await deps.schedule.zrem(GUILD_INVITE_PAUSE_SCHEDULE_KEY, guild.id.toString());
|
||||
}
|
||||
|
||||
export async function applyPauseGuildInvites(
|
||||
deps: GuildInvitePauseDeps,
|
||||
env: ActionOf<'pause_guild_invites'>,
|
||||
): Promise<ActionOutcome> {
|
||||
const guildId = createGuildID(BigInt(env.guild_id));
|
||||
return withGuildUpdateLock(deps.locks, guildId, async () => {
|
||||
const guild = await deps.guilds.findUnique(guildId);
|
||||
if (!guild) return outcomeOf(env, 'ineligible');
|
||||
const current = guild.invitesPausedUntil?.getTime() ?? null;
|
||||
if (!env.on) {
|
||||
if (current === null) return outcomeOf(env, 'noop');
|
||||
if (env.until_ms > 0 && current > env.until_ms) return outcomeOf(env, 'noop', null, `until_ms=${current}`);
|
||||
await liftPause(deps, guild, 'action');
|
||||
return outcomeOf(env, 'applied');
|
||||
}
|
||||
if (current === null && guild.features.has(GuildFeatures.INVITES_DISABLED)) {
|
||||
return outcomeOf(env, 'noop', null, INVITES_ALREADY_PAUSED);
|
||||
}
|
||||
const now = deps.now?.() ?? Date.now();
|
||||
const until = Math.max(current ?? 0, env.until_ms);
|
||||
if (until <= now) return outcomeOf(env, 'noop');
|
||||
const features = new Set(guild.features);
|
||||
for (const feature of INVITE_PAUSE_FEATURES) features.add(feature);
|
||||
const detail = `until_ms=${until}`;
|
||||
await deps.schedule.zadd(GUILD_INVITE_PAUSE_SCHEDULE_KEY, until, guildId.toString());
|
||||
if (until === current && features.size === guild.features.size) return outcomeOf(env, 'noop', null, detail);
|
||||
await writePause(deps, guild, features, new Date(until), 'action');
|
||||
return outcomeOf(env, 'applied', null, detail);
|
||||
});
|
||||
}
|
||||
|
||||
export async function liftExpiredGuildInvitePauses(deps: GuildInvitePauseDeps, limit: number): Promise<number> {
|
||||
const now = deps.now?.() ?? Date.now();
|
||||
const due = await deps.schedule.zrangebyscore(GUILD_INVITE_PAUSE_SCHEDULE_KEY, '-inf', now, 'LIMIT', 0, limit);
|
||||
let lifted = 0;
|
||||
for (const member of due) {
|
||||
const guildId = createGuildID(BigInt(member));
|
||||
try {
|
||||
const done = await withGuildUpdateLock(deps.locks, guildId, async () => {
|
||||
const guild = await deps.guilds.findUnique(guildId);
|
||||
const until = guild?.invitesPausedUntil?.getTime() ?? null;
|
||||
if (!guild || until === null) {
|
||||
await deps.schedule.zrem(GUILD_INVITE_PAUSE_SCHEDULE_KEY, member);
|
||||
return false;
|
||||
}
|
||||
if (until > now) {
|
||||
await deps.schedule.zadd(GUILD_INVITE_PAUSE_SCHEDULE_KEY, until, member);
|
||||
return false;
|
||||
}
|
||||
await liftPause(deps, guild, 'system');
|
||||
return true;
|
||||
});
|
||||
if (done) lifted++;
|
||||
} catch (error) {
|
||||
Logger.error({error, guildId: member}, 'Failed to lift expired guild invite pause');
|
||||
}
|
||||
}
|
||||
return lifted;
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
THREAD_AUDIT_LOG_ACTION_TYPES,
|
||||
} from '@app/api/guild/GuildAuditLogEntryMapper';
|
||||
import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService';
|
||||
import {withGuildUpdateLock} from '@app/api/guild/GuildUpdateLock';
|
||||
import type {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
|
||||
import {GuildChannelService} from '@app/api/guild/services/GuildChannelService';
|
||||
import {GuildContentService} from '@app/api/guild/services/GuildContentService';
|
||||
@@ -53,7 +54,6 @@ import {THREAD_FEATURE_CHANNEL_TYPES} from '@fluxer/constants/src/ThreadConstant
|
||||
import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
|
||||
import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError';
|
||||
import {MissingPermissionsError} from '@fluxer/errors/src/domains/core/MissingPermissionsError';
|
||||
import {ResourceLockedError} from '@fluxer/errors/src/domains/core/ResourceLockedError';
|
||||
import {UnknownGuildEmojiError} from '@fluxer/errors/src/domains/guild/UnknownGuildEmojiError';
|
||||
import {UnknownGuildError} from '@fluxer/errors/src/domains/guild/UnknownGuildError';
|
||||
import {UnknownGuildStickerError} from '@fluxer/errors/src/domains/guild/UnknownGuildStickerError';
|
||||
@@ -80,10 +80,6 @@ interface StoredGuildAuditLogListResponse extends Omit<GuildAuditLogListResponse
|
||||
webhooks: Array<StoredAuditLogWebhookResponse>;
|
||||
}
|
||||
|
||||
const GUILD_UPDATE_LOCK_TTL_SECONDS = 10;
|
||||
const GUILD_UPDATE_LOCK_RETRY_DELAY_MS = 50;
|
||||
const GUILD_UPDATE_LOCK_MAX_WAIT_MS = 5000;
|
||||
|
||||
export class GuildService {
|
||||
public readonly data: GuildDataService;
|
||||
public readonly members: GuildMemberService;
|
||||
@@ -141,6 +137,7 @@ export class GuildService {
|
||||
webhookRepository,
|
||||
guildAuditLogService,
|
||||
limitConfigService,
|
||||
workerService,
|
||||
);
|
||||
this.members = new GuildMemberService(
|
||||
guildRepository,
|
||||
@@ -210,9 +207,10 @@ export class GuildService {
|
||||
auditLogReason?: string | null,
|
||||
): Promise<GuildResponse> {
|
||||
const {guildId, requestCache} = params;
|
||||
const {guild, previousFeatures, updatedFeatures} = await this.withGuildUpdateLock(guildId, requestCache, () =>
|
||||
this.data.operations.updateGuild(params, auditLogReason),
|
||||
);
|
||||
const {guild, previousFeatures, updatedFeatures} = await this.withUpdateLock(guildId, () => {
|
||||
requestCache.guilds.delete(guildId);
|
||||
return this.data.operations.updateGuild(params, auditLogReason);
|
||||
});
|
||||
if (
|
||||
previousFeatures.has(GuildFeatures.TEXT_CHANNEL_FLEXIBLE_NAMES) &&
|
||||
!updatedFeatures.has(GuildFeatures.TEXT_CHANNEL_FLEXIBLE_NAMES)
|
||||
@@ -222,30 +220,8 @@ export class GuildService {
|
||||
return guild;
|
||||
}
|
||||
|
||||
private async withGuildUpdateLock<T>(guildId: GuildID, requestCache: RequestCache, fn: () => Promise<T>): Promise<T> {
|
||||
const lockKey = `guild:${guildId}:update`;
|
||||
const lockToken = await this.acquireGuildUpdateLock(lockKey);
|
||||
if (!lockToken) {
|
||||
throw new ResourceLockedError();
|
||||
}
|
||||
try {
|
||||
requestCache.guilds.delete(guildId);
|
||||
return await fn();
|
||||
} finally {
|
||||
await this.cacheService.releaseLock(lockKey, lockToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async acquireGuildUpdateLock(lockKey: string): Promise<string | null> {
|
||||
const startTime = Date.now();
|
||||
while (Date.now() - startTime < GUILD_UPDATE_LOCK_MAX_WAIT_MS) {
|
||||
const token = await this.cacheService.acquireLock(lockKey, GUILD_UPDATE_LOCK_TTL_SECONDS);
|
||||
if (token) {
|
||||
return token;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, GUILD_UPDATE_LOCK_RETRY_DELAY_MS));
|
||||
}
|
||||
return null;
|
||||
withUpdateLock<T>(guildId: GuildID, fn: () => Promise<T>): Promise<T> {
|
||||
return withGuildUpdateLock(this.cacheService, guildId, fn);
|
||||
}
|
||||
|
||||
async getEmojiMetadata(emojiId: EmojiID): Promise<GuildEmojiMetadataResponse> {
|
||||
|
||||
@@ -25,6 +25,8 @@ import {mapGuildToGuildResponse, mapGuildToPartialResponse} from '@app/api/guild
|
||||
import type {GuildDiscoveryRepository} from '@app/api/guild/repositories/GuildDiscoveryRepository';
|
||||
import type {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
|
||||
import type {GuildDataHelpers} from '@app/api/guild/services/data/GuildDataHelpers';
|
||||
import {memberEndsInvitePause} from '@app/api/guild/services/GuildInvitePauseApplier';
|
||||
import {emitGuildCreated, emitGuildFeaturesChanged} from '@app/api/infrastructure/activity/GuildEvents';
|
||||
import {contentModerationService} from '@app/api/infrastructure/ContentModerationService';
|
||||
import type {EntityAssetService, PreparedAssetUpload} from '@app/api/infrastructure/EntityAssetService';
|
||||
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
|
||||
@@ -48,6 +50,7 @@ import {addGuildToUncategorizedFolder, removeGuildFromUserFolders} from '@app/ap
|
||||
import {serializeGuildForAudit} from '@app/api/utils/AuditSerializationUtils';
|
||||
import {mapWithConcurrency} from '@app/api/utils/ConcurrencyUtils';
|
||||
import type {WebhookRepository} from '@app/api/webhook/WebhookRepository';
|
||||
import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
|
||||
import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType';
|
||||
import {ALL_PERMISSIONS, ChannelTypes, DEFAULT_PERMISSIONS, Permissions} from '@fluxer/constants/src/ChannelConstants';
|
||||
import {DiscoveryApplicationStatus} from '@fluxer/constants/src/DiscoveryConstants';
|
||||
@@ -91,6 +94,7 @@ import type {
|
||||
TemplateSerializedGuild,
|
||||
} from '@fluxer/schema/src/domains/guild/GuildTemplateSchemas';
|
||||
import {extractTimestamp} from '@fluxer/snowflake/src/SnowflakeUtils';
|
||||
import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
|
||||
|
||||
const GUILD_IMAGE_FIELDS = [
|
||||
{field: 'icon', hash: 'icon_hash', dimensions: null, gate: null},
|
||||
@@ -183,6 +187,8 @@ function setsEqual(a: ReadonlySet<string>, b: ReadonlySet<string>): boolean {
|
||||
|
||||
const TEMPLATE_AFK_TIMEOUT_MIN_SECONDS = 60;
|
||||
const THREAD_PURGE_CONCURRENCY = 8;
|
||||
const GUILD_DELETION_MEMBER_CONCURRENCY = 25;
|
||||
const GUILD_DELETION_PENDING_FEATURES = [GuildFeatures.UNAVAILABLE_FOR_EVERYONE, GuildFeatures.UNAVAILABLE_HIDDEN];
|
||||
const TEMPLATE_AFK_TIMEOUT_MAX_SECONDS = 3600;
|
||||
const THE_OTHER_PLATFORM_GUILD_STAGE_VOICE_CHANNEL_TYPE = 13;
|
||||
|
||||
@@ -204,6 +210,7 @@ export class GuildOperationsService {
|
||||
private readonly helpers: GuildDataHelpers,
|
||||
private readonly limitConfigService: LimitConfigService,
|
||||
private readonly discoveryRepository: GuildDiscoveryRepository,
|
||||
private readonly workerService: IWorkerService<WorkerTaskName>,
|
||||
) {}
|
||||
|
||||
async getGuild(params: {userId: UserID; guildId: GuildID}): Promise<GuildResponse> {
|
||||
@@ -388,6 +395,7 @@ export class GuildOperationsService {
|
||||
audit_logs_indexed_at: null,
|
||||
members_indexed_at: null,
|
||||
message_history_cutoff: null,
|
||||
invites_paused_until: null,
|
||||
version: 1,
|
||||
};
|
||||
batch.addPrepared(Guilds.insert(guildData));
|
||||
@@ -420,6 +428,7 @@ export class GuildOperationsService {
|
||||
await batch.execute();
|
||||
await this.channelRepository.threads.ensureGuildMarker(guildId, {permsSeededAt: new Date()});
|
||||
const guild = new Guild(guildData);
|
||||
await emitGuildCreated(guild);
|
||||
await this.gatewayService.startGuild(guildId);
|
||||
await this.gatewayService.joinGuild({userId: user.id, guildId});
|
||||
if (!user.isBot) {
|
||||
@@ -502,6 +511,10 @@ export class GuildOperationsService {
|
||||
}
|
||||
}
|
||||
const updatedFeatures = this.computeUpdatedFeatures(previousFeatures, data.features);
|
||||
const endsInvitePause = memberEndsInvitePause(currentGuild, updatedFeatures);
|
||||
if (endsInvitePause) {
|
||||
updatedFeatures.delete(GuildFeatures.RAID_DETECTED);
|
||||
}
|
||||
const featuresChanged = !setsEqual(previousFeatures, updatedFeatures);
|
||||
const patch: Partial<GuildRow> = {};
|
||||
if (data.name !== undefined) {
|
||||
@@ -612,6 +625,9 @@ export class GuildOperationsService {
|
||||
if (featuresChanged) {
|
||||
patch.features = updatedFeatures;
|
||||
}
|
||||
if (endsInvitePause) {
|
||||
patch.invites_paused_until = null;
|
||||
}
|
||||
const preparedAssets = await this.prepareGuildAssetUpdates(currentGuild, data, patch);
|
||||
let updatedGuild: Guild;
|
||||
if (Object.keys(patch).length === 0) {
|
||||
@@ -630,6 +646,14 @@ export class GuildOperationsService {
|
||||
Logger.error({error, guildId}, 'Failed to commit asset changes after successful guild update');
|
||||
}
|
||||
await this.helpers.dispatchGuildUpdate(updatedGuild);
|
||||
if (featuresChanged) {
|
||||
await emitGuildFeaturesChanged({
|
||||
previous: currentGuild,
|
||||
updated: updatedGuild,
|
||||
actorId: userId,
|
||||
source: 'member',
|
||||
});
|
||||
}
|
||||
const guildSearchService = getGuildSearchService();
|
||||
if (guildSearchService) {
|
||||
let discoveryContext: GuildDiscoveryContext | undefined;
|
||||
@@ -748,18 +772,43 @@ export class GuildOperationsService {
|
||||
if (!guildData || guildData.owner_id !== user.id.toString()) {
|
||||
throw new MissingPermissionsError();
|
||||
}
|
||||
await this.performGuildDeletion(guildId, 'source_deleted');
|
||||
await this.scheduleGuildDeletion(guildId, 'source_deleted');
|
||||
}
|
||||
|
||||
async deleteGuildById(guildId: GuildID): Promise<void> {
|
||||
await this.performGuildDeletion(guildId, 'purge');
|
||||
await this.scheduleGuildDeletion(guildId, 'purge');
|
||||
}
|
||||
|
||||
private async performGuildDeletion(guildId: GuildID, copyMode: ChannelFollowerRemovalCopyMode): Promise<void> {
|
||||
private async scheduleGuildDeletion(guildId: GuildID, copyMode: ChannelFollowerRemovalCopyMode): Promise<void> {
|
||||
const guild = await this.guildRepository.findUnique(guildId);
|
||||
if (!guild) {
|
||||
throw new UnknownGuildError();
|
||||
}
|
||||
const features = new Set(guild.features);
|
||||
for (const feature of GUILD_DELETION_PENDING_FEATURES) {
|
||||
features.add(feature);
|
||||
}
|
||||
await this.guildRepository.upsertPartial(guildId, {features}, guild.toRow());
|
||||
await this.gatewayService.reloadGuild(guildId).catch((error) => {
|
||||
Logger.warn({guildId, error}, 'Failed to reload guild after scheduling deletion');
|
||||
});
|
||||
await this.gatewayService.dispatchGuild({
|
||||
guildId,
|
||||
event: 'GUILD_DELETE',
|
||||
data: {id: guildId.toString()},
|
||||
});
|
||||
await this.workerService.addJob(
|
||||
'guildProcessDeletion',
|
||||
{guildId: guildId.toString(), copyMode},
|
||||
{jobKey: `guild-deletion-${guildId}`},
|
||||
);
|
||||
}
|
||||
|
||||
async purgeGuild(guildId: GuildID, copyMode: ChannelFollowerRemovalCopyMode): Promise<void> {
|
||||
const guild = await this.guildRepository.findUnique(guildId);
|
||||
if (!guild) {
|
||||
return;
|
||||
}
|
||||
const channels = await this.channelRepository.channelData.listGuildChannels(guildId, 'complete');
|
||||
for (const channel of channels) {
|
||||
await scheduleDeletedChannelFollowerRemoval({channel, crossposts: this.channelRepository.crossposts, copyMode});
|
||||
@@ -770,25 +819,19 @@ export class GuildOperationsService {
|
||||
event: 'GUILD_DELETE',
|
||||
data: {id: guildId.toString()},
|
||||
});
|
||||
await Promise.all(
|
||||
members.map(async (member) => {
|
||||
await this.gatewayService.leaveGuild({userId: member.userId, guildId});
|
||||
}),
|
||||
);
|
||||
await Promise.all(members.map((member) => this.userRepository.deleteGuildSettings(member.userId, guildId)));
|
||||
await Promise.all(
|
||||
members.map(async (member) => {
|
||||
const user = await this.userRepository.findUnique(member.userId);
|
||||
if (user && !user.isBot) {
|
||||
await removeGuildFromUserFolders({
|
||||
userId: member.userId,
|
||||
guildId,
|
||||
userRepository: this.userRepository,
|
||||
gatewayService: this.gatewayService,
|
||||
});
|
||||
}
|
||||
}),
|
||||
);
|
||||
await mapWithConcurrency(members, GUILD_DELETION_MEMBER_CONCURRENCY, async (member) => {
|
||||
await this.gatewayService.leaveGuild({userId: member.userId, guildId});
|
||||
await this.userRepository.deleteGuildSettings(member.userId, guildId);
|
||||
const user = await this.userRepository.findUnique(member.userId);
|
||||
if (user && !user.isBot) {
|
||||
await removeGuildFromUserFolders({
|
||||
userId: member.userId,
|
||||
guildId,
|
||||
userRepository: this.userRepository,
|
||||
gatewayService: this.gatewayService,
|
||||
});
|
||||
}
|
||||
});
|
||||
const invites = await this.inviteRepository.listGuildInvites(guildId);
|
||||
await Promise.all(invites.map((invite) => this.inviteRepository.delete(invite.code)));
|
||||
const webhooks = await this.webhookRepository.listByGuild(guildId);
|
||||
|
||||
@@ -23,6 +23,7 @@ import type {GuildMemberSearchIndexService} from '@app/api/guild/services/member
|
||||
import type {GuildMemberValidationService} from '@app/api/guild/services/member/GuildMemberValidationService';
|
||||
import {emitActivity} from '@app/api/infrastructure/activity/ActivityEvents';
|
||||
import type {JoinSource} from '@app/api/infrastructure/activity/Contract.generated';
|
||||
import {emitGuildMemberRemoved} from '@app/api/infrastructure/activity/GuildEvents';
|
||||
import {contentModerationService} from '@app/api/infrastructure/ContentModerationService';
|
||||
import type {EntityAssetService, PreparedAssetUpload} from '@app/api/infrastructure/EntityAssetService';
|
||||
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
|
||||
@@ -388,6 +389,7 @@ export class GuildMemberOperationsService {
|
||||
);
|
||||
}
|
||||
await this.gatewayService.leaveGuild({userId: targetId, guildId});
|
||||
await emitGuildMemberRemoved({guildId, userId: targetId, actorId: userId, kind: 'kicked'});
|
||||
}
|
||||
|
||||
async addUserToGuild(
|
||||
@@ -541,6 +543,7 @@ export class GuildMemberOperationsService {
|
||||
});
|
||||
}
|
||||
await this.gatewayService.leaveGuild({userId, guildId});
|
||||
await emitGuildMemberRemoved({guildId, userId, actorId: null, kind: 'left'});
|
||||
}
|
||||
|
||||
private async validateTimeout(params: {
|
||||
|
||||
@@ -6,7 +6,14 @@ import {createTestBotAccount} from '@app/api/bot/tests/BotTestUtils';
|
||||
import {createPermissionOverwrite} from '@app/api/channel/tests/ChannelTestUtils';
|
||||
import {threadsRequest} from '@app/api/channel/tests/ThreadTestUtils';
|
||||
import {clearChannelThreadsTaintCacheForTesting} from '@app/api/experiment/ChannelThreadsGate';
|
||||
import {createChannel, createGuild} from '@app/api/guild/tests/GuildTestUtils';
|
||||
import {createChannel, createGuild, updateGuild} from '@app/api/guild/tests/GuildTestUtils';
|
||||
import {
|
||||
idleActivityEvents,
|
||||
resetActivityEventsForTests,
|
||||
startActivityEvents,
|
||||
} from '@app/api/infrastructure/activity/ActivityEvents';
|
||||
import type {ActivityPublisher} from '@app/api/infrastructure/activity/ActivitySpool';
|
||||
import type {Event} from '@app/api/infrastructure/activity/Contract.generated';
|
||||
import {setInjectedGatewayService} from '@app/api/middleware/ServiceRegistry';
|
||||
import {getGuildRepository} from '@app/api/middleware/ServiceSingletons';
|
||||
import {banUser} from '@app/api/moderation/tests/ModerationTestUtils';
|
||||
@@ -20,11 +27,13 @@ import {clearWorkerDependencies, setWorkerDependenciesForTest} from '@app/api/wo
|
||||
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
|
||||
import {ChannelTypes, Permissions} from '@fluxer/constants/src/ChannelConstants';
|
||||
import {DiscoveryCategories} from '@fluxer/constants/src/DiscoveryConstants';
|
||||
import {GuildFeatures} from '@fluxer/constants/src/GuildConstants';
|
||||
import type {
|
||||
DiscoveryApplicationResponse,
|
||||
DiscoveryChannelPreviewResponse,
|
||||
DiscoveryGuildListResponse,
|
||||
} from '@fluxer/schema/src/domains/guild/GuildDiscoverySchemas';
|
||||
import {MockKVProvider} from '@pkgs/kv_client/src/testing/MockKVProvider';
|
||||
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
import {afterEach, beforeEach, describe, expect, test} from 'vitest';
|
||||
|
||||
@@ -42,6 +51,14 @@ interface LiveGuildCounts {
|
||||
|
||||
const WORKER_HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
|
||||
|
||||
class CapturingPublisher implements ActivityPublisher {
|
||||
readonly events: Array<Event> = [];
|
||||
|
||||
async publish(_subject: string, payload: string): Promise<void> {
|
||||
this.events.push(JSON.parse(payload) as Event);
|
||||
}
|
||||
}
|
||||
|
||||
class LiveCountsGatewayService extends NoopGatewayService {
|
||||
constructor(private readonly liveCounts: Map<string, LiveGuildCounts>) {
|
||||
super();
|
||||
@@ -248,6 +265,46 @@ describe('Discovery Search and Join', () => {
|
||||
.expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.DISCOVERY_NOT_DISCOVERABLE)
|
||||
.execute();
|
||||
});
|
||||
test('should publish a refusal for a non-member discovery join while invites are disabled', async () => {
|
||||
const publisher = new CapturingPublisher();
|
||||
await startActivityEvents({publisher, kv: new MockKVProvider()});
|
||||
try {
|
||||
const owner = await createTestAccount(harness);
|
||||
const guild = await createGuild(harness, owner.token, 'Closed Discovery Guild');
|
||||
await setGuildMemberCount(harness, guild.id, 10);
|
||||
const admin = await createTestAccount(harness);
|
||||
await setUserACLs(harness, admin, ['admin:authenticate', 'discovery:review']);
|
||||
await applyAndApprove(
|
||||
harness,
|
||||
owner.token,
|
||||
admin.token,
|
||||
guild.id,
|
||||
'Join this closed community',
|
||||
DiscoveryCategories.GAMING,
|
||||
);
|
||||
await updateGuild(harness, owner.token, guild.id, {
|
||||
features: [...guild.features, GuildFeatures.INVITES_DISABLED],
|
||||
});
|
||||
const joiner = await createTestAccount(harness);
|
||||
for (const account of [owner, joiner]) {
|
||||
await createBuilder(harness, account.token)
|
||||
.post(`/discovery/guilds/${guild.id}/join`)
|
||||
.expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.INVITES_DISABLED)
|
||||
.execute();
|
||||
}
|
||||
await idleActivityEvents();
|
||||
const refusals = publisher.events.filter((event) => event.kind === 'guild_join_refused');
|
||||
expect(refusals.map((event) => [event.key, event.data])).toEqual([
|
||||
[
|
||||
joiner.userId,
|
||||
{guild_id: guild.id, user_id: joiner.userId, cause: 'invites_paused', join_source: 'discovery'},
|
||||
],
|
||||
]);
|
||||
} finally {
|
||||
resetActivityEventsForTests();
|
||||
}
|
||||
});
|
||||
|
||||
test('should not allow bot accounts to join discoverable guilds', async () => {
|
||||
const owner = await createTestAccount(harness);
|
||||
const guild = await createGuild(harness, owner.token, 'No Bots Guild');
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {createGuildID, createUserID, type GuildID} from '@app/api/BrandedTypes';
|
||||
import {createTestGuildStack} from '@app/api/channel/tests/CrosspostTestUtils';
|
||||
import {processRecordedGuildDeletions, recordGuildDeletions} from '@app/api/guild/tests/GuildDeletionTestUtils';
|
||||
import {acceptInvite, createChannelInvite, createGuild, getChannel} from '@app/api/guild/tests/GuildTestUtils';
|
||||
import {getGuildRepository} from '@app/api/middleware/ServiceSingletons';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {NoopGatewayService} from '@app/api/test/NoopGatewayService';
|
||||
import type {RecordingWorkerService} from '@app/api/test/RecordingWorkerService';
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
import {createBuilder} from '@app/api/test/TestRequestBuilder';
|
||||
import {GuildFeatures} from '@fluxer/constants/src/GuildConstants';
|
||||
import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas';
|
||||
import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi} from 'vitest';
|
||||
|
||||
describe('Guild deletion job', () => {
|
||||
let harness: ApiTestHarness;
|
||||
let worker: RecordingWorkerService;
|
||||
|
||||
beforeAll(async () => {
|
||||
harness = await createApiTestHarness();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await harness.reset();
|
||||
worker = recordGuildDeletions();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await harness?.shutdown();
|
||||
});
|
||||
|
||||
async function guildWithMember(): Promise<{owner: TestAccount; member: TestAccount; guild: GuildResponse}> {
|
||||
const owner = await createTestAccount(harness);
|
||||
const member = await createTestAccount(harness);
|
||||
const guild = await createGuild(harness, owner.token, 'Deletion Guild');
|
||||
const channel = await getChannel(harness, owner.token, guild.system_channel_id!);
|
||||
const invite = await createChannelInvite(harness, owner.token, channel.id);
|
||||
await acceptInvite(harness, member.token, invite.code);
|
||||
return {owner, member, guild};
|
||||
}
|
||||
|
||||
async function deleteAsOwner(owner: TestAccount, guildId: string): Promise<void> {
|
||||
await createBuilder(harness, owner.token)
|
||||
.post(`/guilds/${guildId}/delete`)
|
||||
.body({password: owner.password})
|
||||
.expect(HTTP_STATUS.NO_CONTENT)
|
||||
.execute();
|
||||
}
|
||||
|
||||
function storedGuild(guildId: string) {
|
||||
return getGuildRepository().findUnique(createGuildID(BigInt(guildId)));
|
||||
}
|
||||
|
||||
test('the request hides the guild and leaves the purge to the worker', async () => {
|
||||
const {owner, member, guild} = await guildWithMember();
|
||||
const dispatched = vi.spyOn(NoopGatewayService.prototype, 'dispatchGuild');
|
||||
const leaves = vi.spyOn(NoopGatewayService.prototype, 'leaveGuild');
|
||||
await deleteAsOwner(owner, guild.id);
|
||||
const jobs = worker.byTask('guildProcessDeletion');
|
||||
expect(jobs).toHaveLength(1);
|
||||
expect(jobs[0]?.payload).toEqual({guildId: guild.id, copyMode: 'source_deleted'});
|
||||
expect(jobs[0]?.options?.jobKey).toBe(`guild-deletion-${guild.id}`);
|
||||
expect(dispatched).toHaveBeenCalledWith(expect.objectContaining({event: 'GUILD_DELETE'}));
|
||||
expect(leaves).not.toHaveBeenCalled();
|
||||
const pending = await storedGuild(guild.id);
|
||||
expect(pending?.features.has(GuildFeatures.UNAVAILABLE_FOR_EVERYONE)).toBe(true);
|
||||
expect(pending?.features.has(GuildFeatures.UNAVAILABLE_HIDDEN)).toBe(true);
|
||||
await createBuilder(harness, member.token).get(`/guilds/${guild.id}`).expect(HTTP_STATUS.FORBIDDEN).execute();
|
||||
await processRecordedGuildDeletions(worker);
|
||||
expect(await storedGuild(guild.id)).toBeNull();
|
||||
await createBuilder(harness, owner.token).get(`/guilds/${guild.id}`).expect(HTTP_STATUS.NOT_FOUND).execute();
|
||||
});
|
||||
|
||||
test('an admin deletion enqueues a purge job', async () => {
|
||||
const admin = await createTestAccount(harness);
|
||||
await setUserACLs(harness, admin, ['admin:authenticate', 'guild:lookup', 'guild:delete']);
|
||||
const guild = await createGuild(harness, admin.token, 'Admin Deletion Guild');
|
||||
await createBuilder(harness, admin.token)
|
||||
.delete(`/admin/guilds/${guild.id}`)
|
||||
.body(null)
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
expect(worker.byTask('guildProcessDeletion').map((job) => job.payload)).toEqual([
|
||||
{guildId: guild.id, copyMode: 'purge'},
|
||||
]);
|
||||
await processRecordedGuildDeletions(worker);
|
||||
expect(await storedGuild(guild.id)).toBeNull();
|
||||
});
|
||||
|
||||
test('a purge interrupted by a gateway failure completes when retried', async () => {
|
||||
const {owner, guild} = await guildWithMember();
|
||||
await deleteAsOwner(owner, guild.id);
|
||||
const leaveGuild = NoopGatewayService.prototype.leaveGuild;
|
||||
const failing = vi
|
||||
.spyOn(NoopGatewayService.prototype, 'leaveGuild')
|
||||
.mockRejectedValueOnce(new Error('gateway timeout'))
|
||||
.mockImplementation(function (this: NoopGatewayService, params) {
|
||||
return leaveGuild.call(this, params);
|
||||
});
|
||||
await expect(processRecordedGuildDeletions(worker)).rejects.toThrow('gateway timeout');
|
||||
expect(await storedGuild(guild.id)).not.toBeNull();
|
||||
await processRecordedGuildDeletions(worker);
|
||||
expect(await storedGuild(guild.id)).toBeNull();
|
||||
await processRecordedGuildDeletions(worker);
|
||||
expect(failing).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('member teardown runs with bounded concurrency', async () => {
|
||||
const {owner, guild} = await guildWithMember();
|
||||
await deleteAsOwner(owner, guild.id);
|
||||
const guildId: GuildID = createGuildID(BigInt(guild.id));
|
||||
const repository = getGuildRepository();
|
||||
const realMembers = await repository.listMembers(guildId);
|
||||
const extraMembers = Array.from({length: 120}, (_, index) => ({
|
||||
...realMembers[0]!,
|
||||
userId: createUserID(BigInt(realMembers[0]!.userId) + BigInt(index + 1)),
|
||||
}));
|
||||
vi.spyOn(Object.getPrototypeOf(repository), 'listMembers').mockResolvedValueOnce([...realMembers, ...extraMembers]);
|
||||
let inFlight = 0;
|
||||
let peak = 0;
|
||||
vi.spyOn(NoopGatewayService.prototype, 'leaveGuild').mockImplementation(async () => {
|
||||
inFlight += 1;
|
||||
peak = Math.max(peak, inFlight);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
inFlight -= 1;
|
||||
});
|
||||
await createTestGuildStack().guildService.data.operations.purgeGuild(guildId, 'source_deleted');
|
||||
expect(peak).toBeGreaterThan(1);
|
||||
expect(peak).toBeLessThanOrEqual(25);
|
||||
expect(await storedGuild(guild.id)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,25 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createGuildID} from '@app/api/BrandedTypes';
|
||||
import {resetApiServicesForTesting} from '@app/api/CreateApiContext';
|
||||
import type {ChannelFollowerRemovalCopyMode} from '@app/api/channel/services/ChannelFollowers';
|
||||
import {createTestGuildStack} from '@app/api/channel/tests/CrosspostTestUtils';
|
||||
import {setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';
|
||||
import {RecordingWorkerService} from '@app/api/test/RecordingWorkerService';
|
||||
|
||||
export function recordGuildDeletions(): RecordingWorkerService {
|
||||
const worker = new RecordingWorkerService();
|
||||
setInjectedWorkerService(worker);
|
||||
resetApiServicesForTesting();
|
||||
return worker;
|
||||
}
|
||||
|
||||
export async function processRecordedGuildDeletions(worker: RecordingWorkerService): Promise<void> {
|
||||
const operations = createTestGuildStack().guildService.data.operations;
|
||||
for (const job of worker.byTask('guildProcessDeletion')) {
|
||||
await operations.purgeGuild(
|
||||
createGuildID(BigInt(String(job.payload.guildId))),
|
||||
job.payload.copyMode as ChannelFollowerRemovalCopyMode,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createTestAccount} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {processRecordedGuildDeletions, recordGuildDeletions} from '@app/api/guild/tests/GuildDeletionTestUtils';
|
||||
import {acceptInvite, createChannelInvite, createGuild, getChannel} from '@app/api/guild/tests/GuildTestUtils';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
@@ -113,11 +114,13 @@ describe('Guild Folder Operations', () => {
|
||||
const invite = await createChannelInvite(harness, owner.token, channel.id);
|
||||
await acceptInvite(harness, member1.token, invite.code);
|
||||
await acceptInvite(harness, member2.token, invite.code);
|
||||
const worker = recordGuildDeletions();
|
||||
await createBuilder(harness, owner.token)
|
||||
.post(`/guilds/${guild.id}/delete`)
|
||||
.body({password: owner.password})
|
||||
.expect(HTTP_STATUS.NO_CONTENT)
|
||||
.executeWithResponse();
|
||||
await processRecordedGuildDeletions(worker);
|
||||
const {json: ownerSettings} = await createBuilder<UserSettingsResponse>(harness, owner.token)
|
||||
.get('/users/@me/settings')
|
||||
.expect(HTTP_STATUS.OK)
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createGuildID, createUserID, type GuildID} from '@app/api/BrandedTypes';
|
||||
import {SYSTEM_USER_ID} from '@app/api/constants/Core';
|
||||
import type {GuildRow} from '@app/api/database/types/GuildTypes';
|
||||
import {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService';
|
||||
import {
|
||||
applyPauseGuildInvites,
|
||||
GUILD_INVITE_PAUSE_SCHEDULE_KEY,
|
||||
type GuildInvitePauseDeps,
|
||||
liftExpiredGuildInvitePauses,
|
||||
} from '@app/api/guild/services/GuildInvitePauseApplier';
|
||||
import {resetActivityEventsForTests, startActivityEvents} from '@app/api/infrastructure/activity/ActivityEvents';
|
||||
import type {ActivityPublisher} from '@app/api/infrastructure/activity/ActivitySpool';
|
||||
import type {ActionEnvelope, Event} from '@app/api/infrastructure/activity/Contract.generated';
|
||||
import {Guild} from '@app/api/models/Guild';
|
||||
import {applyAction} from '@app/api/worker/AccountActionConsumer';
|
||||
import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType';
|
||||
import {GuildFeatures} from '@fluxer/constants/src/GuildConstants';
|
||||
import {MockKVProvider} from '@pkgs/kv_client/src/testing/MockKVProvider';
|
||||
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
const GUILD_ID = '1472623911696138261';
|
||||
const NOW = 1_760_000_000_000;
|
||||
const HOUR_MS = 3_600_000;
|
||||
const BASE_FEATURES = [GuildFeatures.BANNER];
|
||||
|
||||
type PauseEnvelope = Extract<ActionEnvelope, {type: 'pause_guild_invites'}>;
|
||||
|
||||
function pauseEnvelope(overrides: Partial<PauseEnvelope> = {}): PauseEnvelope {
|
||||
return {
|
||||
v: 1,
|
||||
id: 'a:11:guild:0',
|
||||
key: `guild:${GUILD_ID}`,
|
||||
issued_at_ms: NOW,
|
||||
expires_at_ms: NOW + 60_000,
|
||||
type: 'pause_guild_invites',
|
||||
guild_id: GUILD_ID,
|
||||
on: true,
|
||||
until_ms: NOW + 2 * HOUR_MS,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function guildRow(features: Iterable<string>, pausedUntil: Date | null = null): GuildRow {
|
||||
return {
|
||||
guild_id: createGuildID(BigInt(GUILD_ID)),
|
||||
owner_id: createUserID(1472623911696138262n),
|
||||
name: 'Paused',
|
||||
vanity_url_code: null,
|
||||
icon_hash: null,
|
||||
banner_hash: null,
|
||||
banner_width: null,
|
||||
banner_height: null,
|
||||
splash_hash: null,
|
||||
splash_width: null,
|
||||
splash_height: null,
|
||||
splash_card_alignment: null,
|
||||
embed_splash_hash: null,
|
||||
embed_splash_width: null,
|
||||
embed_splash_height: null,
|
||||
features: new Set(features),
|
||||
verification_level: 0,
|
||||
mfa_level: 0,
|
||||
nsfw_level: 0,
|
||||
explicit_content_filter: 0,
|
||||
default_message_notifications: 0,
|
||||
system_channel_id: null,
|
||||
system_channel_flags: 0,
|
||||
rules_channel_id: null,
|
||||
afk_channel_id: null,
|
||||
afk_timeout: 0,
|
||||
disabled_operations: 0,
|
||||
member_count: 12,
|
||||
audit_logs_indexed_at: null,
|
||||
members_indexed_at: null,
|
||||
message_history_cutoff: null,
|
||||
invites_paused_until: pausedUntil,
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
class FakeGuilds {
|
||||
row: GuildRow | null = null;
|
||||
writes = 0;
|
||||
|
||||
current(): Guild {
|
||||
return new Guild(this.row!);
|
||||
}
|
||||
|
||||
async findUnique(_guildId: GuildID): Promise<Guild | null> {
|
||||
return this.row ? new Guild(this.row) : null;
|
||||
}
|
||||
|
||||
async upsertPartial(_guildId: GuildID, patch: Partial<GuildRow>): Promise<Guild> {
|
||||
this.writes++;
|
||||
this.row = {...this.row!, ...patch, version: this.row!.version + 1};
|
||||
return new Guild(this.row);
|
||||
}
|
||||
}
|
||||
|
||||
class CapturingPublisher implements ActivityPublisher {
|
||||
readonly events: Array<Event> = [];
|
||||
|
||||
async publish(_subject: string, payload: string): Promise<void> {
|
||||
this.events.push(JSON.parse(payload) as Event);
|
||||
}
|
||||
|
||||
featureChanges() {
|
||||
return this.events.flatMap((event) => (event.kind === 'guild_features_changed' ? [event] : []));
|
||||
}
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
guilds: FakeGuilds;
|
||||
schedule: MockKVProvider;
|
||||
dispatched: Array<unknown>;
|
||||
audits: Array<{userId: bigint; action: number; changes: unknown}>;
|
||||
clock: {now: number};
|
||||
deps: GuildInvitePauseDeps;
|
||||
}
|
||||
|
||||
function harness(): Harness {
|
||||
const guilds = new FakeGuilds();
|
||||
const schedule = new MockKVProvider();
|
||||
const h: Harness = {guilds, schedule, dispatched: [], audits: [], clock: {now: NOW}, deps: null as never};
|
||||
const auditService = new GuildAuditLogService(null as never, null as never, null as never, null as never);
|
||||
h.deps = {
|
||||
guilds,
|
||||
locks: {acquireLock: async () => 'token', releaseLock: async () => true},
|
||||
schedule,
|
||||
gateway: {
|
||||
dispatchGuild: async (params) => {
|
||||
h.dispatched.push(params.data);
|
||||
},
|
||||
},
|
||||
auditLog: {
|
||||
computeChanges: (previous, next) => auditService.computeChanges(previous, next),
|
||||
record: async (params) => {
|
||||
h.audits.push({userId: params.userId, action: params.action, changes: params.changes});
|
||||
},
|
||||
},
|
||||
now: () => h.clock.now,
|
||||
};
|
||||
return h;
|
||||
}
|
||||
|
||||
async function scheduled(h: Harness): Promise<Array<string>> {
|
||||
return h.schedule.zrangebyscore(GUILD_INVITE_PAUSE_SCHEDULE_KEY, '-inf', '+inf');
|
||||
}
|
||||
|
||||
describe('guild invite pause', () => {
|
||||
let h: Harness;
|
||||
let publisher: CapturingPublisher;
|
||||
|
||||
beforeEach(async () => {
|
||||
h = harness();
|
||||
publisher = new CapturingPublisher();
|
||||
await startActivityEvents({publisher, kv: new MockKVProvider()});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetActivityEventsForTests();
|
||||
});
|
||||
|
||||
it('leaves a pause the guild set itself alone and reports it as already paused', async () => {
|
||||
h.guilds.row = guildRow([...BASE_FEATURES, GuildFeatures.INVITES_DISABLED]);
|
||||
const outcome = await applyPauseGuildInvites(h.deps, pauseEnvelope());
|
||||
expect(outcome).toEqual({
|
||||
action_id: 'a:11:guild:0',
|
||||
action_type: 'pause_guild_invites',
|
||||
status: 'noop',
|
||||
detail: 'already_paused',
|
||||
observed: null,
|
||||
guild_id: GUILD_ID,
|
||||
});
|
||||
expect(h.guilds.writes).toBe(0);
|
||||
expect(await scheduled(h)).toEqual([]);
|
||||
expect(publisher.events).toEqual([]);
|
||||
const lifted = await applyPauseGuildInvites(h.deps, pauseEnvelope({on: false}));
|
||||
expect(lifted.status).toBe('noop');
|
||||
expect(h.guilds.current().features.has(GuildFeatures.INVITES_DISABLED)).toBe(true);
|
||||
});
|
||||
|
||||
it('pauses invites, records it as the system user and schedules the lift', async () => {
|
||||
h.guilds.row = guildRow(BASE_FEATURES);
|
||||
const outcome = await applyPauseGuildInvites(h.deps, pauseEnvelope());
|
||||
expect(outcome).toMatchObject({status: 'applied', detail: `until_ms=${NOW + 2 * HOUR_MS}`, guild_id: GUILD_ID});
|
||||
const guild = h.guilds.current();
|
||||
expect([...guild.features].sort()).toEqual(
|
||||
[...BASE_FEATURES, GuildFeatures.INVITES_DISABLED, GuildFeatures.RAID_DETECTED].sort(),
|
||||
);
|
||||
expect(guild.invitesPausedUntil?.getTime()).toBe(NOW + 2 * HOUR_MS);
|
||||
expect(h.dispatched).toHaveLength(1);
|
||||
expect(h.audits).toEqual([
|
||||
{
|
||||
userId: SYSTEM_USER_ID,
|
||||
action: AuditLogActionType.GUILD_UPDATE,
|
||||
changes: [
|
||||
{
|
||||
key: 'features',
|
||||
old_value: [...BASE_FEATURES],
|
||||
new_value: [...BASE_FEATURES, GuildFeatures.INVITES_DISABLED, GuildFeatures.RAID_DETECTED].sort(),
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
expect(await scheduled(h)).toEqual([GUILD_ID]);
|
||||
const [event] = publisher.featureChanges();
|
||||
expect(event?.key).toBe(`guild:${GUILD_ID}`);
|
||||
expect(event?.meta.channel).toBe('worker');
|
||||
expect(event?.data).toEqual({
|
||||
guild_id: GUILD_ID,
|
||||
actor_id: null,
|
||||
source: 'action',
|
||||
added: [GuildFeatures.INVITES_DISABLED, GuildFeatures.RAID_DETECTED],
|
||||
removed: [],
|
||||
invites_paused_until_ms: NOW + 2 * HOUR_MS,
|
||||
});
|
||||
});
|
||||
|
||||
it('extends its own pause but never shortens it', async () => {
|
||||
h.guilds.row = guildRow(BASE_FEATURES);
|
||||
await applyPauseGuildInvites(h.deps, pauseEnvelope());
|
||||
const extended = await applyPauseGuildInvites(
|
||||
h.deps,
|
||||
pauseEnvelope({id: 'a:11:guild:1', until_ms: NOW + 3 * HOUR_MS}),
|
||||
);
|
||||
expect(extended.status).toBe('applied');
|
||||
expect(h.guilds.current().invitesPausedUntil?.getTime()).toBe(NOW + 3 * HOUR_MS);
|
||||
expect(
|
||||
await h.schedule.zrangebyscore(GUILD_INVITE_PAUSE_SCHEDULE_KEY, NOW + 3 * HOUR_MS, NOW + 3 * HOUR_MS),
|
||||
).toEqual([GUILD_ID]);
|
||||
const shorter = await applyPauseGuildInvites(h.deps, pauseEnvelope({id: 'a:11:guild:2', until_ms: NOW + HOUR_MS}));
|
||||
expect(shorter).toMatchObject({status: 'noop', detail: `until_ms=${NOW + 3 * HOUR_MS}`});
|
||||
expect(h.guilds.current().invitesPausedUntil?.getTime()).toBe(NOW + 3 * HOUR_MS);
|
||||
expect(publisher.featureChanges().map((event) => event.data.invites_paused_until_ms)).toEqual([
|
||||
NOW + 2 * HOUR_MS,
|
||||
NOW + 3 * HOUR_MS,
|
||||
]);
|
||||
});
|
||||
|
||||
it('lifts only its own pause and removes both features', async () => {
|
||||
h.guilds.row = guildRow(BASE_FEATURES);
|
||||
await applyPauseGuildInvites(h.deps, pauseEnvelope());
|
||||
const outcome = await applyAction(
|
||||
{js: {} as never, state: {} as never, guildInvites: h.deps, now: () => NOW},
|
||||
pauseEnvelope({id: 'a:11:guild:3', on: false}),
|
||||
);
|
||||
expect(outcome).toMatchObject({status: 'applied', guild_id: GUILD_ID});
|
||||
const guild = h.guilds.current();
|
||||
expect([...guild.features]).toEqual(BASE_FEATURES);
|
||||
expect(guild.invitesPausedUntil).toBeNull();
|
||||
expect(await scheduled(h)).toEqual([]);
|
||||
expect(publisher.featureChanges().at(-1)?.data).toMatchObject({
|
||||
source: 'action',
|
||||
removed: [GuildFeatures.INVITES_DISABLED, GuildFeatures.RAID_DETECTED],
|
||||
invites_paused_until_ms: null,
|
||||
});
|
||||
const again = await applyPauseGuildInvites(h.deps, pauseEnvelope({id: 'a:11:guild:4', on: false}));
|
||||
expect(again.status).toBe('noop');
|
||||
});
|
||||
|
||||
it('ignores a late lift for an earlier window once the pause was extended', async () => {
|
||||
h.guilds.row = guildRow(BASE_FEATURES);
|
||||
await applyPauseGuildInvites(h.deps, pauseEnvelope());
|
||||
await applyPauseGuildInvites(h.deps, pauseEnvelope({id: 'a:11:guild:5', until_ms: NOW + 3 * HOUR_MS}));
|
||||
const late = await applyPauseGuildInvites(
|
||||
h.deps,
|
||||
pauseEnvelope({id: 'a:11:guild:6', on: false, until_ms: NOW + 2 * HOUR_MS}),
|
||||
);
|
||||
expect(late).toMatchObject({status: 'noop', detail: `until_ms=${NOW + 3 * HOUR_MS}`});
|
||||
expect(h.guilds.current().features.has(GuildFeatures.INVITES_DISABLED)).toBe(true);
|
||||
expect(h.guilds.current().invitesPausedUntil?.getTime()).toBe(NOW + 3 * HOUR_MS);
|
||||
const current = await applyPauseGuildInvites(
|
||||
h.deps,
|
||||
pauseEnvelope({id: 'a:11:guild:7', on: false, until_ms: NOW + 3 * HOUR_MS}),
|
||||
);
|
||||
expect(current.status).toBe('applied');
|
||||
expect(h.guilds.current().invitesPausedUntil).toBeNull();
|
||||
});
|
||||
|
||||
it('keeps the lift scheduled when the gateway fails after the write and a redelivery changes nothing', async () => {
|
||||
h.guilds.row = guildRow(BASE_FEATURES);
|
||||
const dispatch = h.deps.gateway.dispatchGuild;
|
||||
let failures = 1;
|
||||
h.deps.gateway = {
|
||||
dispatchGuild: async (params) => {
|
||||
if (failures-- > 0) throw new Error('gateway timeout');
|
||||
await dispatch(params);
|
||||
},
|
||||
};
|
||||
const first = await applyPauseGuildInvites(h.deps, pauseEnvelope());
|
||||
expect(first).toMatchObject({status: 'applied', guild_id: GUILD_ID});
|
||||
expect(h.audits).toHaveLength(1);
|
||||
expect(publisher.featureChanges()).toHaveLength(1);
|
||||
const redelivered = await applyPauseGuildInvites(h.deps, pauseEnvelope());
|
||||
expect(redelivered).toMatchObject({status: 'noop', detail: `until_ms=${NOW + 2 * HOUR_MS}`});
|
||||
expect(h.guilds.writes).toBe(1);
|
||||
expect(await scheduled(h)).toEqual([GUILD_ID]);
|
||||
h.clock.now = NOW + 2 * HOUR_MS;
|
||||
expect(await liftExpiredGuildInvitePauses(h.deps, 100)).toBe(1);
|
||||
expect([...h.guilds.current().features]).toEqual(BASE_FEATURES);
|
||||
});
|
||||
|
||||
it('restores a lost schedule entry when the same pause is redelivered', async () => {
|
||||
h.guilds.row = guildRow(
|
||||
[...BASE_FEATURES, GuildFeatures.INVITES_DISABLED, GuildFeatures.RAID_DETECTED],
|
||||
new Date(NOW + 2 * HOUR_MS),
|
||||
);
|
||||
const outcome = await applyPauseGuildInvites(h.deps, pauseEnvelope());
|
||||
expect(outcome.status).toBe('noop');
|
||||
expect(h.guilds.writes).toBe(0);
|
||||
expect(await scheduled(h)).toEqual([GUILD_ID]);
|
||||
});
|
||||
|
||||
it('writes nothing when scheduling fails so the retry applies the pause', async () => {
|
||||
h.guilds.row = guildRow(BASE_FEATURES);
|
||||
const zadd = h.schedule.zadd.bind(h.schedule);
|
||||
let failures = 1;
|
||||
h.deps.schedule = {
|
||||
zadd: async (...args: Parameters<typeof zadd>) => {
|
||||
if (failures-- > 0) throw new Error('kv unavailable');
|
||||
return zadd(...args);
|
||||
},
|
||||
zrem: h.schedule.zrem.bind(h.schedule),
|
||||
zrangebyscore: h.schedule.zrangebyscore.bind(h.schedule),
|
||||
};
|
||||
await expect(applyPauseGuildInvites(h.deps, pauseEnvelope())).rejects.toThrow('kv unavailable');
|
||||
expect(h.guilds.writes).toBe(0);
|
||||
expect((await applyPauseGuildInvites(h.deps, pauseEnvelope())).status).toBe('applied');
|
||||
expect(await scheduled(h)).toEqual([GUILD_ID]);
|
||||
});
|
||||
|
||||
it('routes the action through the consumer and answers every outcome with the guild', async () => {
|
||||
h.guilds.row = guildRow(BASE_FEATURES);
|
||||
const consumer = {js: {} as never, state: {} as never, guildInvites: h.deps, now: () => NOW};
|
||||
expect(await applyAction(consumer, pauseEnvelope())).toMatchObject({status: 'applied', guild_id: GUILD_ID});
|
||||
expect(await applyAction(consumer, pauseEnvelope({expires_at_ms: NOW}))).toMatchObject({
|
||||
status: 'expired',
|
||||
guild_id: GUILD_ID,
|
||||
});
|
||||
expect(await applyAction(consumer, {...pauseEnvelope(), v: 2})).toMatchObject({
|
||||
status: 'unsupported',
|
||||
guild_id: GUILD_ID,
|
||||
});
|
||||
h.guilds.row = guildRow([...BASE_FEATURES, GuildFeatures.INVITES_DISABLED]);
|
||||
expect(await applyAction(consumer, pauseEnvelope({id: 'a:11:guild:5'}))).toMatchObject({
|
||||
status: 'noop',
|
||||
detail: 'already_paused',
|
||||
guild_id: GUILD_ID,
|
||||
});
|
||||
expect(h.guilds.writes).toBe(1);
|
||||
});
|
||||
|
||||
it('removes a notice staff set before the pause when the pause ends', async () => {
|
||||
h.guilds.row = guildRow([...BASE_FEATURES, GuildFeatures.RAID_DETECTED]);
|
||||
expect((await applyPauseGuildInvites(h.deps, pauseEnvelope())).status).toBe('applied');
|
||||
h.clock.now = NOW + 2 * HOUR_MS;
|
||||
expect(await liftExpiredGuildInvitePauses(h.deps, 100)).toBe(1);
|
||||
expect([...h.guilds.current().features]).toEqual(BASE_FEATURES);
|
||||
});
|
||||
|
||||
it('reports a missing guild as ineligible', async () => {
|
||||
expect((await applyPauseGuildInvites(h.deps, pauseEnvelope())).status).toBe('ineligible');
|
||||
});
|
||||
|
||||
it('lifts an expired pause on the sweep without any action', async () => {
|
||||
h.guilds.row = guildRow(BASE_FEATURES);
|
||||
await applyPauseGuildInvites(h.deps, pauseEnvelope());
|
||||
expect(await liftExpiredGuildInvitePauses(h.deps, 100)).toBe(0);
|
||||
expect(h.guilds.current().features.has(GuildFeatures.INVITES_DISABLED)).toBe(true);
|
||||
h.clock.now = NOW + 2 * HOUR_MS;
|
||||
expect(await liftExpiredGuildInvitePauses(h.deps, 100)).toBe(1);
|
||||
const guild = h.guilds.current();
|
||||
expect([...guild.features]).toEqual(BASE_FEATURES);
|
||||
expect(guild.invitesPausedUntil).toBeNull();
|
||||
expect(await scheduled(h)).toEqual([]);
|
||||
expect(publisher.featureChanges().at(-1)?.data).toMatchObject({
|
||||
source: 'system',
|
||||
actor_id: null,
|
||||
removed: [GuildFeatures.INVITES_DISABLED, GuildFeatures.RAID_DETECTED],
|
||||
});
|
||||
});
|
||||
|
||||
it('drops a sweep entry once the pause is no longer ours and reschedules an extended one', async () => {
|
||||
h.guilds.row = guildRow([...BASE_FEATURES, GuildFeatures.INVITES_DISABLED]);
|
||||
await h.schedule.zadd(GUILD_INVITE_PAUSE_SCHEDULE_KEY, NOW - 1, GUILD_ID);
|
||||
expect(await liftExpiredGuildInvitePauses(h.deps, 100)).toBe(0);
|
||||
expect(await scheduled(h)).toEqual([]);
|
||||
expect(h.guilds.current().features.has(GuildFeatures.INVITES_DISABLED)).toBe(true);
|
||||
h.guilds.row = guildRow(
|
||||
[...BASE_FEATURES, GuildFeatures.INVITES_DISABLED, GuildFeatures.RAID_DETECTED],
|
||||
new Date(NOW + HOUR_MS),
|
||||
);
|
||||
await h.schedule.zadd(GUILD_INVITE_PAUSE_SCHEDULE_KEY, NOW - 1, GUILD_ID);
|
||||
expect(await liftExpiredGuildInvitePauses(h.deps, 100)).toBe(0);
|
||||
expect(await h.schedule.zrangebyscore(GUILD_INVITE_PAUSE_SCHEDULE_KEY, NOW + HOUR_MS, NOW + HOUR_MS)).toEqual([
|
||||
GUILD_ID,
|
||||
]);
|
||||
expect(h.guilds.writes).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -43,6 +43,7 @@ function createGuild(overrides: Partial<GuildRow> = {}): Guild {
|
||||
audit_logs_indexed_at: null,
|
||||
members_indexed_at: null,
|
||||
message_history_cutoff: null,
|
||||
invites_paused_until: null,
|
||||
version: 1,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
@@ -225,3 +225,55 @@ describe('StorageService.listObjects', () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
function copySourceFor(endpoint: string, forcePathStyle: boolean): Promise<unknown> {
|
||||
const service = new StorageService({
|
||||
endpoint,
|
||||
forcePathStyle,
|
||||
region: 'us-east-1',
|
||||
accessKeyId: 'fluxer',
|
||||
secretAccessKey: 'fluxer-secret',
|
||||
});
|
||||
let copySource: unknown;
|
||||
Object.assign(service, {
|
||||
client: {
|
||||
send: async (command: {input: {CopySource?: unknown}}) => {
|
||||
copySource = command.input.CopySource;
|
||||
return {};
|
||||
},
|
||||
},
|
||||
});
|
||||
return service
|
||||
.copyObject({
|
||||
sourceBucket: 'fluxer-uploads',
|
||||
sourceKey: 'a b/c+d.pdf',
|
||||
destinationBucket: 'fluxer-cdn',
|
||||
destinationKey: 'attachments/1/2/c.pdf',
|
||||
})
|
||||
.then(() => copySource);
|
||||
}
|
||||
|
||||
describe('StorageService.copyObject copy source', () => {
|
||||
it.each([
|
||||
['http://seaweedfs:8333', true],
|
||||
['http://seaweedfs:8333/', true],
|
||||
['https://s3.example.test', false],
|
||||
])(
|
||||
'names only the bucket and key for an endpoint without a path (%s, path style %s)',
|
||||
async (endpoint, pathStyle) => {
|
||||
await expect(copySourceFor(endpoint, pathStyle)).resolves.toBe('fluxer-uploads/a%20b/c%2Bd.pdf');
|
||||
},
|
||||
);
|
||||
|
||||
it('puts the endpoint path in front of the bucket with path-style addressing', async () => {
|
||||
await expect(copySourceFor('https://s3.example.test/real%20bucket/sub/', true)).resolves.toBe(
|
||||
'real%20bucket/sub/fluxer-uploads/a%20b/c%2Bd.pdf',
|
||||
);
|
||||
});
|
||||
|
||||
it('puts the endpoint path between the bucket and the key with virtual-hosted addressing', async () => {
|
||||
await expect(copySourceFor('https://s3.example.test/real%20bucket/sub', false)).resolves.toBe(
|
||||
'fluxer-uploads/real%20bucket/sub/a%20b/c%2Bd.pdf',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -115,6 +115,13 @@ const REJECTED_SERVER_SIDE_COPY_ERRORS = new Set([
|
||||
'MethodNotAllowed',
|
||||
]);
|
||||
|
||||
function encodedEndpointPath(endpoint: string): Array<string> {
|
||||
return (URL.parse(endpoint)?.pathname ?? '')
|
||||
.split('/')
|
||||
.filter((segment) => segment !== '')
|
||||
.map((segment) => encodeURIComponent(decodeURIComponent(segment)));
|
||||
}
|
||||
|
||||
function isRejectedServerSideCopy(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof S3ServiceException &&
|
||||
@@ -126,6 +133,7 @@ export class StorageService implements IStorageService {
|
||||
private readonly client: S3Client;
|
||||
private readonly presignClient: S3Client;
|
||||
private readonly provider: S3ProviderSettings;
|
||||
private readonly endpointPath: Array<string>;
|
||||
|
||||
constructor(provider?: S3ProviderSettings) {
|
||||
this.provider = provider ?? {
|
||||
@@ -136,6 +144,7 @@ export class StorageService implements IStorageService {
|
||||
accessKeyId: Config.s3.accessKeyId,
|
||||
secretAccessKey: Config.s3.secretAccessKey,
|
||||
};
|
||||
this.endpointPath = encodedEndpointPath(this.provider.endpoint);
|
||||
this.client = buildPooledS3Client({
|
||||
endpoint: this.provider.endpoint,
|
||||
region: this.provider.region,
|
||||
@@ -492,7 +501,7 @@ export class StorageService implements IStorageService {
|
||||
new CopyObjectCommand({
|
||||
Bucket: destinationBucket,
|
||||
Key: destinationKey,
|
||||
CopySource: `${encodeURIComponent(sourceBucket)}/${sourceKey.split('/').map(encodeURIComponent).join('/')}`,
|
||||
CopySource: this.copySource(sourceBucket, sourceKey),
|
||||
ContentType: newContentType,
|
||||
MetadataDirective: newContentType ? 'REPLACE' : undefined,
|
||||
}),
|
||||
@@ -508,6 +517,14 @@ export class StorageService implements IStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
private copySource(bucket: string, key: string): string {
|
||||
const encodedBucket = encodeURIComponent(bucket);
|
||||
const location = this.provider.forcePathStyle
|
||||
? [...this.endpointPath, encodedBucket]
|
||||
: [encodedBucket, ...this.endpointPath];
|
||||
return [...location, ...key.split('/').map(encodeURIComponent)].join('/');
|
||||
}
|
||||
|
||||
private async copyObjectThroughApi(
|
||||
{
|
||||
sourceBucket,
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {StorageService} from '@app/api/infrastructure/StorageService';
|
||||
import {server} from '@app/api/test/msw/server';
|
||||
import {HttpResponse, http} from 'msw';
|
||||
import {beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
const HOST = 's3.single-bucket.test';
|
||||
const REAL_BUCKET = 'realbucket';
|
||||
|
||||
function errorResponse(code: string, status: number): HttpResponse<string> {
|
||||
return new HttpResponse(
|
||||
`<?xml version="1.0" encoding="UTF-8"?><Error><Code>${code}</Code><Message></Message><RequestId>tx0</RequestId></Error>`,
|
||||
{status, headers: {'Content-Type': 'application/xml'}},
|
||||
);
|
||||
}
|
||||
|
||||
function splitBucket(path: string): {bucket: string; key: string} {
|
||||
const [bucket = '', ...key] = path.split('/');
|
||||
return {bucket, key: key.join('/')};
|
||||
}
|
||||
|
||||
function singleBucketStore() {
|
||||
const objects = new Map<string, {body: Uint8Array; contentType: string}>();
|
||||
const resolve = (url: URL): {bucket: string; key: string} => {
|
||||
const path = decodeURIComponent(url.pathname.slice(1));
|
||||
if (url.hostname === HOST) {
|
||||
return splitBucket(path);
|
||||
}
|
||||
return {bucket: url.hostname.slice(0, -HOST.length - 1), key: path};
|
||||
};
|
||||
server.use(
|
||||
http.all(/^https:\/\/([a-z0-9-]+\.)?s3\.single-bucket\.test\//u, async ({request}) => {
|
||||
const target = resolve(new URL(request.url));
|
||||
if (target.bucket !== REAL_BUCKET) {
|
||||
return errorResponse('NoSuchBucket', 404);
|
||||
}
|
||||
if (request.method === 'GET') {
|
||||
const object = objects.get(target.key);
|
||||
if (!object) {
|
||||
return errorResponse('NoSuchKey', 404);
|
||||
}
|
||||
return new HttpResponse(object.body, {
|
||||
status: 200,
|
||||
headers: {'Content-Type': object.contentType, 'Content-Length': String(object.body.length)},
|
||||
});
|
||||
}
|
||||
if (request.method !== 'PUT') {
|
||||
return errorResponse('NotImplemented', 501);
|
||||
}
|
||||
const copySource = request.headers.get('x-amz-copy-source');
|
||||
if (!copySource) {
|
||||
objects.set(target.key, {
|
||||
body: new Uint8Array(await request.arrayBuffer()),
|
||||
contentType: request.headers.get('content-type') ?? 'application/octet-stream',
|
||||
});
|
||||
return new HttpResponse(null, {status: 200, headers: {ETag: '"x"'}});
|
||||
}
|
||||
const source = splitBucket(decodeURIComponent(copySource.replace(/^\//u, '')));
|
||||
if (source.bucket !== REAL_BUCKET) {
|
||||
return errorResponse('NoSuchBucket', 404);
|
||||
}
|
||||
const object = objects.get(source.key);
|
||||
if (!object) {
|
||||
return errorResponse('NoSuchKey', 404);
|
||||
}
|
||||
objects.set(target.key, {...object, contentType: request.headers.get('content-type') ?? object.contentType});
|
||||
return new HttpResponse(
|
||||
'<?xml version="1.0" encoding="UTF-8"?><CopyObjectResult><ETag>"x"</ETag></CopyObjectResult>',
|
||||
{status: 200, headers: {'Content-Type': 'application/xml'}},
|
||||
);
|
||||
}),
|
||||
);
|
||||
return objects;
|
||||
}
|
||||
|
||||
function storage(endpoint: string, forcePathStyle: boolean): StorageService {
|
||||
return new StorageService({
|
||||
endpoint,
|
||||
forcePathStyle,
|
||||
region: 'us-east-1',
|
||||
accessKeyId: 'TEST',
|
||||
secretAccessKey: 'TEST',
|
||||
});
|
||||
}
|
||||
|
||||
const PDF = new TextEncoder().encode('%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\n%%EOF\n');
|
||||
|
||||
describe('StorageService with every bucket inside one real bucket', () => {
|
||||
let objects: ReturnType<typeof singleBucketStore>;
|
||||
|
||||
beforeEach(() => {
|
||||
objects = singleBucketStore();
|
||||
});
|
||||
|
||||
it('copies a non-media attachment when the endpoint names the bucket with path-style addressing', async () => {
|
||||
const service = storage(`https://${HOST}/${REAL_BUCKET}`, true);
|
||||
await service.uploadObject({bucket: 'fluxer-uploads', key: 'upload 1', body: PDF});
|
||||
|
||||
await expect(
|
||||
service.copyObjectWithMetadataStripping({
|
||||
sourceBucket: 'fluxer-uploads',
|
||||
sourceKey: 'upload 1',
|
||||
destinationBucket: 'fluxer-cdn',
|
||||
destinationKey: 'attachments/1/2/file.pdf',
|
||||
contentType: 'application/pdf',
|
||||
}),
|
||||
).resolves.toBeNull();
|
||||
|
||||
expect(objects.get('fluxer-cdn/attachments/1/2/file.pdf')?.contentType).toBe('application/pdf');
|
||||
await expect(service.readObject('fluxer-cdn', 'attachments/1/2/file.pdf')).resolves.toEqual(PDF);
|
||||
});
|
||||
|
||||
it('copies within a bucket whose virtual host keeps the endpoint path as a key prefix', async () => {
|
||||
const service = storage(`https://${HOST}/prefix`, false);
|
||||
await service.uploadObject({bucket: REAL_BUCKET, key: 'upload-2', body: PDF});
|
||||
|
||||
await service.copyObject({
|
||||
sourceBucket: REAL_BUCKET,
|
||||
sourceKey: 'upload-2',
|
||||
destinationBucket: REAL_BUCKET,
|
||||
destinationKey: 'attachments/1/3/file.pdf',
|
||||
newContentType: 'application/pdf',
|
||||
});
|
||||
|
||||
expect(objects.get('prefix/attachments/1/3/file.pdf')?.contentType).toBe('application/pdf');
|
||||
await expect(service.readObject(REAL_BUCKET, 'attachments/1/3/file.pdf')).resolves.toEqual(PDF);
|
||||
});
|
||||
});
|
||||
@@ -4,10 +4,10 @@ export type Id = string;
|
||||
export type Flags64 = string;
|
||||
export type Channel = "stable" | "canary" | "worker" | "internal" | "import" | "other";
|
||||
export type Meta = { ip: string | null, country: string | null, ua: string | null, locale: string | null, channel: Channel, request_id: string | null, };
|
||||
export type Kind = "registration" | "email_changed" | "profile_updated" | "account_changed" | "admin_action" | "account_deleted" | "report_filed" | "email_bounced" | "action_outcome" | "login" | "session_started" | "guild_joined" | "dm_opened" | "message_created" | "message_updated" | "friend_request" | "http_errors" | "user_blocked" | "guild_member_banned" | "guild_member_unbanned" | "report_resolved";
|
||||
export type Event = { v: number, id: string, at_ms: number, key: string, meta: Meta, } & ({ "kind": "registration", "data": Registration } | { "kind": "email_changed", "data": EmailChanged } | { "kind": "profile_updated", "data": ProfileUpdated } | { "kind": "account_changed", "data": AccountChanged } | { "kind": "admin_action", "data": AdminAction } | { "kind": "account_deleted", "data": AccountDeleted } | { "kind": "report_filed", "data": ReportFiled } | { "kind": "email_bounced", "data": EmailBounced } | { "kind": "action_outcome", "data": ActionOutcome } | { "kind": "login", "data": Login } | { "kind": "session_started", "data": SessionStarted } | { "kind": "guild_joined", "data": GuildJoined } | { "kind": "dm_opened", "data": DmOpened } | { "kind": "message_created", "data": MessageCreated } | { "kind": "message_updated", "data": MessageUpdated } | { "kind": "friend_request", "data": FriendRequest } | { "kind": "http_errors", "data": HttpErrors } | { "kind": "user_blocked", "data": UserBlocked } | { "kind": "guild_member_banned", "data": GuildMemberBanned } | { "kind": "guild_member_unbanned", "data": GuildMemberUnbanned } | { "kind": "report_resolved", "data": ReportResolved });
|
||||
export type Body = { "kind": "registration", "data": Registration } | { "kind": "email_changed", "data": EmailChanged } | { "kind": "profile_updated", "data": ProfileUpdated } | { "kind": "account_changed", "data": AccountChanged } | { "kind": "admin_action", "data": AdminAction } | { "kind": "account_deleted", "data": AccountDeleted } | { "kind": "report_filed", "data": ReportFiled } | { "kind": "email_bounced", "data": EmailBounced } | { "kind": "action_outcome", "data": ActionOutcome } | { "kind": "login", "data": Login } | { "kind": "session_started", "data": SessionStarted } | { "kind": "guild_joined", "data": GuildJoined } | { "kind": "dm_opened", "data": DmOpened } | { "kind": "message_created", "data": MessageCreated } | { "kind": "message_updated", "data": MessageUpdated } | { "kind": "friend_request", "data": FriendRequest } | { "kind": "http_errors", "data": HttpErrors } | { "kind": "user_blocked", "data": UserBlocked } | { "kind": "guild_member_banned", "data": GuildMemberBanned } | { "kind": "guild_member_unbanned", "data": GuildMemberUnbanned } | { "kind": "report_resolved", "data": ReportResolved };
|
||||
export type Registration = { user_id: Id, method: RegMethod, email: string | null, username: string, username_user_chosen: boolean, global_name: string | null, locale: string | null, timezone: string | null, invite_code: string | null, flags: Flags64, };
|
||||
export type Kind = "registration" | "email_changed" | "profile_updated" | "account_changed" | "admin_action" | "account_deleted" | "report_filed" | "email_bounced" | "action_outcome" | "login" | "session_started" | "guild_joined" | "dm_opened" | "message_created" | "message_updated" | "friend_request" | "http_errors" | "user_blocked" | "guild_member_banned" | "guild_member_unbanned" | "report_resolved" | "guild_created" | "guild_features_changed" | "invite_created" | "invite_deleted" | "guild_member_removed" | "guild_join_refused";
|
||||
export type Event = { v: number, id: string, at_ms: number, key: string, meta: Meta, } & ({ "kind": "registration", "data": Registration } | { "kind": "email_changed", "data": EmailChanged } | { "kind": "profile_updated", "data": ProfileUpdated } | { "kind": "account_changed", "data": AccountChanged } | { "kind": "admin_action", "data": AdminAction } | { "kind": "account_deleted", "data": AccountDeleted } | { "kind": "report_filed", "data": ReportFiled } | { "kind": "email_bounced", "data": EmailBounced } | { "kind": "action_outcome", "data": ActionOutcome } | { "kind": "login", "data": Login } | { "kind": "session_started", "data": SessionStarted } | { "kind": "guild_joined", "data": GuildJoined } | { "kind": "dm_opened", "data": DmOpened } | { "kind": "message_created", "data": MessageCreated } | { "kind": "message_updated", "data": MessageUpdated } | { "kind": "friend_request", "data": FriendRequest } | { "kind": "http_errors", "data": HttpErrors } | { "kind": "user_blocked", "data": UserBlocked } | { "kind": "guild_member_banned", "data": GuildMemberBanned } | { "kind": "guild_member_unbanned", "data": GuildMemberUnbanned } | { "kind": "report_resolved", "data": ReportResolved } | { "kind": "guild_created", "data": GuildCreated } | { "kind": "guild_features_changed", "data": GuildFeaturesChanged } | { "kind": "invite_created", "data": InviteCreated } | { "kind": "invite_deleted", "data": InviteDeleted } | { "kind": "guild_member_removed", "data": GuildMemberRemoved } | { "kind": "guild_join_refused", "data": GuildJoinRefused });
|
||||
export type Body = { "kind": "registration", "data": Registration } | { "kind": "email_changed", "data": EmailChanged } | { "kind": "profile_updated", "data": ProfileUpdated } | { "kind": "account_changed", "data": AccountChanged } | { "kind": "admin_action", "data": AdminAction } | { "kind": "account_deleted", "data": AccountDeleted } | { "kind": "report_filed", "data": ReportFiled } | { "kind": "email_bounced", "data": EmailBounced } | { "kind": "action_outcome", "data": ActionOutcome } | { "kind": "login", "data": Login } | { "kind": "session_started", "data": SessionStarted } | { "kind": "guild_joined", "data": GuildJoined } | { "kind": "dm_opened", "data": DmOpened } | { "kind": "message_created", "data": MessageCreated } | { "kind": "message_updated", "data": MessageUpdated } | { "kind": "friend_request", "data": FriendRequest } | { "kind": "http_errors", "data": HttpErrors } | { "kind": "user_blocked", "data": UserBlocked } | { "kind": "guild_member_banned", "data": GuildMemberBanned } | { "kind": "guild_member_unbanned", "data": GuildMemberUnbanned } | { "kind": "report_resolved", "data": ReportResolved } | { "kind": "guild_created", "data": GuildCreated } | { "kind": "guild_features_changed", "data": GuildFeaturesChanged } | { "kind": "invite_created", "data": InviteCreated } | { "kind": "invite_deleted", "data": InviteDeleted } | { "kind": "guild_member_removed", "data": GuildMemberRemoved } | { "kind": "guild_join_refused", "data": GuildJoinRefused };
|
||||
export type Registration = { user_id: Id, method: RegMethod, email: string | null, username: string, username_user_chosen: boolean, global_name: string | null, locale: string | null, timezone: string | null, invite_code: string | null, flags: Flags64, date_of_birth: string | null, };
|
||||
export type RegMethod = "password" | "unclaimed" | "oauth" | "other";
|
||||
export type EmailChanged = { user_id: Id, new_email: string, was_unclaimed: boolean, has_ever_purchased: boolean, };
|
||||
export type ProfileUpdated = { user_id: Id, username: string | null, global_name: string | null, bio: string | null, avatar_hash: string | null, pronouns: string | null, };
|
||||
@@ -38,9 +38,18 @@ export type BanBy = "moderator" | "staff";
|
||||
export type ReportResolved = { report_id: Id, reporter_id: Id, category: string, target_type: ReportTarget, reported_user_id: Id | null, outcome: ReportOutcome, resolved_by: ResolvedBy, };
|
||||
export type ReportOutcome = "actioned" | "no_violation" | "duplicate" | "auto_resolved" | "unspecified";
|
||||
export type ResolvedBy = "staff" | "system";
|
||||
export type ActionEnvelope = { v: number, id: string, key: string, issued_at_ms: number, expires_at_ms: number, } & ({ "type": "set_account_limit", user_id: Id, on: boolean, } | { "type": "temp_ban_ip", ip: string, until_ms: number, } | { "type": "limit_new_conversations", user_id: Id, on: boolean, until_ms: number, } | { "type": "hide_recent_messages", user_id: Id, since_ms: number, on: boolean, } | { "type": "hide_profile", user_id: Id, on: boolean, } | { "type": "delete_user_messages", user_id: Id, on: boolean, });
|
||||
export type Action = { "type": "set_account_limit", user_id: Id, on: boolean, } | { "type": "temp_ban_ip", ip: string, until_ms: number, } | { "type": "limit_new_conversations", user_id: Id, on: boolean, until_ms: number, } | { "type": "hide_recent_messages", user_id: Id, since_ms: number, on: boolean, } | { "type": "hide_profile", user_id: Id, on: boolean, } | { "type": "delete_user_messages", user_id: Id, on: boolean, };
|
||||
export type ActionOutcome = { action_id: string, action_type: string, status: OutcomeStatus, detail: string | null, observed: Observed | null, user_id?: Id, };
|
||||
export type GuildCreated = { guild_id: Id, owner_id: Id, member_count: number | null, };
|
||||
export type GuildFeaturesChanged = { guild_id: Id, actor_id: Id | null, source: FeatureSource, added: Array<string>, removed: Array<string>, invites_paused_until_ms: number | null, };
|
||||
export type FeatureSource = "member" | "staff" | "action" | "system";
|
||||
export type InviteCreated = { guild_id: Id, channel_id: Id | null, code: string, inviter_id: Id | null, max_uses: number, max_age_s: number, temporary: boolean, };
|
||||
export type InviteDeleted = { guild_id: Id, code: string, };
|
||||
export type GuildMemberRemoved = { guild_id: Id, user_id: Id, actor_id: Id | null, kind: RemovalKind, };
|
||||
export type RemovalKind = "left" | "kicked";
|
||||
export type GuildJoinRefused = { guild_id: Id, user_id: Id | null, cause: RefusalCause, join_source: JoinSource, };
|
||||
export type RefusalCause = "invites_paused" | "other";
|
||||
export type ActionEnvelope = { v: number, id: string, key: string, issued_at_ms: number, expires_at_ms: number, } & ({ "type": "set_account_limit", user_id: Id, on: boolean, } | { "type": "temp_ban_ip", ip: string, until_ms: number, } | { "type": "limit_new_conversations", user_id: Id, on: boolean, until_ms: number, } | { "type": "hide_recent_messages", user_id: Id, since_ms: number, on: boolean, } | { "type": "hide_profile", user_id: Id, on: boolean, } | { "type": "delete_user_messages", user_id: Id, on: boolean, } | { "type": "pause_guild_invites", guild_id: Id, on: boolean, until_ms: number, });
|
||||
export type Action = { "type": "set_account_limit", user_id: Id, on: boolean, } | { "type": "temp_ban_ip", ip: string, until_ms: number, } | { "type": "limit_new_conversations", user_id: Id, on: boolean, until_ms: number, } | { "type": "hide_recent_messages", user_id: Id, since_ms: number, on: boolean, } | { "type": "hide_profile", user_id: Id, on: boolean, } | { "type": "delete_user_messages", user_id: Id, on: boolean, } | { "type": "pause_guild_invites", guild_id: Id, on: boolean, until_ms: number, };
|
||||
export type ActionOutcome = { action_id: string, action_type: string, status: OutcomeStatus, detail: string | null, observed: Observed | null, user_id?: Id, guild_id?: Id, };
|
||||
export type OutcomeStatus = "applied" | "noop" | "conflict" | "expired" | "ineligible" | "exempt" | "unsupported" | "failed";
|
||||
export type Observed = { flags: Flags64, deleted: boolean, };
|
||||
|
||||
@@ -49,6 +58,7 @@ export const EVENTS_STREAM = 'EVENTS';
|
||||
export const ACTIONS_STREAM = 'ACTIONS';
|
||||
export const ACTION_PARTITIONS = 64;
|
||||
export const MAX_ACTION_ATTEMPTS = 10;
|
||||
export const INVITES_ALREADY_PAUSED = 'already_paused';
|
||||
export const LISTS_BUCKET = 'lists';
|
||||
export const SHARED_LISTS = ['ip_blocked', 'email_tld_blocked', 'email_domain_exempt'] as const;
|
||||
export const HEADER = {
|
||||
@@ -80,6 +90,12 @@ export const EVENT_KINDS: ReadonlyArray<EventKind> = [
|
||||
'guild_member_banned',
|
||||
'guild_member_unbanned',
|
||||
'report_resolved',
|
||||
'guild_created',
|
||||
'guild_features_changed',
|
||||
'invite_created',
|
||||
'invite_deleted',
|
||||
'guild_member_removed',
|
||||
'guild_join_refused',
|
||||
];
|
||||
export const EVENT_MAJOR: Record<EventKind, number> = {
|
||||
registration: 1,
|
||||
@@ -103,6 +119,12 @@ export const EVENT_MAJOR: Record<EventKind, number> = {
|
||||
guild_member_banned: 1,
|
||||
guild_member_unbanned: 1,
|
||||
report_resolved: 1,
|
||||
guild_created: 1,
|
||||
guild_features_changed: 1,
|
||||
invite_created: 1,
|
||||
invite_deleted: 1,
|
||||
guild_member_removed: 1,
|
||||
guild_join_refused: 1,
|
||||
};
|
||||
export const EVENT_TTL: Record<EventKind, string> = {
|
||||
registration: 'never',
|
||||
@@ -126,6 +148,12 @@ export const EVENT_TTL: Record<EventKind, string> = {
|
||||
guild_member_banned: '3024000',
|
||||
guild_member_unbanned: '3024000',
|
||||
report_resolved: 'never',
|
||||
guild_created: '3024000',
|
||||
guild_features_changed: '3024000',
|
||||
invite_created: '3024000',
|
||||
invite_deleted: '3024000',
|
||||
guild_member_removed: '3024000',
|
||||
guild_join_refused: '3024000',
|
||||
};
|
||||
export const EVENT_CLASS: Record<EventKind, 'fact' | 'signal'> = {
|
||||
registration: 'fact',
|
||||
@@ -149,10 +177,17 @@ export const EVENT_CLASS: Record<EventKind, 'fact' | 'signal'> = {
|
||||
guild_member_banned: 'fact',
|
||||
guild_member_unbanned: 'fact',
|
||||
report_resolved: 'fact',
|
||||
guild_created: 'fact',
|
||||
guild_features_changed: 'fact',
|
||||
invite_created: 'fact',
|
||||
invite_deleted: 'fact',
|
||||
guild_member_removed: 'signal',
|
||||
guild_join_refused: 'signal',
|
||||
};
|
||||
export const effectsConsumer = (p: number): string => `effects-${String(p).padStart(2, '0')}`;
|
||||
export function keyToken(key: string): string {
|
||||
if (/^\d+$/.test(key)) return key;
|
||||
return createHash('sha256').update(key).digest('hex').slice(0, 16);
|
||||
}
|
||||
export const guildKey = (guildId: Id): string => `guild:${guildId}`;
|
||||
export const eventSubject = (kind: EventKind, key: string): string => `evt.in.${kind}.${keyToken(key)}`;
|
||||
@@ -0,0 +1,110 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {GuildID, UserID} from '@app/api/BrandedTypes';
|
||||
import {emitActivity} from '@app/api/infrastructure/activity/ActivityEvents';
|
||||
import {
|
||||
type FeatureSource,
|
||||
guildKey,
|
||||
type JoinSource,
|
||||
type Meta,
|
||||
type RemovalKind,
|
||||
} from '@app/api/infrastructure/activity/Contract.generated';
|
||||
import type {Guild} from '@app/api/models/Guild';
|
||||
import type {Invite} from '@app/api/models/Invite';
|
||||
|
||||
export async function emitGuildCreated(guild: Guild): Promise<void> {
|
||||
await emitActivity(
|
||||
'guild_created',
|
||||
guild.ownerId.toString(),
|
||||
{
|
||||
guild_id: guild.id.toString(),
|
||||
owner_id: guild.ownerId.toString(),
|
||||
member_count: guild.memberCount,
|
||||
},
|
||||
null,
|
||||
guild.id.toString(),
|
||||
);
|
||||
}
|
||||
|
||||
export async function emitGuildFeaturesChanged(params: {
|
||||
previous: Guild;
|
||||
updated: Guild;
|
||||
actorId: UserID | null;
|
||||
source: FeatureSource;
|
||||
meta?: Meta | null;
|
||||
}): Promise<void> {
|
||||
const {previous, updated} = params;
|
||||
const added = [...updated.features].filter((feature) => !previous.features.has(feature)).sort();
|
||||
const removed = [...previous.features].filter((feature) => !updated.features.has(feature)).sort();
|
||||
const pausedUntilMs = updated.invitesPausedUntil?.getTime() ?? null;
|
||||
const pauseChanged = pausedUntilMs !== (previous.invitesPausedUntil?.getTime() ?? null);
|
||||
if (added.length === 0 && removed.length === 0 && !pauseChanged) return;
|
||||
const guildId = updated.id.toString();
|
||||
await emitActivity(
|
||||
'guild_features_changed',
|
||||
guildKey(guildId),
|
||||
{
|
||||
guild_id: guildId,
|
||||
actor_id: params.actorId?.toString() ?? null,
|
||||
source: params.source,
|
||||
added,
|
||||
removed,
|
||||
invites_paused_until_ms: pausedUntilMs,
|
||||
},
|
||||
params.meta ?? null,
|
||||
`${guildId}:${updated.version}`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function emitInviteCreated(invite: Invite): Promise<void> {
|
||||
if (!invite.guildId) return;
|
||||
const guildId = invite.guildId.toString();
|
||||
await emitActivity(
|
||||
'invite_created',
|
||||
guildKey(guildId),
|
||||
{
|
||||
guild_id: guildId,
|
||||
channel_id: invite.channelId?.toString() ?? null,
|
||||
code: invite.code,
|
||||
inviter_id: invite.inviterId?.toString() ?? null,
|
||||
max_uses: invite.maxUses,
|
||||
max_age_s: invite.maxAge,
|
||||
temporary: invite.temporary,
|
||||
},
|
||||
null,
|
||||
invite.code,
|
||||
);
|
||||
}
|
||||
|
||||
export async function emitInviteDeleted(invite: Invite): Promise<void> {
|
||||
if (!invite.guildId) return;
|
||||
const guildId = invite.guildId.toString();
|
||||
await emitActivity('invite_deleted', guildKey(guildId), {guild_id: guildId, code: invite.code}, null, invite.code);
|
||||
}
|
||||
|
||||
export async function emitGuildMemberRemoved(params: {
|
||||
guildId: GuildID;
|
||||
userId: UserID;
|
||||
actorId: UserID | null;
|
||||
kind: RemovalKind;
|
||||
}): Promise<void> {
|
||||
await emitActivity('guild_member_removed', params.userId.toString(), {
|
||||
guild_id: params.guildId.toString(),
|
||||
user_id: params.userId.toString(),
|
||||
actor_id: params.actorId?.toString() ?? null,
|
||||
kind: params.kind,
|
||||
});
|
||||
}
|
||||
|
||||
export async function emitGuildJoinRefused(params: {
|
||||
guildId: GuildID;
|
||||
userId: UserID;
|
||||
joinSource: JoinSource;
|
||||
}): Promise<void> {
|
||||
await emitActivity('guild_join_refused', params.userId.toString(), {
|
||||
guild_id: params.guildId.toString(),
|
||||
user_id: params.userId.toString(),
|
||||
cause: 'invites_paused',
|
||||
join_source: params.joinSource,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createTestAccount, setUserACLs} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {createGuildID} from '@app/api/BrandedTypes';
|
||||
import {applyPauseGuildInvites, type GuildInvitePauseDeps} from '@app/api/guild/services/GuildInvitePauseApplier';
|
||||
import {
|
||||
acceptInvite,
|
||||
createChannelInvite,
|
||||
createGuild,
|
||||
deleteInvite,
|
||||
leaveGuild,
|
||||
setupTestGuildWithMembers,
|
||||
updateGuild,
|
||||
} from '@app/api/guild/tests/GuildTestUtils';
|
||||
import {
|
||||
idleActivityEvents,
|
||||
resetActivityEventsForTests,
|
||||
startActivityEvents,
|
||||
} from '@app/api/infrastructure/activity/ActivityEvents';
|
||||
import type {ActivityPublisher} from '@app/api/infrastructure/activity/ActivitySpool';
|
||||
import type {Event} from '@app/api/infrastructure/activity/Contract.generated';
|
||||
import {getGatewayService, getKVClient} from '@app/api/middleware/ServiceRegistry';
|
||||
import {getCacheService, getGuildAuditLogService, getGuildRepository} from '@app/api/middleware/ServiceSingletons';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
import {createBuilder} from '@app/api/test/TestRequestBuilder';
|
||||
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
|
||||
import {GuildFeatures} from '@fluxer/constants/src/GuildConstants';
|
||||
import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas';
|
||||
import {MockKVProvider} from '@pkgs/kv_client/src/testing/MockKVProvider';
|
||||
import {afterEach, beforeEach, describe, expect, test} from 'vitest';
|
||||
|
||||
const HOUR_MS = 3_600_000;
|
||||
|
||||
class CapturingPublisher implements ActivityPublisher {
|
||||
readonly events: Array<Event> = [];
|
||||
|
||||
async publish(_subject: string, payload: string): Promise<void> {
|
||||
this.events.push(JSON.parse(payload) as Event);
|
||||
}
|
||||
|
||||
of<K extends Event['kind']>(kind: K): Array<Extract<Event, {kind: K}>> {
|
||||
return this.events.filter((event): event is Extract<Event, {kind: K}> => event.kind === kind);
|
||||
}
|
||||
}
|
||||
|
||||
function pauseDeps(): GuildInvitePauseDeps {
|
||||
return {
|
||||
guilds: getGuildRepository(),
|
||||
locks: getCacheService(),
|
||||
schedule: getKVClient(),
|
||||
gateway: getGatewayService(),
|
||||
auditLog: getGuildAuditLogService(),
|
||||
};
|
||||
}
|
||||
|
||||
async function pauseInvites(guildId: string, untilMs: number): Promise<void> {
|
||||
const outcome = await applyPauseGuildInvites(pauseDeps(), {
|
||||
v: 1,
|
||||
id: `a:11:${guildId}:0`,
|
||||
key: `guild:${guildId}`,
|
||||
issued_at_ms: Date.now(),
|
||||
expires_at_ms: Date.now() + 60_000,
|
||||
type: 'pause_guild_invites',
|
||||
guild_id: guildId,
|
||||
on: true,
|
||||
until_ms: untilMs,
|
||||
});
|
||||
expect(outcome.status).toBe('applied');
|
||||
}
|
||||
|
||||
async function storedPause(guildId: string): Promise<{features: Set<string>; until: Date | null}> {
|
||||
const guild = await getGuildRepository().findUnique(createGuildID(BigInt(guildId)));
|
||||
return {features: new Set(guild!.features), until: guild!.invitesPausedUntil};
|
||||
}
|
||||
|
||||
describe('guild activity events', () => {
|
||||
let harness: ApiTestHarness;
|
||||
let publisher: CapturingPublisher;
|
||||
|
||||
beforeEach(async () => {
|
||||
harness = await createApiTestHarness();
|
||||
publisher = new CapturingPublisher();
|
||||
await startActivityEvents({publisher, kv: new MockKVProvider()});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
resetActivityEventsForTests();
|
||||
await harness?.shutdown();
|
||||
});
|
||||
|
||||
test('creating a guild publishes who owns it', async () => {
|
||||
const owner = await createTestAccount(harness);
|
||||
const guild = await createGuild(harness, owner.token, 'Fresh guild');
|
||||
const [created] = publisher.of('guild_created');
|
||||
expect(created?.key).toBe(owner.userId);
|
||||
expect(created?.data).toEqual({guild_id: guild.id, owner_id: owner.userId, member_count: 1});
|
||||
});
|
||||
|
||||
test('a member feature toggle publishes the actor and the member source', async () => {
|
||||
const owner = await createTestAccount(harness);
|
||||
const guild = await createGuild(harness, owner.token, 'Toggle guild');
|
||||
await updateGuild(harness, owner.token, guild.id, {features: [...guild.features, GuildFeatures.INVITES_DISABLED]});
|
||||
const [changed] = publisher.of('guild_features_changed');
|
||||
expect(changed?.key).toBe(`guild:${guild.id}`);
|
||||
expect(changed?.data).toEqual({
|
||||
guild_id: guild.id,
|
||||
actor_id: owner.userId,
|
||||
source: 'member',
|
||||
added: [GuildFeatures.INVITES_DISABLED],
|
||||
removed: [],
|
||||
invites_paused_until_ms: null,
|
||||
});
|
||||
});
|
||||
|
||||
test('creating and deleting an invite publishes both with the invite settings', async () => {
|
||||
const owner = await createTestAccount(harness);
|
||||
const guild = await createGuild(harness, owner.token, 'Invite guild');
|
||||
const invite = await createChannelInvite(harness, owner.token, guild.system_channel_id!);
|
||||
await deleteInvite(harness, owner.token, invite.code);
|
||||
const [created] = publisher.of('invite_created');
|
||||
expect(created?.key).toBe(`guild:${guild.id}`);
|
||||
expect(created?.data).toEqual({
|
||||
guild_id: guild.id,
|
||||
channel_id: guild.system_channel_id,
|
||||
code: invite.code,
|
||||
inviter_id: owner.userId,
|
||||
max_uses: invite.max_uses,
|
||||
max_age_s: invite.max_age,
|
||||
temporary: false,
|
||||
});
|
||||
const [deleted] = publisher.of('invite_deleted');
|
||||
expect(deleted?.key).toBe(`guild:${guild.id}`);
|
||||
expect(deleted?.data).toEqual({guild_id: guild.id, code: invite.code});
|
||||
});
|
||||
|
||||
test('leaving and being kicked publish the removal kind and actor', async () => {
|
||||
const {owner, members, guild} = await setupTestGuildWithMembers(harness, 2);
|
||||
const [leaver, kicked] = members;
|
||||
await leaveGuild(harness, leaver!.token, guild.id);
|
||||
await createBuilder(harness, owner.token)
|
||||
.delete(`/guilds/${guild.id}/members/${kicked!.userId}`)
|
||||
.expect(HTTP_STATUS.NO_CONTENT)
|
||||
.execute();
|
||||
await idleActivityEvents();
|
||||
expect(publisher.of('guild_member_removed').map((event) => [event.key, event.data])).toEqual([
|
||||
[leaver!.userId, {guild_id: guild.id, user_id: leaver!.userId, actor_id: null, kind: 'left'}],
|
||||
[kicked!.userId, {guild_id: guild.id, user_id: kicked!.userId, actor_id: owner.userId, kind: 'kicked'}],
|
||||
]);
|
||||
});
|
||||
|
||||
test('a join refused while invites are paused publishes the refusal only for non-members', async () => {
|
||||
const owner = await createTestAccount(harness);
|
||||
const joiner = await createTestAccount(harness);
|
||||
const guild = await createGuild(harness, owner.token, 'Paused guild');
|
||||
const invite = await createChannelInvite(harness, owner.token, guild.system_channel_id!);
|
||||
await pauseInvites(guild.id, Date.now() + HOUR_MS);
|
||||
for (const account of [owner, joiner]) {
|
||||
await createBuilder(harness, account.token)
|
||||
.post(`/invites/${invite.code}`)
|
||||
.body(null)
|
||||
.expect(HTTP_STATUS.FORBIDDEN, APIErrorCodes.INVITES_DISABLED)
|
||||
.execute();
|
||||
}
|
||||
await idleActivityEvents();
|
||||
const refusals = publisher.of('guild_join_refused');
|
||||
expect(refusals).toHaveLength(1);
|
||||
const [refused] = refusals;
|
||||
expect(refused?.key).toBe(joiner.userId);
|
||||
expect(refused?.data).toEqual({
|
||||
guild_id: guild.id,
|
||||
user_id: joiner.userId,
|
||||
cause: 'invites_paused',
|
||||
join_source: 'invite',
|
||||
});
|
||||
});
|
||||
|
||||
describe('managed invite pause', () => {
|
||||
let owner: Awaited<ReturnType<typeof createTestAccount>>;
|
||||
let guild: GuildResponse;
|
||||
let untilMs: number;
|
||||
|
||||
beforeEach(async () => {
|
||||
owner = await createTestAccount(harness);
|
||||
guild = await createGuild(harness, owner.token, 'Managed pause');
|
||||
untilMs = Date.now() + 2 * HOUR_MS;
|
||||
await pauseInvites(guild.id, untilMs);
|
||||
});
|
||||
|
||||
test('the owner turning invites back on ends the pause and drops the notice', async () => {
|
||||
const paused = await storedPause(guild.id);
|
||||
expect(paused.until?.getTime()).toBe(untilMs);
|
||||
expect(paused.features.has(GuildFeatures.RAID_DETECTED)).toBe(true);
|
||||
const features = [...paused.features].filter((feature) => feature !== GuildFeatures.INVITES_DISABLED);
|
||||
const updated = await updateGuild(harness, owner.token, guild.id, {features});
|
||||
expect(updated.features).not.toContain(GuildFeatures.INVITES_DISABLED);
|
||||
expect(updated.features).not.toContain(GuildFeatures.RAID_DETECTED);
|
||||
const after = await storedPause(guild.id);
|
||||
expect(after.until).toBeNull();
|
||||
const memberChange = publisher.of('guild_features_changed').at(-1);
|
||||
expect(memberChange?.data).toEqual({
|
||||
guild_id: guild.id,
|
||||
actor_id: owner.userId,
|
||||
source: 'member',
|
||||
added: [],
|
||||
removed: [GuildFeatures.INVITES_DISABLED, GuildFeatures.RAID_DETECTED],
|
||||
invites_paused_until_ms: null,
|
||||
});
|
||||
const joiner = await createTestAccount(harness);
|
||||
const invite = await createChannelInvite(harness, owner.token, guild.system_channel_id!);
|
||||
await acceptInvite(harness, joiner.token, invite.code);
|
||||
});
|
||||
|
||||
test('an unrelated member edit keeps the pause', async () => {
|
||||
await updateGuild(harness, owner.token, guild.id, {name: 'Renamed'});
|
||||
expect((await storedPause(guild.id)).until?.getTime()).toBe(untilMs);
|
||||
});
|
||||
|
||||
test('a staff edit of the pause features takes the pause over from the timer', async () => {
|
||||
const admin = await setUserACLs(harness, await createTestAccount(harness), [
|
||||
'admin:authenticate',
|
||||
'guild:update:features',
|
||||
]);
|
||||
await createBuilder(harness, admin.token)
|
||||
.patch(`/admin/guilds/${guild.id}`)
|
||||
.body({remove_features: [GuildFeatures.RAID_DETECTED]})
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
const after = await storedPause(guild.id);
|
||||
expect(after.until).toBeNull();
|
||||
expect(after.features.has(GuildFeatures.INVITES_DISABLED)).toBe(true);
|
||||
expect(after.features.has(GuildFeatures.RAID_DETECTED)).toBe(false);
|
||||
expect(publisher.of('guild_features_changed').at(-1)?.data).toEqual({
|
||||
guild_id: guild.id,
|
||||
actor_id: admin.userId,
|
||||
source: 'staff',
|
||||
added: [],
|
||||
removed: [GuildFeatures.RAID_DETECTED],
|
||||
invites_paused_until_ms: null,
|
||||
});
|
||||
});
|
||||
|
||||
test('a staff edit of other features leaves the timer running', async () => {
|
||||
const admin = await setUserACLs(harness, await createTestAccount(harness), [
|
||||
'admin:authenticate',
|
||||
'guild:update:features',
|
||||
]);
|
||||
await createBuilder(harness, admin.token)
|
||||
.patch(`/admin/guilds/${guild.id}`)
|
||||
.body({add_features: [GuildFeatures.VERIFIED]})
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
expect((await storedPause(guild.id)).until?.getTime()).toBe(untilMs);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -13,6 +13,7 @@ import type {ChannelService} from '@app/api/channel/services/ChannelService';
|
||||
import {THREAD_FEATURE_CHANNEL_TYPES, type ThreadViewer} from '@app/api/experiment/ChannelThreadsGate';
|
||||
import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService';
|
||||
import type {GuildService} from '@app/api/guild/services/GuildService';
|
||||
import {emitGuildJoinRefused, emitInviteCreated, emitInviteDeleted} from '@app/api/infrastructure/activity/GuildEvents';
|
||||
import type {InviteRepository} from '@app/api/invite/InviteRepository';
|
||||
import {Logger} from '@app/api/Logger';
|
||||
import type {LimitConfigService} from '@app/api/limits/LimitConfigService';
|
||||
@@ -243,6 +244,7 @@ export class InviteService {
|
||||
action: 'create',
|
||||
auditLogReason,
|
||||
});
|
||||
await emitInviteCreated(newInvite);
|
||||
}
|
||||
return {invite: newInvite, isNew: true};
|
||||
}
|
||||
@@ -291,13 +293,18 @@ export class InviteService {
|
||||
if ((guild.disabledOperations & GuildOperations.INSTANT_INVITES) !== 0) {
|
||||
throw new FeatureTemporarilyDisabledError();
|
||||
}
|
||||
if (guild.features.has(GuildFeatures.INVITES_DISABLED)) {
|
||||
throw new InvitesDisabledError();
|
||||
}
|
||||
const vanityCode = guild.vanityUrlCode ? vanityCodeToInviteCode(guild.vanityUrlCode) : null;
|
||||
const isVanityInvite = invite.code === vanityCode;
|
||||
const existingMember = await this.apiContext.services.gateway.hasGuildMember({
|
||||
guildId: invite.guildId,
|
||||
userId,
|
||||
});
|
||||
if (guild.features.has(GuildFeatures.INVITES_DISABLED)) {
|
||||
if (!existingMember) {
|
||||
await emitGuildJoinRefused({guildId: guild.id, userId, joinSource: isVanityInvite ? 'vanity' : 'invite'});
|
||||
}
|
||||
throw new InvitesDisabledError();
|
||||
}
|
||||
if (existingMember) {
|
||||
return invite;
|
||||
}
|
||||
@@ -308,8 +315,6 @@ export class InviteService {
|
||||
throw new TemporaryInviteRequiresPresenceError();
|
||||
}
|
||||
}
|
||||
const vanityCode = guild.vanityUrlCode ? vanityCodeToInviteCode(guild.vanityUrlCode) : null;
|
||||
const isVanityInvite = invite.code === vanityCode;
|
||||
const guildId = invite.guildId;
|
||||
const reservedInvite = await this.reserveInviteUse(invite);
|
||||
await this.withReservedInviteUse(reservedInvite, () =>
|
||||
@@ -459,6 +464,7 @@ export class InviteService {
|
||||
action: 'delete',
|
||||
auditLogReason,
|
||||
});
|
||||
await emitInviteDeleted(invite);
|
||||
}
|
||||
|
||||
async dispatchInviteCreate(
|
||||
|
||||
@@ -3,19 +3,20 @@
|
||||
import {Config} from '@app/api/Config';
|
||||
import {RequireClientIpMiddleware} from '@app/api/middleware/RequireClientIpMiddleware';
|
||||
import type {HonoEnv} from '@app/api/types/HonoEnv';
|
||||
import type {ClientIpResolution} from '@app/api/utils/RequestClientIp';
|
||||
import {type ClientIpResolution, PeerAddressMiddleware} from '@app/api/utils/RequestClientIp';
|
||||
import {AppErrorHandler} from '@fluxer/errors/src/domains/core/ErrorHandlers';
|
||||
import {Hono} from 'hono';
|
||||
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
interface Harness {
|
||||
request: (headers: Record<string, string>) => Promise<Response>;
|
||||
request: (headers: Record<string, string>, peerAddress?: string) => Promise<Response>;
|
||||
resolutions: Array<ClientIpResolution | undefined>;
|
||||
}
|
||||
|
||||
function createHarness(path = 'http://localhost/v1/messages'): Harness {
|
||||
const resolutions: Array<ClientIpResolution | undefined> = [];
|
||||
const app = new Hono<HonoEnv>();
|
||||
app.use(PeerAddressMiddleware);
|
||||
app.use(RequireClientIpMiddleware());
|
||||
app.get('/v1/messages', (ctx) => {
|
||||
resolutions.push(ctx.get('clientIpResolution'));
|
||||
@@ -25,7 +26,8 @@ function createHarness(path = 'http://localhost/v1/messages'): Harness {
|
||||
app.get('/internal/rpc', (ctx) => ctx.text('OK'));
|
||||
app.onError(AppErrorHandler);
|
||||
return {
|
||||
request: async (headers) => app.request(path, {headers}),
|
||||
request: async (headers, peerAddress) =>
|
||||
app.request(path, {headers}, peerAddress ? {incoming: {socket: {remoteAddress: peerAddress}}} : undefined),
|
||||
resolutions,
|
||||
};
|
||||
}
|
||||
@@ -53,7 +55,15 @@ describe('RequireClientIpMiddleware', () => {
|
||||
expect(await response.json()).toMatchObject({code: 'FORBIDDEN'});
|
||||
});
|
||||
|
||||
it('rejects a request when the header is not trusted with 403', async () => {
|
||||
it('passes an untrusted request on its peer address and ignores the header', async () => {
|
||||
Config.proxy.trust_client_ip_header = false;
|
||||
const harness = createHarness();
|
||||
const response = await harness.request({'x-forwarded-for': '203.0.113.10'}, '::ffff:198.51.100.20');
|
||||
expect(response.status).toBe(200);
|
||||
expect(harness.resolutions[0]?.ip).toBe('198.51.100.20');
|
||||
});
|
||||
|
||||
it('rejects an untrusted request with no peer address with 403', async () => {
|
||||
Config.proxy.trust_client_ip_header = false;
|
||||
const harness = createHarness();
|
||||
const response = await harness.request({'x-forwarded-for': '203.0.113.10'});
|
||||
@@ -61,6 +71,12 @@ describe('RequireClientIpMiddleware', () => {
|
||||
expect(await response.json()).toMatchObject({code: 'FORBIDDEN'});
|
||||
});
|
||||
|
||||
it('ignores the peer address while the header is trusted', async () => {
|
||||
const harness = createHarness();
|
||||
const response = await harness.request({}, '198.51.100.20');
|
||||
expect(response.status).toBe(403);
|
||||
});
|
||||
|
||||
it('rejects a request with no client ip header with 403 FORBIDDEN', async () => {
|
||||
const harness = createHarness();
|
||||
const response = await harness.request({});
|
||||
|
||||
@@ -49,6 +49,7 @@ export class Guild {
|
||||
readonly auditLogsIndexedAt: Date | null;
|
||||
readonly membersIndexedAt: Date | null;
|
||||
readonly messageHistoryCutoff: Date | null;
|
||||
readonly invitesPausedUntil: Date | null;
|
||||
readonly version: number;
|
||||
|
||||
constructor(row: GuildRow) {
|
||||
@@ -89,6 +90,7 @@ export class Guild {
|
||||
this.auditLogsIndexedAt = row.audit_logs_indexed_at ?? null;
|
||||
this.membersIndexedAt = row.members_indexed_at ?? null;
|
||||
this.messageHistoryCutoff = row.message_history_cutoff ?? null;
|
||||
this.invitesPausedUntil = row.invites_paused_until ?? null;
|
||||
this.version = row.version;
|
||||
}
|
||||
|
||||
@@ -128,6 +130,7 @@ export class Guild {
|
||||
audit_logs_indexed_at: this.auditLogsIndexedAt,
|
||||
members_indexed_at: this.membersIndexedAt,
|
||||
message_history_cutoff: this.messageHistoryCutoff,
|
||||
invites_paused_until: this.invitesPausedUntil,
|
||||
version: this.version,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -43,6 +43,7 @@ function guildRow(features: Set<string>): GuildRow {
|
||||
audit_logs_indexed_at: null,
|
||||
members_indexed_at: null,
|
||||
message_history_cutoff: null,
|
||||
invites_paused_until: null,
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -123,7 +123,7 @@ function observedOf(user: User): Observed {
|
||||
}
|
||||
|
||||
export function outcomeOf(
|
||||
env: Pick<ActionEnvelope, 'id'> & {type: string; user_id?: string},
|
||||
env: Pick<ActionEnvelope, 'id'> & {type: string; user_id?: string; guild_id?: string},
|
||||
status: OutcomeStatus,
|
||||
user: User | null = null,
|
||||
detail: string | null = null,
|
||||
@@ -136,6 +136,7 @@ export function outcomeOf(
|
||||
observed: user ? observedOf(user) : null,
|
||||
};
|
||||
if (env.user_id) outcome.user_id = env.user_id;
|
||||
if (env.guild_id) outcome.guild_id = env.guild_id;
|
||||
return outcome;
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,9 @@ import {
|
||||
requireClientIp,
|
||||
resolveClientIpHeaderName,
|
||||
} from '@fluxer/ip_utils/src/ClientIp';
|
||||
import type {HttpBindings} from '@hono/node-server';
|
||||
import type {Context} from 'hono';
|
||||
import {createMiddleware} from 'hono/factory';
|
||||
|
||||
export interface ClientIpResolution {
|
||||
trustClientIpHeader: boolean;
|
||||
@@ -22,6 +24,20 @@ interface ClientIpResolutionOptions {
|
||||
clientIpHeaderName: string;
|
||||
}
|
||||
|
||||
const peerAddresses = new WeakMap<Request, string>();
|
||||
|
||||
export const PeerAddressMiddleware = createMiddleware<HonoEnv>(async (ctx, next) => {
|
||||
const remoteAddress = (ctx.env as Partial<HttpBindings> | undefined)?.incoming?.socket?.remoteAddress;
|
||||
if (remoteAddress) {
|
||||
peerAddresses.set(ctx.req.raw, remoteAddress);
|
||||
}
|
||||
await next();
|
||||
});
|
||||
|
||||
function withPeerAddress(request: Request, options: ClientIpResolutionOptions) {
|
||||
return {...options, peerAddress: peerAddresses.get(request) ?? null};
|
||||
}
|
||||
|
||||
export function resolveClientIpWithOptions(ctx: Context<HonoEnv>, options: ClientIpResolutionOptions): string | null {
|
||||
const clientIpHeaderName = resolveClientIpHeaderName(options.clientIpHeaderName);
|
||||
const {trustClientIpHeader} = options;
|
||||
@@ -33,7 +49,7 @@ export function resolveClientIpWithOptions(ctx: Context<HonoEnv>, options: Clien
|
||||
) {
|
||||
return cached.ip;
|
||||
}
|
||||
const ip = extractClientIp(ctx.req.raw, {trustClientIpHeader, clientIpHeaderName});
|
||||
const ip = extractClientIp(ctx.req.raw, withPeerAddress(ctx.req.raw, {trustClientIpHeader, clientIpHeaderName}));
|
||||
ctx.set('clientIpResolution', {trustClientIpHeader, clientIpHeaderName, ip});
|
||||
return ip;
|
||||
}
|
||||
@@ -43,11 +59,11 @@ function proxyClientIpOptions(proxy: APIConfig['proxy']): ClientIpResolutionOpti
|
||||
}
|
||||
|
||||
export function extractConfiguredClientIp(request: Request): string | null {
|
||||
return extractClientIp(request, proxyClientIpOptions(Config.proxy));
|
||||
return extractClientIp(request, withPeerAddress(request, proxyClientIpOptions(Config.proxy)));
|
||||
}
|
||||
|
||||
export function requireConfiguredClientIp(request: Request, proxy: APIConfig['proxy']): string {
|
||||
return requireClientIp(request, proxyClientIpOptions(proxy));
|
||||
return requireClientIp(request, withPeerAddress(request, proxyClientIpOptions(proxy)));
|
||||
}
|
||||
|
||||
export function getRequestClientIp(ctx: Context<HonoEnv>): string | null {
|
||||
|
||||
@@ -47,6 +47,7 @@ function createGuild(overrides: Partial<GuildRow> = {}): Guild {
|
||||
audit_logs_indexed_at: null,
|
||||
members_indexed_at: null,
|
||||
message_history_cutoff: null,
|
||||
invites_paused_until: null,
|
||||
version: 1,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
@@ -1,11 +1,19 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {Config} from '@app/api/Config';
|
||||
import type {HonoEnv} from '@app/api/types/HonoEnv';
|
||||
import {getRequestClientIp, requireRequestClientIp, resolveClientIpWithOptions} from '@app/api/utils/RequestClientIp';
|
||||
import {
|
||||
extractConfiguredClientIp,
|
||||
getRequestClientIp,
|
||||
PeerAddressMiddleware,
|
||||
requireConfiguredClientIp,
|
||||
requireRequestClientIp,
|
||||
resolveClientIpWithOptions,
|
||||
} from '@app/api/utils/RequestClientIp';
|
||||
import {AppErrorHandler} from '@fluxer/errors/src/domains/core/ErrorHandlers';
|
||||
import {MissingClientIpError} from '@fluxer/ip_utils/src/ClientIp';
|
||||
import {Hono} from 'hono';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {afterEach, describe, expect, it} from 'vitest';
|
||||
|
||||
interface CountingRequest {
|
||||
request: Request;
|
||||
@@ -28,6 +36,12 @@ function createCountingRequest(headers: Record<string, string>): CountingRequest
|
||||
}
|
||||
|
||||
describe('RequestClientIp', () => {
|
||||
const previousTrustClientIpHeader = Config.proxy.trust_client_ip_header;
|
||||
|
||||
afterEach(() => {
|
||||
Config.proxy.trust_client_ip_header = previousTrustClientIpHeader;
|
||||
});
|
||||
|
||||
it('normalizes the resolved address the same way for every consumer', async () => {
|
||||
const seen: Array<string | null> = [];
|
||||
const app = new Hono<HonoEnv>();
|
||||
@@ -102,4 +116,22 @@ describe('RequestClientIp', () => {
|
||||
expect(response.status).toBe(403);
|
||||
expect(await response.json()).toMatchObject({code: 'FORBIDDEN'});
|
||||
});
|
||||
it('resolves the peer address for request only callers when the header is untrusted', async () => {
|
||||
Config.proxy.trust_client_ip_header = false;
|
||||
const seen: Array<string | null> = [];
|
||||
const app = new Hono<HonoEnv>();
|
||||
app.use(PeerAddressMiddleware);
|
||||
app.get('/v1/messages', (ctx) => {
|
||||
seen.push(getRequestClientIp(ctx));
|
||||
seen.push(extractConfiguredClientIp(ctx.req.raw));
|
||||
seen.push(requireConfiguredClientIp(ctx.req.raw, Config.proxy));
|
||||
return ctx.text('ok');
|
||||
});
|
||||
await app.request(
|
||||
'http://localhost/v1/messages',
|
||||
{headers: {'x-forwarded-for': '203.0.113.10'}},
|
||||
{incoming: {socket: {remoteAddress: '::ffff:198.51.100.20'}}},
|
||||
);
|
||||
expect(seen).toEqual(['198.51.100.20', '198.51.100.20', '198.51.100.20']);
|
||||
});
|
||||
});
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
setupFanoutWorld,
|
||||
} from '@app/api/channel/tests/CrosspostWorkerTestUtils';
|
||||
import {getPngDataUrl} from '@app/api/emoji/tests/EmojiTestUtils';
|
||||
import {processRecordedGuildDeletions, recordGuildDeletions} from '@app/api/guild/tests/GuildDeletionTestUtils';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {NoopGatewayService} from '@app/api/test/NoopGatewayService';
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
@@ -295,11 +296,13 @@ describe('Follower webhook management', () => {
|
||||
await follow(harness, b.owner.token, a.ann.id, b.t1.id);
|
||||
await follow(harness, b.owner.token, a.ann.id, b.t2.id);
|
||||
expect(await sourceIndex(a.ann.id)).toHaveLength(2);
|
||||
const worker = recordGuildDeletions();
|
||||
await createBuilder(harness, b.owner.token)
|
||||
.post(`/guilds/${b.guild.id}/delete`)
|
||||
.body({password: b.owner.password})
|
||||
.expect(HTTP_STATUS.NO_CONTENT)
|
||||
.execute();
|
||||
await processRecordedGuildDeletions(worker);
|
||||
expect(await sourceIndex(a.ann.id)).toEqual([]);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {applyPauseGuildInvites, type GuildInvitePauseDeps} from '@app/api/guild/services/GuildInvitePauseApplier';
|
||||
import {emitActivity} from '@app/api/infrastructure/activity/ActivityEvents';
|
||||
import {workerMeta} from '@app/api/infrastructure/activity/ActivityMeta';
|
||||
import {
|
||||
@@ -27,6 +28,7 @@ import {AckPolicy, type ConsumerMessages, type JetStreamClient, type JsMsg} from
|
||||
export interface AccountActionDeps {
|
||||
js: Pick<JetStreamClient, 'consumers'>;
|
||||
state: AccountStateDeps;
|
||||
guildInvites: GuildInvitePauseDeps;
|
||||
publishOutcome?: (key: string, outcome: ActionOutcome) => Promise<void>;
|
||||
now?: () => number;
|
||||
retryDelayMs?: number;
|
||||
@@ -79,6 +81,8 @@ export function applyAction(deps: AccountActionDeps, env: ActionEnvelope): Promi
|
||||
return applyHideRecentMessages(deps.state, env);
|
||||
case 'delete_user_messages':
|
||||
return applyDeleteUserMessages(deps.state, env);
|
||||
case 'pause_guild_invites':
|
||||
return applyPauseGuildInvites(deps.guildInvites, env);
|
||||
default:
|
||||
return Promise.resolve(outcomeOf(env as ActionEnvelope, 'unsupported'));
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import {DonationRepository} from '@app/api/donation/DonationRepository';
|
||||
import type {FavoriteMemeRepository} from '@app/api/favorite_meme/FavoriteMemeRepository';
|
||||
import type {GuildAuditLogService} from '@app/api/guild/GuildAuditLogService';
|
||||
import type {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
|
||||
import type {GuildInvitePauseDeps} from '@app/api/guild/services/GuildInvitePauseApplier';
|
||||
import type {GuildService} from '@app/api/guild/services/GuildService';
|
||||
import type {AvatarService} from '@app/api/infrastructure/AvatarService';
|
||||
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
|
||||
@@ -281,3 +282,13 @@ export async function initializeWorkerDependencies(snowflakeService: ISnowflakeS
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function guildInvitePauseDeps(deps: WorkerDependencies): GuildInvitePauseDeps {
|
||||
return {
|
||||
guilds: deps.guildRepository,
|
||||
locks: deps.cacheService,
|
||||
schedule: deps.kvClient,
|
||||
gateway: deps.gatewayService,
|
||||
auditLog: deps.guildAuditLogService,
|
||||
};
|
||||
}
|
||||
@@ -39,6 +39,7 @@ const LANE_CONFIG = {
|
||||
'userProcessPendingDeletion',
|
||||
'userProcessPendingDeletions',
|
||||
'applicationProcessDeletion',
|
||||
'guildProcessDeletion',
|
||||
'bulkDeleteSelfMessagesImmediate',
|
||||
'bulkDeleteUserMessages',
|
||||
'deleteUserMessagesInGuildByTime',
|
||||
@@ -58,6 +59,7 @@ const LANE_CONFIG = {
|
||||
'bulkBanFileShas',
|
||||
'bulkDeleteMessagesForUsers',
|
||||
'drainActivitySpool',
|
||||
'liftExpiredGuildInvitePauses',
|
||||
] as const,
|
||||
retiredTasks: ['sendScheduledMessage', 'finalizeNcmecAttachmentReport'],
|
||||
concurrency: 8,
|
||||
|
||||
@@ -40,7 +40,11 @@ import {queueBlocklistFeedStartupJobs} from '@app/api/worker/BlocklistFeedStartu
|
||||
import {CronScheduler} from '@app/api/worker/CronScheduler';
|
||||
import {JetStreamWorkerQueue, JOBS_STREAM_MAX_AGE_MS} from '@app/api/worker/JetStreamWorkerQueue';
|
||||
import {clearWorkerDependencies, setWorkerDependencies} from '@app/api/worker/WorkerContext';
|
||||
import {initializeWorkerDependencies, type WorkerDependencies} from '@app/api/worker/WorkerDependencies';
|
||||
import {
|
||||
guildInvitePauseDeps,
|
||||
initializeWorkerDependencies,
|
||||
type WorkerDependencies,
|
||||
} from '@app/api/worker/WorkerDependencies';
|
||||
import {WorkerHeartbeat} from '@app/api/worker/WorkerHeartbeat';
|
||||
import {
|
||||
resolveCronSchedulerEnabled,
|
||||
@@ -108,6 +112,7 @@ function registerCronJobs(cron: CronScheduler, jobsStreamMaxAgeMs: number): void
|
||||
}
|
||||
cron.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/10 * * * * *', {ledger: false});
|
||||
cron.upsert('drainActivitySpool', 'drainActivitySpool', {}, '*/5 * * * * *', {ledger: false});
|
||||
cron.upsert('liftExpiredGuildInvitePauses', 'liftExpiredGuildInvitePauses', {}, '0 * * * * *', {ledger: false});
|
||||
cron.upsert('archiveInactiveThreads', 'archiveInactiveThreads', {}, '0 * * * * *', {
|
||||
ledger: false,
|
||||
enabled: () => true,
|
||||
@@ -328,6 +333,7 @@ export async function startWorkerMain(): Promise<void> {
|
||||
},
|
||||
dependencies.channelRepository,
|
||||
),
|
||||
guildInvites: guildInvitePauseDeps(dependencies),
|
||||
});
|
||||
Logger.info('Account action consumer started');
|
||||
}
|
||||
|
||||
@@ -20,12 +20,14 @@ import expireReportSnapshots from '@app/api/worker/tasks/ExpireReportSnapshots';
|
||||
import expireStaleJobs from '@app/api/worker/tasks/ExpireStaleJobs';
|
||||
import extractEmbeds from '@app/api/worker/tasks/ExtractEmbeds';
|
||||
import flushUserActivityBuffer from '@app/api/worker/tasks/FlushUserActivityBuffer';
|
||||
import guildProcessDeletion from '@app/api/worker/tasks/GuildProcessDeletion';
|
||||
import handleMentionChunk from '@app/api/worker/tasks/HandleMentionChunk';
|
||||
import handleMentions from '@app/api/worker/tasks/HandleMentions';
|
||||
import harvestGuildData from '@app/api/worker/tasks/HarvestGuildData';
|
||||
import harvestUserData from '@app/api/worker/tasks/HarvestUserData';
|
||||
import indexChannelMessages from '@app/api/worker/tasks/IndexChannelMessages';
|
||||
import indexGuildMembers from '@app/api/worker/tasks/IndexGuildMembers';
|
||||
import liftExpiredGuildInvitePauses from '@app/api/worker/tasks/LiftExpiredGuildInvitePauses';
|
||||
import messageShred from '@app/api/worker/tasks/MessageShred';
|
||||
import pollAppStoreNotificationHistory from '@app/api/worker/tasks/PollAppStoreNotificationHistory';
|
||||
import pollGooglePlayVoidedPurchases from '@app/api/worker/tasks/PollGooglePlayVoidedPurchases';
|
||||
@@ -91,12 +93,14 @@ export const workerTasks: Record<WorkerTaskName, WorkerTaskHandler> = {
|
||||
expireReportSnapshots,
|
||||
expireStaleJobs,
|
||||
extractEmbeds,
|
||||
guildProcessDeletion,
|
||||
handleMentions,
|
||||
handleMentionChunk,
|
||||
harvestGuildData,
|
||||
harvestUserData,
|
||||
indexChannelMessages,
|
||||
indexGuildMembers,
|
||||
liftExpiredGuildInvitePauses,
|
||||
messageShred,
|
||||
processAssetDeletionQueue,
|
||||
processCachePurgeQueue,
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createGuildID} from '@app/api/BrandedTypes';
|
||||
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
|
||||
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
import {z} from 'zod';
|
||||
|
||||
const PayloadSchema = z.object({
|
||||
guildId: z.string(),
|
||||
copyMode: z.enum(['source_deleted', 'purge']),
|
||||
});
|
||||
|
||||
const guildProcessDeletion: WorkerTaskHandler = async (payload, helpers) => {
|
||||
const validated = PayloadSchema.parse(payload);
|
||||
helpers.logger.debug({payload: validated}, 'Processing guildProcessDeletion task');
|
||||
const {guildService} = getWorkerDependencies();
|
||||
await guildService.data.operations.purgeGuild(createGuildID(BigInt(validated.guildId)), validated.copyMode);
|
||||
};
|
||||
|
||||
export default guildProcessDeletion;
|
||||
@@ -0,0 +1,15 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {liftExpiredGuildInvitePauses} from '@app/api/guild/services/GuildInvitePauseApplier';
|
||||
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
|
||||
import {guildInvitePauseDeps} from '@app/api/worker/WorkerDependencies';
|
||||
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
|
||||
const BATCH_SIZE = 100;
|
||||
|
||||
const liftExpiredGuildInvitePausesTask: WorkerTaskHandler = async (_payload, helpers) => {
|
||||
const lifted = await liftExpiredGuildInvitePauses(guildInvitePauseDeps(getWorkerDependencies()), BATCH_SIZE);
|
||||
if (lifted > 0) helpers.logger.info({lifted}, 'Lifted expired guild invite pauses');
|
||||
};
|
||||
|
||||
export default liftExpiredGuildInvitePausesTask;
|
||||
@@ -208,7 +208,12 @@ function harness(): Harness {
|
||||
authored: authored as unknown as AccountStateDeps['authored'],
|
||||
now: () => NOW,
|
||||
};
|
||||
h.deps = {js: {} as AccountActionDeps['js'], state, now: () => NOW};
|
||||
h.deps = {
|
||||
js: {} as AccountActionDeps['js'],
|
||||
state,
|
||||
guildInvites: {} as AccountActionDeps['guildInvites'],
|
||||
now: () => NOW,
|
||||
};
|
||||
return h;
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
<meta name="msapplication-TileColor" content="#4641D9">
|
||||
<meta name="format-detection" content="telephone=no">
|
||||
<meta name="display" content="standalone">
|
||||
<link rel="license" href="/assets/fonts-NOTICE.txt">
|
||||
<link rel="expect" href="#fluxer-boot-end" blocking="render">
|
||||
<script nonce="{{CSP_NONCE_PLACEHOLDER}}">(function(){try{var loc=window.location;if(loc.pathname==='/'){var target='/channels/@me';if(loc.search)target+=loc.search;if(loc.hash)target+=loc.hash;loc.replace(target);}}catch(e){}})();</script>
|
||||
<script nonce="{{CSP_NONCE_PLACEHOLDER}}">(function(){try{var t=localStorage.getItem('theme');if(t){document.documentElement.classList.add('theme-'+t)}}catch{}})()</script>
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createHash} from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import {sources} from '@rspack/core';
|
||||
import {HtmlRspackPlugin, sources} from '@rspack/core';
|
||||
|
||||
const STATIC_CDN_ENDPOINT_PLACEHOLDER = '{{STATIC_CDN_ENDPOINT}}';
|
||||
|
||||
@@ -104,22 +105,41 @@ export class StaticFilesPlugin {
|
||||
}
|
||||
|
||||
emitLicenseFiles(compilation, dir, files, subject) {
|
||||
const assetNames = new Map();
|
||||
if (!dir) {
|
||||
return;
|
||||
return assetNames;
|
||||
}
|
||||
for (const {source, asset} of files) {
|
||||
const sourcePath = path.join(dir, source);
|
||||
compilation.fileDependencies.add(sourcePath);
|
||||
if (!fs.existsSync(sourcePath)) {
|
||||
throw new Error(
|
||||
`StaticFilesPlugin: ${sourcePath} is missing. The bundled ${subject} may not be redistributed without it.`,
|
||||
);
|
||||
}
|
||||
compilation.emitAsset(asset, new sources.RawSource(fs.readFileSync(sourcePath)));
|
||||
const content = fs.readFileSync(sourcePath);
|
||||
const extension = path.extname(asset);
|
||||
const hash = createHash('sha256').update(content).digest('hex').slice(0, 16);
|
||||
const name = `${asset.slice(0, -extension.length)}.${hash}${extension}`;
|
||||
compilation.emitAsset(name, new sources.RawSource(content));
|
||||
assetNames.set(source, name);
|
||||
}
|
||||
return assetNames;
|
||||
}
|
||||
|
||||
apply(compiler) {
|
||||
compiler.hooks.thisCompilation.tap('StaticFilesPlugin', (compilation) => {
|
||||
let fontNoticeAsset;
|
||||
HtmlRspackPlugin.getCompilationHooks(compilation).alterAssetTagGroups.tap('StaticFilesPlugin', (data) => {
|
||||
if (fontNoticeAsset) {
|
||||
data.headTags.push({
|
||||
tagName: 'link',
|
||||
voidTag: true,
|
||||
attributes: {rel: 'license', href: `${data.publicPath}${fontNoticeAsset}`},
|
||||
});
|
||||
}
|
||||
return data;
|
||||
});
|
||||
compilation.hooks.processAssets.tap(
|
||||
{
|
||||
name: 'StaticFilesPlugin',
|
||||
@@ -132,7 +152,9 @@ export class StaticFilesPlugin {
|
||||
new sources.RawSource(generateBrowserConfig(this.staticCdnEndpoint)),
|
||||
);
|
||||
compilation.emitAsset('robots.txt', new sources.RawSource(generateRobotsTxt()));
|
||||
this.emitLicenseFiles(compilation, this.fontsDir, FONT_LICENSE_FILES, 'fonts');
|
||||
fontNoticeAsset = this.emitLicenseFiles(compilation, this.fontsDir, FONT_LICENSE_FILES, 'fonts').get(
|
||||
'NOTICE.md',
|
||||
);
|
||||
this.emitLicenseFiles(compilation, this.wasmCratesDir, WASM_LICENSE_FILES, 'WebAssembly modules');
|
||||
},
|
||||
);
|
||||
|
||||
@@ -9,7 +9,7 @@ import {type InstanceHTTPTarget, instanceRequest} from '@app/features/platform/t
|
||||
import {HttpError} from '@app/features/platform/types/EndpointError';
|
||||
import {Logger} from '@app/features/platform/utils/AppLogger';
|
||||
import {failureCode, ipAuthorizationRequiredResponseFromError} from '@app/features/platform/utils/ResponseInspection';
|
||||
import UserSettings from '@app/features/user/state/UserSettings';
|
||||
import {getCurrentOrDetectedLocale} from '@app/features/user/utils/LocaleUtils';
|
||||
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
|
||||
import type {ValueOf} from '@fluxer/constants/src/ValueOf';
|
||||
import type {
|
||||
@@ -27,7 +27,7 @@ import type {AuthenticationResponseJSON, PublicKeyCredentialRequestOptionsJSON}
|
||||
|
||||
const logger = new Logger('AuthService');
|
||||
const withAuthLocaleHeader = (headers?: Record<string, string>): Record<string, string> => ({
|
||||
'Accept-Language': UserSettings.locale,
|
||||
'Accept-Language': getCurrentOrDetectedLocale(),
|
||||
...(headers ?? {}),
|
||||
});
|
||||
|
||||
|
||||
@@ -65,7 +65,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 500,
|
||||
url: 'https://media.internal/avatars/1.webp',
|
||||
@@ -104,7 +103,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 500,
|
||||
url: 'https://media.internal/avatars/1.webp',
|
||||
@@ -161,7 +159,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 500,
|
||||
url: 'https://media.mixed.example/avatars/1.webp',
|
||||
@@ -193,44 +190,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
outboundHTTP.cleanup();
|
||||
});
|
||||
|
||||
test('a request that cannot be constructed releases its in-flight slot', async () => {
|
||||
const {port, server} = await startServer({
|
||||
'/ok': (_request, response) => sendImage(response, 'image/png', PNG_BYTES),
|
||||
});
|
||||
try {
|
||||
const outboundHTTP = new DesktopOutboundHTTP({
|
||||
resolveHostAddresses: stubResolver({'lan.example': ['127.0.0.1']}),
|
||||
});
|
||||
await outboundHTTP.registerAnchoredOrigins({
|
||||
anchorOrigin: `http://lan.example:${port}`,
|
||||
origins: [`http://lan.example:${port}`],
|
||||
});
|
||||
const send = (headers) =>
|
||||
outboundHTTP.request({
|
||||
body: null,
|
||||
expectedOrigin: `http://lan.example:${port}`,
|
||||
headers,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: 'slot-leak',
|
||||
signal: null,
|
||||
timeoutMs: 2000,
|
||||
url: `http://lan.example:${port}/ok`,
|
||||
});
|
||||
|
||||
for (let attempt = 0; attempt < 80; attempt += 1) {
|
||||
await assert.rejects(send({'x-bad': 'a\u0001b'}));
|
||||
}
|
||||
const message = await send(null);
|
||||
|
||||
assert.equal(message.status, 200);
|
||||
message.message.resume();
|
||||
outboundHTTP.cleanup();
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('a request whose signal already aborted is refused instead of sent', async () => {
|
||||
const seen = [];
|
||||
const {port, server} = await startServer({
|
||||
@@ -257,7 +216,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: 'aborted',
|
||||
signal: controller.signal,
|
||||
timeoutMs: 2000,
|
||||
url: `http://lan.example:${port}/ok`,
|
||||
@@ -289,7 +247,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 2000,
|
||||
url: `http://media.lan.example:${port}/media/ok`,
|
||||
@@ -331,7 +288,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: {'content-type': 'application/json', 'x-fluxer-sudo-mode-jwt': 'previous'},
|
||||
method,
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 2000,
|
||||
url: `http://lan.example:${port}/api/v1/oauth2/applications/1`,
|
||||
@@ -549,7 +505,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.BOUND,
|
||||
serviceName: 'test',
|
||||
signal,
|
||||
timeoutMs: 2000,
|
||||
url: `https://home.example:${port}/ping`,
|
||||
@@ -577,7 +532,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.BOUND,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 200,
|
||||
url: 'http://blackhole.example:9/ping',
|
||||
@@ -606,7 +560,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.BOUND,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 2000,
|
||||
url: `http://mixed.example:${port}/ping`,
|
||||
@@ -632,7 +585,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.BOUND,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 2000,
|
||||
url: `http://127.0.0.1:${other.port}/steal`,
|
||||
@@ -670,7 +622,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.BOUND,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 5000,
|
||||
url: `http://127.0.0.1:${port}/ping`,
|
||||
@@ -720,7 +671,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: {'content-type': 'text/plain'},
|
||||
method: 'PUT',
|
||||
originTrust: DesktopOriginTrust.BOUND,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 5000,
|
||||
url: `http://127.0.0.1:${port}/upload`,
|
||||
@@ -775,7 +725,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 2000,
|
||||
url: 'https://unrouted.example/',
|
||||
@@ -826,7 +775,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 2000,
|
||||
url: `http://dual.example:${port}/ping`,
|
||||
@@ -857,7 +805,6 @@ describe('DesktopOutboundHTTP origin anchoring', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 3000,
|
||||
url: `http://multi.example:${port}/ping`,
|
||||
@@ -1007,7 +954,6 @@ describe('DesktopOutboundHTTP system proxy routes', () => {
|
||||
headers: {Accept: 'image/png'},
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 500,
|
||||
url: 'https://media.blocked.example/avatar.png',
|
||||
@@ -1061,7 +1007,6 @@ describe('DesktopOutboundHTTP system proxy routes', () => {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 500,
|
||||
url,
|
||||
|
||||
@@ -28,8 +28,6 @@ const DESKTOP_OUTBOUND_HTTP_RESOLUTION_TIMEOUT_MS = 10_000;
|
||||
const DESKTOP_OUTBOUND_HTTP_MAX_TARGET_URL_BYTES = 16 * 1024;
|
||||
const DESKTOP_OUTBOUND_HTTP_MAX_REDIRECT_LOCATION_BYTES = 4096;
|
||||
const DESKTOP_OUTBOUND_HTTP_MAX_ORIGINS = 2048;
|
||||
const DESKTOP_OUTBOUND_HTTP_MAX_ORIGIN_REQUESTS_IN_FLIGHT = 128;
|
||||
const DESKTOP_OUTBOUND_HTTP_MAX_IN_FLIGHT_PER_SERVICE = 64;
|
||||
const DESKTOP_OUTBOUND_HTTP_MAX_PENDING_RESOLUTIONS = 64;
|
||||
const DESKTOP_OUTBOUND_HTTP_MAX_REQUEST_BODY_BYTES = 1024 * 1024;
|
||||
const DESKTOP_OUTBOUND_HTTP_STALE_SOCKET_ATTEMPTS = 1;
|
||||
@@ -113,7 +111,6 @@ export interface DesktopOutboundHTTPRequest {
|
||||
readonly maximumRequestBodyBytes?: number;
|
||||
readonly method: string;
|
||||
readonly originTrust: DesktopOriginTrust;
|
||||
readonly serviceName: string;
|
||||
readonly signal: AbortSignal | null;
|
||||
readonly timeoutMs: number;
|
||||
readonly url: string;
|
||||
@@ -565,7 +562,7 @@ interface DesktopProxiedSend {
|
||||
readonly body: Uint8Array | Readable | null;
|
||||
readonly headers: Readonly<Record<string, string>>;
|
||||
readonly method: string;
|
||||
readonly release: () => void;
|
||||
readonly release?: () => void;
|
||||
readonly signal: AbortSignal | null;
|
||||
readonly target: URL;
|
||||
readonly timeout: () => Error;
|
||||
@@ -578,20 +575,15 @@ export class DesktopOutboundHTTP {
|
||||
maxSockets: DESKTOP_OUTBOUND_HTTP_MAX_SOCKETS,
|
||||
});
|
||||
private readonly httpsAgent: https.Agent;
|
||||
private readonly originRequestHttpAgent = new HappyEyeballsHttpAgent({
|
||||
keepAlive: true,
|
||||
maxSockets: DESKTOP_OUTBOUND_HTTP_MAX_IN_FLIGHT_PER_SERVICE,
|
||||
});
|
||||
private readonly originRequestHttpAgent = new HappyEyeballsHttpAgent({keepAlive: true});
|
||||
private readonly originRequestHttpsAgent: https.Agent;
|
||||
private readonly bindings = new Map<string, DesktopOriginAddressBinding>();
|
||||
private readonly pendingBindings = new Map<string, DesktopPendingOriginBinding>();
|
||||
private readonly registeredRequirements = new Map<string, DesktopAddressRequirement>();
|
||||
private readonly originRequestsInFlightByService = new Map<string, number>();
|
||||
private readonly resolveHostAddresses: DesktopHostAddressResolver;
|
||||
private readonly resolveProxy: DesktopProxyResolver;
|
||||
private readonly sendThroughSession: DesktopSessionHTTPSender;
|
||||
private inFlight = 0;
|
||||
private originRequestsInFlight = 0;
|
||||
private activeResolutions = 0;
|
||||
private acceptingRequests = true;
|
||||
|
||||
@@ -608,7 +600,6 @@ export class DesktopOutboundHTTP {
|
||||
});
|
||||
this.originRequestHttpsAgent = new HappyEyeballsHttpsAgent({
|
||||
keepAlive: true,
|
||||
maxSockets: DESKTOP_OUTBOUND_HTTP_MAX_IN_FLIGHT_PER_SERVICE,
|
||||
...trust,
|
||||
});
|
||||
}
|
||||
@@ -783,7 +774,6 @@ export class DesktopOutboundHTTP {
|
||||
this.originRequestHttpsAgent.destroy();
|
||||
this.bindings.clear();
|
||||
this.registeredRequirements.clear();
|
||||
this.originRequestsInFlightByService.clear();
|
||||
}
|
||||
|
||||
private async routesDirect(url: URL): Promise<boolean> {
|
||||
@@ -813,15 +803,6 @@ export class DesktopOutboundHTTP {
|
||||
request: DesktopOutboundHTTPRequest,
|
||||
target: URL,
|
||||
): Promise<DesktopOutboundHTTPMessage> {
|
||||
this.acquireSlot(request.serviceName);
|
||||
let released = false;
|
||||
const release = (): void => {
|
||||
if (released) {
|
||||
return;
|
||||
}
|
||||
released = true;
|
||||
this.releaseSlot(request.serviceName);
|
||||
};
|
||||
return this.sendProxied({
|
||||
body: this.proxiedRequestBody(request),
|
||||
headers: requestHeaders(request.headers),
|
||||
@@ -830,7 +811,6 @@ export class DesktopOutboundHTTP {
|
||||
target,
|
||||
timeout: () => new DesktopOutboundHTTPRequestTimeoutError(target.toString(), request.timeoutMs),
|
||||
timeoutMs: request.timeoutMs,
|
||||
release,
|
||||
}).catch((error: unknown) => {
|
||||
if (request.signal?.aborted === true) {
|
||||
throw new DesktopOutboundHTTPRequestAbortedError(target.toString());
|
||||
@@ -854,7 +834,7 @@ export class DesktopOutboundHTTP {
|
||||
timer.unref();
|
||||
const settle = (): void => {
|
||||
clearTimeout(timer);
|
||||
release();
|
||||
release?.();
|
||||
};
|
||||
try {
|
||||
const response = await this.sendThroughSession({
|
||||
@@ -1083,42 +1063,11 @@ export class DesktopOutboundHTTP {
|
||||
}
|
||||
}
|
||||
|
||||
private acquireSlot(serviceName: string): void {
|
||||
if (this.originRequestsInFlight >= DESKTOP_OUTBOUND_HTTP_MAX_ORIGIN_REQUESTS_IN_FLIGHT) {
|
||||
throw new DesktopOutboundHTTPCapacityError('in-flight request');
|
||||
}
|
||||
const perService = this.originRequestsInFlightByService.get(serviceName) ?? 0;
|
||||
if (perService >= DESKTOP_OUTBOUND_HTTP_MAX_IN_FLIGHT_PER_SERVICE) {
|
||||
throw new DesktopOutboundHTTPCapacityError(`in-flight request for ${serviceName}`);
|
||||
}
|
||||
this.originRequestsInFlight += 1;
|
||||
this.originRequestsInFlightByService.set(serviceName, perService + 1);
|
||||
}
|
||||
|
||||
private releaseSlot(serviceName: string): void {
|
||||
this.originRequestsInFlight -= 1;
|
||||
const perService = (this.originRequestsInFlightByService.get(serviceName) ?? 1) - 1;
|
||||
if (perService <= 0) {
|
||||
this.originRequestsInFlightByService.delete(serviceName);
|
||||
} else {
|
||||
this.originRequestsInFlightByService.set(serviceName, perService);
|
||||
}
|
||||
}
|
||||
|
||||
private issueOriginRequest(
|
||||
request: DesktopOutboundHTTPRequest,
|
||||
target: URL,
|
||||
binding: DesktopOriginAddressBinding,
|
||||
): Promise<DesktopOutboundHTTPMessage> {
|
||||
this.acquireSlot(request.serviceName);
|
||||
let released = false;
|
||||
const release = (): void => {
|
||||
if (released) {
|
||||
return;
|
||||
}
|
||||
released = true;
|
||||
this.releaseSlot(request.serviceName);
|
||||
};
|
||||
return new Promise<DesktopOutboundHTTPMessage>((resolve, reject) => {
|
||||
let attemptsRemaining = isReplayableRequest(request) ? DESKTOP_OUTBOUND_HTTP_STALE_SOCKET_ATTEMPTS : 0;
|
||||
let active: http.ClientRequest | null = null;
|
||||
@@ -1144,7 +1093,6 @@ export class DesktopOutboundHTTP {
|
||||
const settle = (): void => {
|
||||
clearTimeout(timeout);
|
||||
request.signal?.removeEventListener('abort', onAbort);
|
||||
release();
|
||||
};
|
||||
const destroyActive = (error: Error): void => {
|
||||
active?.destroy(error);
|
||||
|
||||
@@ -16,10 +16,12 @@ function createLogger() {
|
||||
|
||||
function createSession() {
|
||||
const session = {
|
||||
requestFilter: null,
|
||||
requestListener: null,
|
||||
webRequest: {
|
||||
onBeforeSendHeaders: (listener) => {
|
||||
session.requestListener = listener;
|
||||
onBeforeSendHeaders: (...args) => {
|
||||
session.requestFilter = args.length > 1 ? args[0] : null;
|
||||
session.requestListener = args.at(-1);
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -141,6 +143,12 @@ describe('DesktopRuntimeSecurity', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('only intercepts local app requests and the YouTube embed frame', () => {
|
||||
const {session} = createInstalled();
|
||||
|
||||
assert.deepEqual(session.requestFilter, {urls: ['fluxer-app://*/*', 'https://www.youtube.com/embed/*']});
|
||||
});
|
||||
|
||||
test('installs once per session and stops listening after cleanup', () => {
|
||||
const {security, session} = createInstalled();
|
||||
const firstRequestListener = session.requestListener;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {STABLE_APP_URL} from '@electron/common/Constants';
|
||||
import {DESKTOP_APP_SCHEME, STABLE_APP_URL} from '@electron/common/Constants';
|
||||
import {isPlainRecord} from '@electron/common/PlainRecord';
|
||||
|
||||
const SUB_FRAME_RESOURCE_TYPE = 'subFrame';
|
||||
@@ -9,6 +9,10 @@ const REFERER_REQUEST_HEADER = 'Referer';
|
||||
const REFERER_HEADER_NAME_LOWERCASE = 'referer';
|
||||
const YOUTUBE_EMBED_FRAME_ORIGIN = 'https://www.youtube.com';
|
||||
const YOUTUBE_EMBED_FRAME_PATH_PREFIX = '/embed/';
|
||||
const DESKTOP_RUNTIME_SECURITY_REQUEST_URL_PATTERNS = [
|
||||
`${DESKTOP_APP_SCHEME}://*/*`,
|
||||
`${YOUTUBE_EMBED_FRAME_ORIGIN}${YOUTUBE_EMBED_FRAME_PATH_PREFIX}*`,
|
||||
];
|
||||
|
||||
interface DesktopRuntimeSecurityLogger {
|
||||
error: (...args: Array<unknown>) => void;
|
||||
@@ -62,9 +66,12 @@ export class DesktopRuntimeSecurity {
|
||||
}
|
||||
this.installedSessions.add(session);
|
||||
try {
|
||||
session.webRequest.onBeforeSendHeaders((details: unknown, callback: unknown) => {
|
||||
this.settleRequestHeaders(details, callback);
|
||||
});
|
||||
session.webRequest.onBeforeSendHeaders(
|
||||
{urls: DESKTOP_RUNTIME_SECURITY_REQUEST_URL_PATTERNS},
|
||||
(details: unknown, callback: unknown) => {
|
||||
this.settleRequestHeaders(details, callback);
|
||||
},
|
||||
);
|
||||
} catch (installationError) {
|
||||
this.uninstall(session);
|
||||
throw installationError;
|
||||
|
||||
@@ -41,7 +41,6 @@ async function fetchThrough(certificates, port) {
|
||||
headers: null,
|
||||
method: 'GET',
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: 'test',
|
||||
signal: null,
|
||||
timeoutMs: 3000,
|
||||
url: `https://ca.example:${port}/`,
|
||||
|
||||
@@ -20,8 +20,6 @@ const LOCAL_APP_PROXY_REQUEST_BODY_MAX_BYTES = 100 * 1024 * 1024;
|
||||
export const LOCAL_APP_PROXY_ACCEPT_ENCODING = 'gzip, br';
|
||||
|
||||
const LOCAL_APP_PROXY_RESOURCE_RESPONSE_HEADERS_DEADLINE_MS = 30_000;
|
||||
const LOCAL_APP_PROXY_MAX_CONCURRENT_REQUESTS = 32;
|
||||
const LOCAL_APP_PROXY_MAX_QUEUED_REQUESTS = 4096;
|
||||
const UPLOAD_PROGRESS_MIN_BYTES = 64 * 1024;
|
||||
const UPLOAD_PROGRESS_MIN_INTERVAL_MS = 100;
|
||||
const MINIMUM_RESPONSE_STATUS = 200;
|
||||
@@ -42,101 +40,6 @@ class LocalAppProxyResponseHeadersDeadlineError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
class LocalAppProxyQueueOverflowError extends Error {
|
||||
public constructor() {
|
||||
super('Local app proxy has too many queued requests');
|
||||
this.name = 'LocalAppProxyQueueOverflowError';
|
||||
}
|
||||
}
|
||||
|
||||
class LocalAppProxyConcurrencyPermit {
|
||||
private releasePermit: (() => void) | null;
|
||||
|
||||
public constructor(releasePermit: () => void) {
|
||||
this.releasePermit = releasePermit;
|
||||
}
|
||||
|
||||
public release(): void {
|
||||
const releasePermit = this.releasePermit;
|
||||
if (releasePermit == null) {
|
||||
return;
|
||||
}
|
||||
this.releasePermit = null;
|
||||
releasePermit();
|
||||
}
|
||||
}
|
||||
|
||||
class LocalAppProxyResponseBodyLease {
|
||||
public readonly reader: ReadableStreamDefaultReader<Uint8Array>;
|
||||
private readonly permit: LocalAppProxyConcurrencyPermit;
|
||||
private settled = false;
|
||||
|
||||
public constructor(body: ReadableStream<Uint8Array>, permit: LocalAppProxyConcurrencyPermit) {
|
||||
this.reader = body.getReader();
|
||||
this.permit = permit;
|
||||
}
|
||||
|
||||
public release(): void {
|
||||
if (this.settled) {
|
||||
return;
|
||||
}
|
||||
this.settled = true;
|
||||
try {
|
||||
this.reader.releaseLock();
|
||||
} finally {
|
||||
this.permit.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class LocalAppProxyConcurrencyGate {
|
||||
private active = 0;
|
||||
private readonly waiting: Array<{readonly admit: () => void}> = [];
|
||||
private readonly newestFirst: boolean;
|
||||
|
||||
public constructor(newestFirst: boolean) {
|
||||
this.newestFirst = newestFirst;
|
||||
}
|
||||
|
||||
public acquire(signal: AbortSignal): Promise<LocalAppProxyConcurrencyPermit> {
|
||||
signal.throwIfAborted();
|
||||
if (this.active < LOCAL_APP_PROXY_MAX_CONCURRENT_REQUESTS) {
|
||||
this.active += 1;
|
||||
return Promise.resolve(new LocalAppProxyConcurrencyPermit(() => this.release()));
|
||||
}
|
||||
if (this.waiting.length >= LOCAL_APP_PROXY_MAX_QUEUED_REQUESTS) {
|
||||
throw new LocalAppProxyQueueOverflowError();
|
||||
}
|
||||
return new Promise<LocalAppProxyConcurrencyPermit>((resolve, reject) => {
|
||||
let waiter: (typeof this.waiting)[number];
|
||||
const onAbort = () => {
|
||||
const index = this.waiting.indexOf(waiter);
|
||||
if (index === -1) {
|
||||
return;
|
||||
}
|
||||
this.waiting.splice(index, 1);
|
||||
reject(signal.reason);
|
||||
};
|
||||
const admit = () => {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
resolve(new LocalAppProxyConcurrencyPermit(() => this.release()));
|
||||
};
|
||||
waiter = {admit};
|
||||
this.waiting.push(waiter);
|
||||
signal.addEventListener('abort', onAbort, {once: true});
|
||||
});
|
||||
}
|
||||
|
||||
private release(): void {
|
||||
const next = this.newestFirst ? this.waiting.pop() : this.waiting.shift();
|
||||
if (next != null) {
|
||||
next.admit();
|
||||
return;
|
||||
}
|
||||
this.active -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
interface DesktopLocalAppProxyClientDependencies {
|
||||
readonly outboundHTTP: DesktopOutboundHTTP;
|
||||
readonly traffic: LocalAppProxyTraffic;
|
||||
@@ -226,35 +129,20 @@ function emitLocalAppUploadProgress(progress: DesktopLocalAppUploadProgress): vo
|
||||
|
||||
export class DesktopLocalAppProxyClient {
|
||||
private readonly outboundHTTP: DesktopOutboundHTTP;
|
||||
private readonly traffic: LocalAppProxyTraffic;
|
||||
private readonly concurrency: LocalAppProxyConcurrencyGate;
|
||||
private readonly responseHeadersDeadlineMs: number | null;
|
||||
|
||||
public constructor(dependencies: DesktopLocalAppProxyClientDependencies) {
|
||||
this.outboundHTTP = dependencies.outboundHTTP;
|
||||
this.traffic = dependencies.traffic;
|
||||
const remoteResource = dependencies.traffic === LocalAppProxyTraffic.REMOTE_RESOURCE;
|
||||
this.concurrency = new LocalAppProxyConcurrencyGate(remoteResource);
|
||||
this.responseHeadersDeadlineMs = remoteResource ? LOCAL_APP_PROXY_RESOURCE_RESPONSE_HEADERS_DEADLINE_MS : null;
|
||||
this.responseHeadersDeadlineMs =
|
||||
dependencies.traffic === LocalAppProxyTraffic.REMOTE_RESOURCE
|
||||
? LOCAL_APP_PROXY_RESOURCE_RESPONSE_HEADERS_DEADLINE_MS
|
||||
: null;
|
||||
}
|
||||
|
||||
public async fetch(request: LocalAppProxyFetchRequest): Promise<Response> {
|
||||
const permit = await this.concurrency.acquire(request.signal);
|
||||
try {
|
||||
return await this.runFetchWithinHeadersDeadline(request, permit);
|
||||
} catch (error) {
|
||||
permit.release();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async runFetchWithinHeadersDeadline(
|
||||
request: LocalAppProxyFetchRequest,
|
||||
permit: LocalAppProxyConcurrencyPermit,
|
||||
): Promise<Response> {
|
||||
const deadlineMs = this.responseHeadersDeadlineMs;
|
||||
if (deadlineMs == null || requestHasBody(request.method)) {
|
||||
return await this.runFetch(request, permit);
|
||||
return await this.runFetch(request);
|
||||
}
|
||||
const deadline = new AbortController();
|
||||
const timer = setTimeout(
|
||||
@@ -263,16 +151,13 @@ export class DesktopLocalAppProxyClient {
|
||||
);
|
||||
timer.unref();
|
||||
try {
|
||||
return await this.runFetch({...request, signal: AbortSignal.any([request.signal, deadline.signal])}, permit);
|
||||
return await this.runFetch({...request, signal: AbortSignal.any([request.signal, deadline.signal])});
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
private async runFetch(
|
||||
request: LocalAppProxyFetchRequest,
|
||||
permit: LocalAppProxyConcurrencyPermit,
|
||||
): Promise<Response> {
|
||||
private async runFetch(request: LocalAppProxyFetchRequest): Promise<Response> {
|
||||
const target = new URL(request.targetURL);
|
||||
const body = requestHasBody(request.method) ? request.body : null;
|
||||
const outboundRequest: DesktopOutboundHTTPRequest = {
|
||||
@@ -282,12 +167,11 @@ export class DesktopLocalAppProxyClient {
|
||||
maximumRequestBodyBytes: LOCAL_APP_PROXY_REQUEST_BODY_MAX_BYTES,
|
||||
method: request.method,
|
||||
originTrust: DesktopOriginTrust.REGISTERED,
|
||||
serviceName: this.traffic,
|
||||
signal: request.signal,
|
||||
timeoutMs: LOCAL_APP_PROXY_DEADLINE_MS,
|
||||
url: target.href,
|
||||
};
|
||||
return buildProxyResponse(await this.outboundHTTP.request(outboundRequest), request.method, permit);
|
||||
return buildProxyResponse(await this.outboundHTTP.request(outboundRequest), request.method);
|
||||
}
|
||||
|
||||
public async settleFailure({
|
||||
@@ -368,11 +252,7 @@ function instrumentedRequestBody(
|
||||
);
|
||||
}
|
||||
|
||||
function buildProxyResponse(
|
||||
message: DesktopOutboundHTTPMessage,
|
||||
method: string,
|
||||
permit: LocalAppProxyConcurrencyPermit,
|
||||
): Response {
|
||||
function buildProxyResponse(message: DesktopOutboundHTTPMessage, method: string): Response {
|
||||
try {
|
||||
const status = message.status;
|
||||
if (status < MINIMUM_RESPONSE_STATUS || status > MAXIMUM_RESPONSE_STATUS) {
|
||||
@@ -381,59 +261,20 @@ function buildProxyResponse(
|
||||
const headers = responseHeaders(message.headers);
|
||||
if (method === LOCAL_APP_PROXY_HEAD_METHOD || NULL_BODY_STATUSES.has(status)) {
|
||||
message.message.resume();
|
||||
permit.release();
|
||||
return new Response(null, {status, statusText: message.statusText, headers});
|
||||
}
|
||||
const decoded = decodedMessageStream(message.message, contentEncoding(message.headers));
|
||||
const body = holdPermitForResponseBody(Readable.toWeb(decoded) as ReadableStream<Uint8Array>, permit);
|
||||
return new Response(body, {
|
||||
return new Response(Readable.toWeb(decoded) as ReadableStream<Uint8Array>, {
|
||||
status,
|
||||
statusText: message.statusText,
|
||||
headers,
|
||||
});
|
||||
} catch (error) {
|
||||
message.message.destroy();
|
||||
permit.release();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function holdPermitForResponseBody(
|
||||
body: ReadableStream<Uint8Array>,
|
||||
permit: LocalAppProxyConcurrencyPermit,
|
||||
): ReadableStream<Uint8Array> {
|
||||
const lease = new LocalAppProxyResponseBodyLease(body, permit);
|
||||
return new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
try {
|
||||
const chunk = await lease.reader.read();
|
||||
if (chunk.done) {
|
||||
try {
|
||||
controller.close();
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
return;
|
||||
}
|
||||
controller.enqueue(chunk.value);
|
||||
} catch (error) {
|
||||
try {
|
||||
controller.error(error);
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
}
|
||||
},
|
||||
async cancel(reason) {
|
||||
try {
|
||||
await lease.reader.cancel(reason);
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function responseHeaders(incoming: http.IncomingHttpHeaders): Headers {
|
||||
const headers = new Headers();
|
||||
for (const [name, value] of Object.entries(incoming)) {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import assert from 'node:assert/strict';
|
||||
import {Buffer} from 'node:buffer';
|
||||
import {Readable} from 'node:stream';
|
||||
import {PassThrough, Readable} from 'node:stream';
|
||||
import {describe, mock, test} from 'node:test';
|
||||
import {installElectronStub} from './LocalAppTestSupport.test.mjs';
|
||||
|
||||
@@ -10,17 +10,9 @@ installElectronStub();
|
||||
|
||||
const {DesktopLocalAppProxyClient, LocalAppProxyTraffic} = await import('./LocalAppProxyClient.ts');
|
||||
|
||||
const PROXY_MAX_CONCURRENT_REQUESTS = 32;
|
||||
const PROXY_MAX_QUEUED_REQUESTS = 4096;
|
||||
const OUTBOUND_MAX_IN_FLIGHT_PER_SERVICE = 64;
|
||||
const RENDERED_AVATAR_COUNT = 120;
|
||||
|
||||
class DesktopOutboundHTTPCapacityError extends Error {
|
||||
constructor(limit) {
|
||||
super(`Desktop outbound HTTP exceeded its ${limit} limit`);
|
||||
this.name = 'DesktopOutboundHTTPCapacityError';
|
||||
}
|
||||
}
|
||||
const PARKED_MEDIA_BODY_COUNT = 200;
|
||||
const RESOURCE_HEADERS_DEADLINE_MS = 30_000;
|
||||
|
||||
function tick() {
|
||||
return new Promise((resolve) => {
|
||||
@@ -28,33 +20,6 @@ function tick() {
|
||||
});
|
||||
}
|
||||
|
||||
function createOutboundHTTP() {
|
||||
const state = {inFlight: 0, peakInFlight: 0, started: 0, pending: []};
|
||||
const outboundHTTP = {
|
||||
request: (request) => {
|
||||
if (state.inFlight >= OUTBOUND_MAX_IN_FLIGHT_PER_SERVICE) {
|
||||
throw new DesktopOutboundHTTPCapacityError(`in-flight request for ${request.serviceName}`);
|
||||
}
|
||||
state.inFlight += 1;
|
||||
state.started += 1;
|
||||
state.peakInFlight = Math.max(state.peakInFlight, state.inFlight);
|
||||
return new Promise((resolve) => {
|
||||
state.pending.push(() => {
|
||||
state.inFlight -= 1;
|
||||
resolve({
|
||||
headers: {'content-type': 'image/webp'},
|
||||
message: Readable.from([Buffer.from('avatar-bytes')]),
|
||||
status: 200,
|
||||
statusText: 'OK',
|
||||
url: new URL(request.url),
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
return {outboundHTTP, state};
|
||||
}
|
||||
|
||||
function avatarRequest(index, signal) {
|
||||
return {
|
||||
targetURL: `https://media.fluxer.app/avatars/${index}.webp`,
|
||||
@@ -68,131 +33,17 @@ function avatarRequest(index, signal) {
|
||||
};
|
||||
}
|
||||
|
||||
async function pump(state, work) {
|
||||
let finished = false;
|
||||
const result = work.finally(() => {
|
||||
finished = true;
|
||||
});
|
||||
result.catch(() => {});
|
||||
while (!finished) {
|
||||
for (const settle of state.pending.splice(0)) {
|
||||
settle();
|
||||
}
|
||||
await tick();
|
||||
}
|
||||
return await result;
|
||||
}
|
||||
|
||||
function readAvatar(client, index, signal) {
|
||||
return client.fetch(avatarRequest(index, signal)).then(async (response) => ({
|
||||
status: response.status,
|
||||
text: await response.text(),
|
||||
}));
|
||||
}
|
||||
|
||||
describe('the local app proxy bounds its own concurrency instead of overrunning outbound HTTP', () => {
|
||||
test('every avatar of a fully rendered chat resolves rather than tripping the outbound capacity cap', async () => {
|
||||
const {outboundHTTP, state} = createOutboundHTTP();
|
||||
const client = new DesktopLocalAppProxyClient({outboundHTTP, traffic: LocalAppProxyTraffic.REMOTE_RESOURCE});
|
||||
const controller = new AbortController();
|
||||
|
||||
const fetches = [];
|
||||
for (let index = 0; index < RENDERED_AVATAR_COUNT; index += 1) {
|
||||
fetches.push(readAvatar(client, index, controller.signal));
|
||||
}
|
||||
const responses = await pump(state, Promise.all(fetches));
|
||||
|
||||
assert.equal(responses.length, RENDERED_AVATAR_COUNT);
|
||||
assert.deepEqual(new Set(responses.map((response) => response.status)), new Set([200]));
|
||||
assert.equal(state.started, RENDERED_AVATAR_COUNT);
|
||||
assert.deepEqual(
|
||||
responses.map((response) => response.text),
|
||||
new Array(RENDERED_AVATAR_COUNT).fill('avatar-bytes'),
|
||||
);
|
||||
});
|
||||
|
||||
test('no more requests are in flight at any instant than the proxy concurrency limit', async () => {
|
||||
const {outboundHTTP, state} = createOutboundHTTP();
|
||||
const client = new DesktopLocalAppProxyClient({outboundHTTP, traffic: LocalAppProxyTraffic.REMOTE_RESOURCE});
|
||||
const controller = new AbortController();
|
||||
|
||||
const fetches = [];
|
||||
for (let index = 0; index < RENDERED_AVATAR_COUNT; index += 1) {
|
||||
fetches.push(readAvatar(client, index, controller.signal));
|
||||
}
|
||||
await tick();
|
||||
const peakBeforeAnyCompletion = state.peakInFlight;
|
||||
await pump(state, Promise.all(fetches));
|
||||
|
||||
assert.equal(peakBeforeAnyCompletion, PROXY_MAX_CONCURRENT_REQUESTS);
|
||||
assert.equal(state.peakInFlight, PROXY_MAX_CONCURRENT_REQUESTS);
|
||||
});
|
||||
|
||||
test('an aborted waiter rejects with the abort reason and leaves its queue slot to the next waiter', async () => {
|
||||
const {outboundHTTP, state} = createOutboundHTTP();
|
||||
const client = new DesktopLocalAppProxyClient({outboundHTTP, traffic: LocalAppProxyTraffic.REMOTE_RESOURCE});
|
||||
const held = new AbortController();
|
||||
|
||||
const holding = [];
|
||||
for (let index = 0; index < PROXY_MAX_CONCURRENT_REQUESTS; index += 1) {
|
||||
holding.push(readAvatar(client, index, held.signal));
|
||||
}
|
||||
await tick();
|
||||
assert.equal(state.started, PROXY_MAX_CONCURRENT_REQUESTS);
|
||||
|
||||
const abandoned = new AbortController();
|
||||
const abandonedFetch = client.fetch(avatarRequest(998, abandoned.signal));
|
||||
const survivor = readAvatar(client, 999, held.signal);
|
||||
const reason = new Error('the renderer navigated away');
|
||||
abandoned.abort(reason);
|
||||
|
||||
await assert.rejects(abandonedFetch, (error) => error === reason);
|
||||
assert.equal(state.started, PROXY_MAX_CONCURRENT_REQUESTS);
|
||||
|
||||
state.pending.shift()();
|
||||
await tick();
|
||||
await tick();
|
||||
assert.equal(state.started, PROXY_MAX_CONCURRENT_REQUESTS + 1);
|
||||
|
||||
const settled = await pump(state, Promise.all([survivor, ...holding]));
|
||||
assert.equal(settled[0].status, 200);
|
||||
assert.equal(settled.length, PROXY_MAX_CONCURRENT_REQUESTS + 1);
|
||||
});
|
||||
|
||||
test('the queue refuses a request past its bound rather than growing without limit', async () => {
|
||||
const {outboundHTTP, state} = createOutboundHTTP();
|
||||
const client = new DesktopLocalAppProxyClient({outboundHTTP, traffic: LocalAppProxyTraffic.REMOTE_RESOURCE});
|
||||
const controller = new AbortController();
|
||||
|
||||
const accepted = [];
|
||||
for (let index = 0; index < PROXY_MAX_CONCURRENT_REQUESTS + PROXY_MAX_QUEUED_REQUESTS; index += 1) {
|
||||
accepted.push(readAvatar(client, index, controller.signal));
|
||||
}
|
||||
|
||||
await assert.rejects(
|
||||
client.fetch(avatarRequest(Number.MAX_SAFE_INTEGER, controller.signal)),
|
||||
(error) => error.name === 'LocalAppProxyQueueOverflowError',
|
||||
);
|
||||
|
||||
const settled = await pump(state, Promise.all(accepted));
|
||||
assert.equal(settled.length, PROXY_MAX_CONCURRENT_REQUESTS + PROXY_MAX_QUEUED_REQUESTS);
|
||||
});
|
||||
});
|
||||
|
||||
const RESOURCE_HEADERS_DEADLINE_MS = 30_000;
|
||||
|
||||
function createSilentUpstream() {
|
||||
function createUpstream({streamBodies}) {
|
||||
const state = {started: [], aborted: 0};
|
||||
const outboundHTTP = {
|
||||
request: (request) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const entry = {
|
||||
url: request.url,
|
||||
serviceName: request.serviceName,
|
||||
answer: () =>
|
||||
resolve({
|
||||
headers: {'content-type': 'image/webp'},
|
||||
message: Readable.from([Buffer.from('avatar-bytes')]),
|
||||
message: streamBodies ? new PassThrough() : Readable.from([Buffer.from('avatar-bytes')]),
|
||||
status: 200,
|
||||
statusText: 'OK',
|
||||
url: new URL(request.url),
|
||||
@@ -212,80 +63,64 @@ function createSilentUpstream() {
|
||||
return {outboundHTTP, state};
|
||||
}
|
||||
|
||||
function apiRequest(signal) {
|
||||
return {...avatarRequest(0, signal), targetURL: 'https://api.fluxer.app/v1/users/@me'};
|
||||
}
|
||||
describe('the local app proxy never queues a request behind another', () => {
|
||||
test('every avatar of a fully rendered chat starts at once', async () => {
|
||||
const {outboundHTTP, state} = createUpstream({streamBodies: false});
|
||||
const client = new DesktopLocalAppProxyClient({outboundHTTP, traffic: LocalAppProxyTraffic.REMOTE_RESOURCE});
|
||||
const renderer = new AbortController();
|
||||
|
||||
describe('media that a slow or silent self-host never answers cannot wedge the local app proxy', () => {
|
||||
test('avatars dropped while their upstream sends no headers free their permits at the headers deadline', async () => {
|
||||
const fetches = [];
|
||||
for (let index = 0; index < RENDERED_AVATAR_COUNT; index += 1) {
|
||||
fetches.push(client.fetch(avatarRequest(index, renderer.signal)));
|
||||
}
|
||||
await tick();
|
||||
assert.equal(state.started.length, RENDERED_AVATAR_COUNT);
|
||||
|
||||
for (const entry of state.started) {
|
||||
entry.answer();
|
||||
}
|
||||
const texts = await Promise.all(fetches.map(async (fetch) => await (await fetch).text()));
|
||||
assert.deepEqual(texts, new Array(RENDERED_AVATAR_COUNT).fill('avatar-bytes'));
|
||||
});
|
||||
|
||||
test('a media request starts at once while earlier response bodies sit unread', async () => {
|
||||
const {outboundHTTP, state} = createUpstream({streamBodies: true});
|
||||
const client = new DesktopLocalAppProxyClient({outboundHTTP, traffic: LocalAppProxyTraffic.REMOTE_RESOURCE});
|
||||
const renderer = new AbortController();
|
||||
|
||||
for (let index = 0; index < PARKED_MEDIA_BODY_COUNT; index += 1) {
|
||||
const fetch = client.fetch(avatarRequest(index, renderer.signal));
|
||||
await tick();
|
||||
state.started.at(-1).answer();
|
||||
assert.equal((await fetch).status, 200);
|
||||
}
|
||||
|
||||
const next = client.fetch(avatarRequest(PARKED_MEDIA_BODY_COUNT, renderer.signal));
|
||||
await tick();
|
||||
assert.equal(state.started.length, PARKED_MEDIA_BODY_COUNT + 1);
|
||||
state.started.at(-1).answer();
|
||||
assert.equal((await next).status, 200);
|
||||
renderer.abort();
|
||||
});
|
||||
});
|
||||
|
||||
describe('media that a slow or silent self-host never answers is cut off', () => {
|
||||
test('a silent upstream is aborted at the headers deadline', async () => {
|
||||
mock.timers.enable({apis: ['setTimeout']});
|
||||
try {
|
||||
const {outboundHTTP, state} = createSilentUpstream();
|
||||
const {outboundHTTP, state} = createUpstream({streamBodies: false});
|
||||
const client = new DesktopLocalAppProxyClient({outboundHTTP, traffic: LocalAppProxyTraffic.REMOTE_RESOURCE});
|
||||
const renderer = new AbortController();
|
||||
const dropped = [];
|
||||
for (let index = 0; index < PROXY_MAX_CONCURRENT_REQUESTS; index += 1) {
|
||||
const fetch = client.fetch(avatarRequest(index, renderer.signal));
|
||||
fetch.catch(() => {});
|
||||
dropped.push(fetch);
|
||||
}
|
||||
const silent = client.fetch(avatarRequest(0, renderer.signal));
|
||||
silent.catch(() => {});
|
||||
await tick();
|
||||
const fresh = client.fetch(avatarRequest(1000, renderer.signal));
|
||||
await tick();
|
||||
assert.equal(state.started.length, PROXY_MAX_CONCURRENT_REQUESTS);
|
||||
|
||||
mock.timers.tick(RESOURCE_HEADERS_DEADLINE_MS);
|
||||
await Promise.allSettled(dropped);
|
||||
await tick();
|
||||
|
||||
assert.equal(state.aborted, PROXY_MAX_CONCURRENT_REQUESTS);
|
||||
assert.equal(state.started.length, PROXY_MAX_CONCURRENT_REQUESTS + 1);
|
||||
state.started.at(-1).answer();
|
||||
const response = await fresh;
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(await response.text(), 'avatar-bytes');
|
||||
await assert.rejects(silent, (error) => error.name === 'LocalAppProxyResponseHeadersDeadlineError');
|
||||
assert.equal(state.aborted, 1);
|
||||
} finally {
|
||||
mock.timers.reset();
|
||||
}
|
||||
});
|
||||
|
||||
test('API requests keep flowing while every media permit is held by a silent upstream', async () => {
|
||||
const {outboundHTTP, state} = createSilentUpstream();
|
||||
const media = new DesktopLocalAppProxyClient({outboundHTTP, traffic: LocalAppProxyTraffic.REMOTE_RESOURCE});
|
||||
const api = new DesktopLocalAppProxyClient({outboundHTTP, traffic: LocalAppProxyTraffic.API});
|
||||
const renderer = new AbortController();
|
||||
for (let index = 0; index < PROXY_MAX_CONCURRENT_REQUESTS * 2; index += 1) {
|
||||
media.fetch(avatarRequest(index, renderer.signal)).catch(() => {});
|
||||
}
|
||||
await tick();
|
||||
const apiCall = api.fetch(apiRequest(renderer.signal));
|
||||
await tick();
|
||||
const started = state.started.at(-1);
|
||||
assert.equal(started.url, 'https://api.fluxer.app/v1/users/@me');
|
||||
assert.equal(started.serviceName, LocalAppProxyTraffic.API);
|
||||
started.answer();
|
||||
assert.equal((await apiCall).status, 200);
|
||||
renderer.abort();
|
||||
});
|
||||
|
||||
test('the newest queued media request is admitted first so what is on screen loads before what scrolled away', async () => {
|
||||
const {outboundHTTP, state} = createSilentUpstream();
|
||||
const client = new DesktopLocalAppProxyClient({outboundHTTP, traffic: LocalAppProxyTraffic.REMOTE_RESOURCE});
|
||||
const renderer = new AbortController();
|
||||
const fetches = [];
|
||||
for (let index = 0; index < PROXY_MAX_CONCURRENT_REQUESTS + 3; index += 1) {
|
||||
const fetch = client.fetch(avatarRequest(index, renderer.signal));
|
||||
fetch.catch(() => {});
|
||||
fetches.push(fetch);
|
||||
}
|
||||
await tick();
|
||||
state.started[0].answer();
|
||||
await (await fetches[0]).text();
|
||||
await tick();
|
||||
assert.equal(
|
||||
state.started.at(-1).url,
|
||||
`https://media.fluxer.app/avatars/${PROXY_MAX_CONCURRENT_REQUESTS + 2}.webp`,
|
||||
);
|
||||
renderer.abort();
|
||||
});
|
||||
});
|
||||
@@ -15,7 +15,6 @@ const INSTANCE_RESPONSE_MAX_BYTES = 1024 * 1024;
|
||||
const INSTANCE_RESPONSE_MAX_CHUNKS = 1024;
|
||||
const INSTANCE_FETCH_TIMEOUT_MS = 15_000;
|
||||
|
||||
const SELECTED_INSTANCE_SERVICE_NAME = 'desktop_selected_instance';
|
||||
const INSTANCE_RESPONSE_DESCRIPTION = 'Instance response';
|
||||
const RESET_CONTENT_STATUS = 205;
|
||||
|
||||
@@ -80,7 +79,6 @@ export class DesktopSelectedInstanceClient {
|
||||
headers: request.headers ?? null,
|
||||
method,
|
||||
originTrust: DesktopOriginTrust.BOUND,
|
||||
serviceName: SELECTED_INSTANCE_SERVICE_NAME,
|
||||
signal: request.signal ?? null,
|
||||
timeoutMs,
|
||||
url: request.url,
|
||||
|
||||
@@ -405,7 +405,7 @@ Each applied group records one Admin audit entry with the target type `guild` an
|
||||
|
||||
<RouteHeader method="DELETE" path="/v1/admin/guilds/{guild_id}" auditReason />
|
||||
|
||||
Permanently deletes a guild and every record it owns. Requires `guild:delete`.
|
||||
Permanently deletes a guild and every record it owns. Requires `guild:delete`. The guild becomes unavailable to every member at once, and a `guildProcessDeletion` job on the `lifecycle` lane removes its records.
|
||||
|
||||
### Path parameters
|
||||
|
||||
|
||||
@@ -126,7 +126,7 @@ The queue drops a job 7 days after it was queued. A job still recorded as `queue
|
||||
|
||||
Use the returned `task_type` value to filter [List jobs](#list-jobs).
|
||||
|
||||
The tasks queued by [Bulk jobs](/admin-api/bulk-jobs/) are `bulkUpdateUserFlags`, `bulkUpdateGuildFeatures`, `bulkAddGuildMembers`, and `bulkScheduleUserDeletion`. `harvestUserData` and `harvestGuildData` build [archives](/admin-api/archives/), `sendSystemDm` delivers a [system DM broadcast](/admin-api/system-dms/), and `refreshSearchIndex` rebuilds a [search index](/admin-api/search-indexes/). `removeChannelFollowers` runs on the `crosspost` lane after an announcement channel is deleted or converted into a text channel, and removes the channel follower webhooks that follow it.
|
||||
The tasks queued by [Bulk jobs](/admin-api/bulk-jobs/) are `bulkUpdateUserFlags`, `bulkUpdateGuildFeatures`, `bulkAddGuildMembers`, and `bulkScheduleUserDeletion`. `harvestUserData` and `harvestGuildData` build [archives](/admin-api/archives/), `sendSystemDm` delivers a [system DM broadcast](/admin-api/system-dms/), and `refreshSearchIndex` rebuilds a [search index](/admin-api/search-indexes/). `guildProcessDeletion` removes the records of a deleted guild after [Delete guild](/admin-api/guilds/#delete-guild) or the owner deletes it. `removeChannelFollowers` runs on the `crosspost` lane after an announcement channel is deleted or converted into a text channel, and removes the channel follower webhooks that follow it.
|
||||
|
||||
Thread work runs on the `batch` lane. `deleteChannelThreads` deletes the threads of a deleted parent channel, `removeThreadMembershipsForGuildMember` removes the thread memberships of a user who is no longer a guild member, and `syncThreadSearchDocument` and `backfillThreadSearch` keep the thread search index current. `archiveInactiveThreads` runs every minute with no job record and archives the threads whose auto archive time has passed. `repairThreadIndexes` rebuilds the thread list entries of the threads it names. `seedThreadPermissions` runs once for a guild. It clears the thread permission bits from every overwrite mask. It then adds `CREATE_PUBLIC_THREADS`, `CREATE_PRIVATE_THREADS`, and `SEND_MESSAGES_IN_THREADS` to each overwrite mask that has `SEND_MESSAGES`, and to the everyone role when that role has `SEND_MESSAGES`.
|
||||
|
||||
|
||||
@@ -739,7 +739,7 @@ Replacing an image removes the previous asset. A failed replacement leaves the p
|
||||
|
||||
<RouteHeader method="POST" path="/v1/guilds/{guild_id}/delete" mfa />
|
||||
|
||||
Permanently deletes the guild and returns 204 with an empty body. Requires the guild owner and sudo mode. Emits a [Guild Delete](/gateway/events/#guild-delete) Gateway event to every member session.
|
||||
Permanently deletes the guild and returns 204 with an empty body. Requires the guild owner and sudo mode. Emits a [Guild Delete](/gateway/events/#guild-delete) Gateway event to every member session. The guild is unavailable from that point, and its channels, messages and other records are removed in the background shortly after.
|
||||
|
||||
Fluxer refuses a non-owner with 403 `MISSING_PERMISSIONS`. A bot can never own a guild, so a bot credential never satisfies the requirement.
|
||||
|
||||
|
||||
@@ -332,7 +332,7 @@ All are optional.
|
||||
|
||||
#### `FLUXER_TRUST_CLIENT_IP_HEADER`
|
||||
|
||||
Default `false`. Whether the client-IP header is trusted. Compose forwards it from `.env` with a default of `true`, because the edge sets the header on every upstream hop. The API has no peer-address fallback, so `false` on `api` makes every non-exempt request a 403.
|
||||
Default `false`. Whether the client-IP header is trusted. Compose forwards it from `.env` with a default of `true`, because the edge sets the header on every upstream hop. With `false`, `api` ignores the header and uses the address of the peer that opened the connection. That suits an `api` that clients reach directly. Behind any proxy it makes every request look as though it came from the proxy.
|
||||
|
||||
#### `FLUXER_CLIENT_IP_HEADER_NAME`
|
||||
|
||||
@@ -342,7 +342,7 @@ Default `x-forwarded-for`. Which header has the client IP. Read by `api`, `worke
|
||||
|
||||
Default empty. Addresses and CIDR ranges exempt from IP bans. Comma separated. A bare IPv4 address exempts that address, a bare IPv6 address exempts its /64, and a CIDR range such as `2001:db8:1200::/56` exempts every address in it. Every entry must parse as an IP or a CIDR range or the API fails at boot. Compose forwards it from `.env`.
|
||||
|
||||
The API returns 403 for any request whose client-IP header is missing, empty, or not a parsable address, and for every request while `FLUXER_TRUST_CLIENT_IP_HEADER` is `false`. The exceptions are `/_health`, `/webhooks/livekit`, `/test`, and the Bluesky client metadata and JWKS routes. The edge sets the header on every upstream hop, so a missing or unparsable header happens only in a layout that puts something other than the edge directly in front of `api`. A proxy in front of the edge that never sets the header passes the check, and every request then looks as though it came from the proxy.
|
||||
While `FLUXER_TRUST_CLIENT_IP_HEADER` is `true`, the API returns 403 for any request whose client-IP header is missing, empty, or not a parsable address. The exceptions are `/_health`, `/webhooks/livekit`, `/test`, and the Bluesky client metadata and JWKS routes. The edge sets the header on every upstream hop, so a missing or unparsable header happens only in a layout that puts something other than the edge directly in front of `api`. A proxy in front of the edge that never sets the header passes the check, and every request then looks as though it came from the proxy.
|
||||
|
||||
#### `FLUXER_API_TRUSTED_CALLERS`
|
||||
|
||||
@@ -514,6 +514,8 @@ Every uploaded file lands in an S3-compatible object store, which the stack prov
|
||||
|
||||
Default `http://localhost:3900`. The S3 API address. `media-proxy` defaults to empty instead. Compose forwards it from `.env` with a default of `http://seaweedfs:8333`, the bundled store.
|
||||
|
||||
An endpoint with a bucket in its path, such as `https://s3.example.com/my-bucket` with `FLUXER_S3_FORCE_PATH_STYLE=true`, keeps every Fluxer bucket inside that one bucket. Each bucket name becomes a key prefix there, so uploads land under `fluxer-uploads/` in `my-bucket`.
|
||||
|
||||
#### `FLUXER_S3_PUBLIC_ENDPOINT`
|
||||
|
||||
No default. The host substituted into presigned URLs, which are the only place it appears. Compose forwards it from `.env`, and uses `FLUXER_S3_ENDPOINT` when it is unset, except on `app-proxy`, which gets only a value `.env` sets and adds it to its Content Security Policy. `admin` reads it to find the origin that report downloads are presigned against. See `FLUXER_S3_BUCKET_REPORTS`.
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {ParsedIpAddress} from '@fluxer/ip_utils/src/IpAddress';
|
||||
import {parseIpAddress} from '@fluxer/ip_utils/src/IpAddress';
|
||||
import {canonicalizeIpAddress} from '@fluxer/ip_utils/src/IpAddress';
|
||||
|
||||
interface ClientIpExtractionOptions {
|
||||
trustClientIpHeader?: boolean;
|
||||
clientIpHeaderName?: string;
|
||||
peerAddress?: string | null;
|
||||
}
|
||||
|
||||
export class MissingClientIpError extends Error {
|
||||
@@ -13,7 +13,7 @@ export class MissingClientIpError extends Error {
|
||||
readonly status = 403;
|
||||
|
||||
constructor() {
|
||||
super('Client IP header is required');
|
||||
super('Client IP is required');
|
||||
this.name = 'MissingClientIpError';
|
||||
}
|
||||
}
|
||||
@@ -28,20 +28,12 @@ export function resolveClientIpHeaderName(clientIpHeaderName?: string): string {
|
||||
return normalizeHeaderName(clientIpHeaderName ?? DEFAULT_CLIENT_IP_HEADER_NAME);
|
||||
}
|
||||
|
||||
function parseClientIpHeaderValue(value: string | null): ParsedIpAddress | null {
|
||||
if (value === null) {
|
||||
return null;
|
||||
}
|
||||
return parseIpAddress(value.split(',', 1)[0]);
|
||||
}
|
||||
|
||||
export function extractClientIp(req: Request, options?: ClientIpExtractionOptions): string | null {
|
||||
if (!options?.trustClientIpHeader) {
|
||||
return null;
|
||||
return options?.peerAddress ? canonicalizeIpAddress(options.peerAddress) : null;
|
||||
}
|
||||
return (
|
||||
parseClientIpHeaderValue(req.headers.get(resolveClientIpHeaderName(options.clientIpHeaderName)))?.normalized ?? null
|
||||
);
|
||||
const headerValue = req.headers.get(resolveClientIpHeaderName(options.clientIpHeaderName));
|
||||
return headerValue === null ? null : canonicalizeIpAddress(headerValue.split(',', 1)[0]);
|
||||
}
|
||||
|
||||
export function requireClientIp(req: Request, options?: ClientIpExtractionOptions): string {
|
||||
|
||||
@@ -102,6 +102,17 @@ export function parseIpAddress(value: string): ParsedIpAddress | null {
|
||||
};
|
||||
}
|
||||
|
||||
export function canonicalizeIpAddress(value: string): string | null {
|
||||
const parsed = parseIpAddress(value);
|
||||
if (!parsed) {
|
||||
return null;
|
||||
}
|
||||
if (parsed.family === 'ipv4') {
|
||||
return parsed.normalized;
|
||||
}
|
||||
return getIpv4MappedIpv6(expandIpv6ToGroups(parsed.normalized)) ?? parsed.normalized;
|
||||
}
|
||||
|
||||
export function isValidIp(value: string): boolean {
|
||||
return parseIpAddress(value) !== null;
|
||||
}
|
||||
|
||||
@@ -50,18 +50,43 @@ describe('extractClientIp', () => {
|
||||
'192.168.1.1',
|
||||
);
|
||||
});
|
||||
it('returns null when trust is disabled', () => {
|
||||
it('ignores the header and uses the peer address when trust is disabled', () => {
|
||||
const request = new Request('http://example.com', {
|
||||
headers: {'X-Forwarded-For': '192.168.1.1'},
|
||||
});
|
||||
expect(extractClientIp(request, {trustClientIpHeader: false})).toBeNull();
|
||||
expect(extractClientIp(request)).toBeNull();
|
||||
expect(extractClientIp(request, {trustClientIpHeader: false, peerAddress: '203.0.113.7'})).toBe('203.0.113.7');
|
||||
expect(extractClientIp(request, {peerAddress: '203.0.113.7'})).toBe('203.0.113.7');
|
||||
});
|
||||
it('returns null when no options are provided', () => {
|
||||
it('uses the peer address when trust is disabled and no header is sent', () => {
|
||||
const request = new Request('http://example.com');
|
||||
expect(extractClientIp(request, {trustClientIpHeader: false, peerAddress: '2001:db8::7'})).toBe('2001:db8::7');
|
||||
});
|
||||
it('unmaps an ipv4-mapped peer address', () => {
|
||||
const request = new Request('http://example.com');
|
||||
expect(extractClientIp(request, {peerAddress: '::ffff:203.0.113.7'})).toBe('203.0.113.7');
|
||||
});
|
||||
it('returns null when trust is disabled and there is no usable peer address', () => {
|
||||
const request = new Request('http://example.com', {
|
||||
headers: {'X-Forwarded-For': '192.168.1.1'},
|
||||
});
|
||||
expect(extractClientIp(request)).toBeNull();
|
||||
expect(extractClientIp(request, {trustClientIpHeader: false, peerAddress: null})).toBeNull();
|
||||
expect(extractClientIp(request, {trustClientIpHeader: false, peerAddress: 'not-an-ip'})).toBeNull();
|
||||
});
|
||||
it('ignores the peer address when the header is trusted', () => {
|
||||
const request = new Request('http://example.com', {
|
||||
headers: {'X-Forwarded-For': '198.51.100.4'},
|
||||
});
|
||||
expect(extractClientIp(request, {trustClientIpHeader: true, peerAddress: '10.0.0.2'})).toBe('198.51.100.4');
|
||||
expect(
|
||||
extractClientIp(new Request('http://example.com'), {trustClientIpHeader: true, peerAddress: '10.0.0.2'}),
|
||||
).toBeNull();
|
||||
});
|
||||
it('unmaps an ipv4-mapped header value', () => {
|
||||
const request = new Request('http://example.com', {
|
||||
headers: {'X-Forwarded-For': '::ffff:198.51.100.4, 10.0.0.1'},
|
||||
});
|
||||
expect(extractClientIp(request, {trustClientIpHeader: true})).toBe('198.51.100.4');
|
||||
});
|
||||
it('returns null for missing or invalid headers', () => {
|
||||
expect(extractClientIp(new Request('http://example.com'), {trustClientIpHeader: true})).toBeNull();
|
||||
@@ -115,9 +140,16 @@ describe('requireClientIp', () => {
|
||||
});
|
||||
expect(requireClientIp(request, {trustClientIpHeader: true})).toBe('192.168.1.1');
|
||||
});
|
||||
it('returns the peer address when the header is not trusted', () => {
|
||||
const request = new Request('http://example.com', {
|
||||
headers: {'X-Forwarded-For': '192.168.1.1'},
|
||||
});
|
||||
expect(requireClientIp(request, {trustClientIpHeader: false, peerAddress: '203.0.113.7'})).toBe('203.0.113.7');
|
||||
});
|
||||
it('throws typed error when missing', () => {
|
||||
const request = new Request('http://example.com');
|
||||
expect(() => requireClientIp(request, {trustClientIpHeader: true})).toThrow(MissingClientIpError);
|
||||
expect(() => requireClientIp(request, {trustClientIpHeader: true})).toThrow('Client IP header is required');
|
||||
expect(() => requireClientIp(request, {trustClientIpHeader: true})).toThrow('Client IP is required');
|
||||
expect(() => requireClientIp(request, {trustClientIpHeader: false})).toThrow(MissingClientIpError);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {
|
||||
canonicalizeIpAddress,
|
||||
getIpNetworkKey,
|
||||
getSameIpDecisionKey,
|
||||
getSubnet,
|
||||
@@ -92,6 +93,21 @@ describe('parseIpAddress', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('canonicalizeIpAddress', () => {
|
||||
it('keeps ipv4 and plain ipv6 in their normalised form', () => {
|
||||
expect(canonicalizeIpAddress(' 203.0.113.5 ')).toBe('203.0.113.5');
|
||||
expect(canonicalizeIpAddress('[2001:DB8::1]')).toBe('2001:db8::1');
|
||||
});
|
||||
it('unmaps ipv4-mapped ipv6 in dotted and hex form', () => {
|
||||
expect(canonicalizeIpAddress('::ffff:203.0.113.5')).toBe('203.0.113.5');
|
||||
expect(canonicalizeIpAddress('::ffff:cb00:7105')).toBe('203.0.113.5');
|
||||
});
|
||||
it('returns null for anything that is not an address', () => {
|
||||
expect(canonicalizeIpAddress('')).toBeNull();
|
||||
expect(canonicalizeIpAddress('not-an-ip')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('isValidIp', () => {
|
||||
it('accepts valid values', () => {
|
||||
expect(isValidIp('192.168.1.1')).toBe(true);
|
||||
|
||||
@@ -5903,6 +5903,10 @@
|
||||
{
|
||||
"name": "content_warning_text",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "invites_paused_until",
|
||||
"type": "timestamp"
|
||||
}
|
||||
],
|
||||
"primary_key": "((guild_id))",
|
||||
|
||||
Reference in new issue
Block a user