perf(repo): parallelise api tests and cache image builds in ghcr (#1984)

This commit is contained in:
Hampus authored and GitHub committed 2026-08-24 21:24:39 +02:00
1 parent 3afad20e36
commit c226eb7211
38 files changed
+701 -264

No files matched your search

@@ -45,6 +45,10 @@ class SudoModeService {
let sudoModeServiceInstance: SudoModeService | null = null;
export function resetSudoModeServiceForTesting(): void {
sudoModeServiceInstance = null;
}
export function getSudoModeService(): SudoModeService {
if (!sudoModeServiceInstance) {
sudoModeServiceInstance = new SudoModeService();
@@ -25,6 +25,10 @@ interface NormalizedSsoConfigValidationResult extends SsoConfigValidationInput {
let ssoRequestUrlPolicy: RequestUrlPolicy | null = null;
export function resetSsoRequestUrlPolicyForTesting(): void {
ssoRequestUrlPolicy = null;
}
export function getSsoRequestUrlPolicy(): RequestUrlPolicy {
if (ssoRequestUrlPolicy === null) {
ssoRequestUrlPolicy = createPublicInternetRequestUrlPolicy({
@@ -174,6 +174,10 @@ export class LimitConfigService {
}
}
export function resetGlobalLimitConfigServiceForTesting(): void {
globalLimitConfigService = null;
}
export function getGlobalLimitConfigSnapshot(): LimitConfigSnapshot {
if (!globalLimitConfigService) {
throw new Error('LimitConfigService global instance has not been initialized');
@@ -89,6 +89,15 @@ class BannedAvatarHashCache {
return this.banned.size;
}
resetForTesting(): void {
this.shutdown();
this.banned = new Set();
this.kvClient = null;
this.subscriberInitialized = false;
this.consecutiveFailures = 0;
this.isInitialized = false;
}
shutdown(): void {
if (this.kvSubscription && this.messageHandler) {
this.kvSubscription.off('message', this.messageHandler);
@@ -85,6 +85,15 @@ class FileShaCache {
return this.banned.size;
}
resetForTesting(): void {
this.shutdown();
this.banned = new Set();
this.kvClient = null;
this.subscriberInitialized = false;
this.consecutiveFailures = 0;
this.isInitialized = false;
}
shutdown(): void {
if (this.kvSubscription && this.messageHandler) {
this.kvSubscription.off('message', this.messageHandler);
@@ -144,6 +144,17 @@ export class PhraseBlocklistCache {
return this.rawPhraseSet.size;
}
resetForTesting(): void {
this.shutdown();
this.rawPhrases = [];
this.rawPhraseSet = new Set();
this.rebuildMatchers();
this.kvClient = null;
this.subscriberInitialized = false;
this.consecutiveFailures = 0;
this.isInitialized = false;
}
shutdown(): void {
if (this.kvSubscription && this.messageHandler) {
this.kvSubscription.off('message', this.messageHandler);
@@ -180,6 +180,15 @@ export class ProfileSubstringBlocklistCache {
return !!canonical && this.getMatchersForCheck(scope).some((matchers) => matchers.rawSet.has(canonical));
}
resetForTesting(): void {
this.shutdown();
this.byScope = new Map();
this.kvClient = null;
this.subscriberInitialized = false;
this.consecutiveFailures = 0;
this.isInitialized = false;
}
shutdown(): void {
if (this.kvSubscription && this.messageHandler) {
this.kvSubscription.off('message', this.messageHandler);
@@ -763,3 +763,15 @@ export const ServiceMiddleware = createMiddleware<HonoEnv>(async (ctx, next) =>
ctx.set('ncmecSubmissionService', getNcmecSubmissionService());
await next();
});
export function resetServiceMiddlewareForTesting(): void {
shutdownReportService();
_inboundSmsChallengeService = null;
_registrationEventsRepository = null;
_riskAssessmentRepository = null;
_historicalOutcomeRepository = null;
_suspiciousIpRepository = null;
_ipInfoService = null;
_registrationRiskEvaluator = null;
_liveKitWebhookService = null;
}
@@ -306,3 +306,20 @@ export function getLiveKitServiceInstance(): ILiveKitService | null {
export function getVoiceRoomStoreInstance(): IVoiceRoomStore | null {
return voiceRoomStoreInstance;
}
export function resetServiceRegistryForTesting(): void {
_kvClient = null;
_snowflakeService = null;
_billingRepository = null;
_blueskyOAuthService = undefined;
_blueskyOAuthInitializationPromise = null;
_blueskyOAuthConfigSignature = null;
_blueskyOAuthInitializationSignature = null;
_disabledBlueskyOAuthService = undefined;
voiceTopology = null;
voiceAvailabilityService = null;
liveKitServiceInstance = null;
voiceRoomStoreInstance = null;
voiceConfigSubscriber = null;
voiceInitializationPromise = null;
}
@@ -97,7 +97,7 @@ import {
getSnowflakeService,
getWorkerService,
} from './ServiceRegistry';
import {singleton} from './Singleton';
import {clearSingletonsForTesting, singleton} from './Singleton';
export const getUserRepository = singleton(() => new UserRepository(getKVClient()));
export const getGuildRepository = singleton(() => new GuildRepository());
@@ -121,7 +121,10 @@ export const getPasswordChangeRepository = singleton(() => new PasswordChangeRep
const getUserContactChangeLogRepository = singleton(() => new UserContactChangeLogRepository());
export const getDonationRepository = singleton(() => new DonationRepository());
const getAdminApiKeyRepository = singleton(() => new AdminApiKeyRepository());
export const getInstanceConfigRepository = singleton(() => new InstanceConfigRepository(getKVClient()));
export const getInstanceConfigRepository = singleton(
() => new InstanceConfigRepository(getKVClient()),
(repository) => repository.shutdown(),
);
export const getGatewayRolloutConfigPublisher = singleton(
() =>
new GatewayRolloutConfigPublisher(
@@ -200,6 +203,7 @@ const getDownloadsStorageService: () => IStorageService = (() => {
export const getErrorI18nService = singleton(() => new ErrorI18nService());
export const getLimitConfigService = singleton(
() => new LimitConfigService(getInstanceConfigRepository(), getCacheService(), getKVClient()),
(service) => service.shutdown(),
);
export const getPurgeQueue: () => IPurgeQueue = singleton(() =>
Config.bunny.purgeEnabled ? new BunnyPurgeQueue(getKVClient()) : new NoopPurgeQueue(),
@@ -434,3 +438,20 @@ export async function initializeServiceSingletons(): Promise<void> {
}
await serviceSingletonInitializationPromise;
}
export function resetServiceSingletonsForTesting(): void {
activityTracker?.shutdown();
clearSingletonsForTesting();
_virusScanInitPromise = null;
serviceSingletonInitializationPromise = null;
bulkMessageDeletionQueue = null;
bulkMessageDeletionQueueClient = null;
premiumStateQueue = null;
premiumStateQueueClient = null;
activityTracker = null;
activityTrackerClient = null;
activityBuffer = null;
activityBufferClient = null;
accountDeletionQueue = null;
accountDeletionQueueClient = null;
}
+15 -1
View File
@@ -1,7 +1,15 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export function singleton<T>(factory: () => T): () => T {
const singletonClearers: Array<() => void> = [];
export function singleton<T>(factory: () => T, teardown?: (instance: T) => void): () => T {
let instance: T | undefined;
singletonClearers.push(() => {
if (instance !== undefined) {
teardown?.(instance);
}
instance = undefined;
});
return () => {
if (instance === undefined) {
instance = factory();
@@ -9,3 +17,9 @@ export function singleton<T>(factory: () => T): () => T {
return instance;
};
}
export function clearSingletonsForTesting(): void {
for (const clear of singletonClearers) {
clear();
}
}
@@ -146,6 +146,17 @@ class UrlBlocklistCache {
};
}
resetForTesting(): void {
this.shutdown();
this.exactUrls = new Set();
this.blockedDomains = new Set();
this.kvClient = null;
this.storageService = null;
this.subscriberInitialized = false;
this.consecutiveFailures = 0;
this.isInitialized = false;
}
shutdown(): void {
if (this.kvSubscription && this.messageHandler) {
this.kvSubscription.off('message', this.messageHandler);
@@ -26,6 +26,10 @@ const FETCH_APPLICATIONS_BY_IDS_CQL = Applications.selectCql({
let cachedAdminSecretHash: string | null = null;
export function resetAdminSecretHashForTesting(): void {
cachedAdminSecretHash = null;
}
async function getAdminSecretHash(): Promise<string | null> {
const secret = Config.admin.oauthClientSecret;
if (!secret) {
@@ -28,3 +28,7 @@ export function isIpBanExempt(ip: string | null | undefined): boolean {
const key = getSameIpDecisionKey(ip);
return key !== null && getExemptDecisionKeys().has(key);
}
export function resetIpBanExemptionsForTesting(): void {
exemptDecisionKeys = null;
}
@@ -42,6 +42,7 @@ import {NoopLogger} from './mocks/NoopLogger';
import {NoopUnfurlerService} from './mocks/NoopUnfurlerService';
import {NoopGatewayService} from './NoopGatewayService';
import {NoopWorkerService} from './NoopWorkerService';
import {resetServiceStateForTesting} from './ResetServiceState';
import {InMemorySearchProvider} from './search/InMemorySearchProvider';
import {TestMediaService} from './TestMediaService';
@@ -91,6 +92,7 @@ export async function createApiTestHarness(options: CreateApiTestHarnessOptions
const mockBlueskyOAuthService = new MockBlueskyOAuthService();
setInjectedBlueskyOAuthService(mockBlueskyOAuthService);
setInjectedAccountPolicyEvaluator(createCurrentBehaviorTestAccountPolicyEvaluator());
resetServiceStateForTesting();
const {
app,
initialize: initializeApp,
@@ -152,6 +154,7 @@ export async function createApiTestHarness(options: CreateApiTestHarnessOptions
setInjectedMediaService(new TestMediaService(fallbackStorageService));
setInjectedSearchProviderService(new NullSearchProvider());
setInjectedBlueskyOAuthService(new MockBlueskyOAuthService());
resetServiceStateForTesting();
resetApiServicesForTesting();
}
async function requestJson(params: {
@@ -0,0 +1,39 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {resetSudoModeServiceForTesting} from '../auth/services/SudoModeService';
import {resetSsoRequestUrlPolicyForTesting} from '../instance/SsoConfigValidation';
import {resetGlobalLimitConfigServiceForTesting} from '../limits/LimitConfigService';
import {bannedAvatarHashCache} from '../middleware/BannedAvatarHashCache';
import {fileShaCache} from '../middleware/FileShaCache';
import {ipBanCache} from '../middleware/IpBanMiddleware';
import {phraseBlocklistCache} from '../middleware/PhraseBlocklistCache';
import {profileSubstringBlocklistCache} from '../middleware/ProfileSubstringBlocklistCache';
import {resetServiceMiddlewareForTesting} from '../middleware/ServiceMiddleware';
import {resetServiceRegistryForTesting} from '../middleware/ServiceRegistry';
import {resetServiceSingletonsForTesting} from '../middleware/ServiceSingletons';
import {torExitListCache} from '../middleware/TorExitListCache';
import {urlBlocklistCache} from '../middleware/UrlBlocklistCache';
import {resetAdminSecretHashForTesting} from '../oauth/repositories/ApplicationRepository';
import {resetIpBanExemptionsForTesting} from '../risk/IpBanExemptions';
import {setThemeCssMaxBytesForTesting} from '../theme/ThemeService';
export function resetServiceStateForTesting(): void {
resetServiceRegistryForTesting();
resetServiceSingletonsForTesting();
resetServiceMiddlewareForTesting();
resetIpBanExemptionsForTesting();
resetGlobalLimitConfigServiceForTesting();
resetSudoModeServiceForTesting();
resetSsoRequestUrlPolicyForTesting();
resetAdminSecretHashForTesting();
setThemeCssMaxBytesForTesting(undefined);
ipBanCache.shutdown();
ipBanCache.resetCaches();
torExitListCache.shutdown();
torExitListCache.clearForTesting();
urlBlocklistCache.resetForTesting();
fileShaCache.resetForTesting();
phraseBlocklistCache.resetForTesting();
bannedAvatarHashCache.resetForTesting();
profileSubstringBlocklistCache.resetForTesting();
}
+15 -2
View File
@@ -5,7 +5,20 @@ import {FileSizeTooLargeError} from '@fluxer/errors/src/domains/core/FileSizeToo
import {Config} from '../Config';
import type {IStorageService} from '../infrastructure/IStorageService';
const MAX_CSS_BYTES = 8 * 1024 * 1024;
export const THEME_CSS_MAX_BYTES = 8 * 1024 * 1024;
let themeCssMaxBytesOverride: number | undefined;
export function setThemeCssMaxBytesForTesting(bytes: number | undefined): void {
themeCssMaxBytesOverride = bytes;
}
export function resolveThemeCssMaxBytes(): number {
if (Config.dev.testModeEnabled && themeCssMaxBytesOverride !== undefined) {
return themeCssMaxBytesOverride;
}
return THEME_CSS_MAX_BYTES;
}
export class ThemeService {
constructor(private readonly storageService: IStorageService) {}
@@ -14,7 +27,7 @@ export class ThemeService {
id: string;
}> {
const cssBytes = Buffer.from(css, 'utf-8');
if (cssBytes.length > MAX_CSS_BYTES) {
if (cssBytes.length > resolveThemeCssMaxBytes()) {
throw new FileSizeTooLargeError();
}
const themeId = randomBytes(8).toString('hex');
@@ -2,15 +2,17 @@
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
import {createTestAccount} from '../../auth/tests/AuthTestUtils';
import {getConfig} from '../../Config';
import {type ApiTestHarness, createApiTestHarness} from '../../test/ApiTestHarness';
import {HTTP_STATUS} from '../../test/TestConstants';
import {createBuilder} from '../../test/TestRequestBuilder';
import {resolveThemeCssMaxBytes, setThemeCssMaxBytesForTesting, THEME_CSS_MAX_BYTES} from '../ThemeService';
interface ThemeCreateResponse {
id: string;
}
const MAX_CSS_BYTES = 8 * 1024 * 1024;
const TEST_MAX_CSS_BYTES = 4096;
describe('Theme CSS size limits', () => {
let harness: ApiTestHarness;
@@ -19,22 +21,33 @@ describe('Theme CSS size limits', () => {
});
beforeEach(async () => {
await harness.reset();
setThemeCssMaxBytesForTesting(TEST_MAX_CSS_BYTES);
});
afterAll(async () => {
setThemeCssMaxBytesForTesting(undefined);
await harness?.shutdown();
});
it('rejects CSS that exceeds the 8MB limit', async () => {
const user = await createTestAccount(harness);
const oversizedCss = 'a'.repeat(MAX_CSS_BYTES + 1);
await createBuilder(harness, user.token)
.post('/users/@me/themes')
.body({css: oversizedCss})
.expect(HTTP_STATUS.BAD_REQUEST, 'FILE_SIZE_TOO_LARGE')
.execute();
it('enforces the 8MB limit in production and only honours the override in test mode', () => {
const config = getConfig();
expect(THEME_CSS_MAX_BYTES).toBe(8 * 1024 * 1024);
expect(resolveThemeCssMaxBytes()).toBe(TEST_MAX_CSS_BYTES);
config.dev.testModeEnabled = false;
try {
expect(resolveThemeCssMaxBytes()).toBe(THEME_CSS_MAX_BYTES);
} finally {
config.dev.testModeEnabled = true;
}
setThemeCssMaxBytesForTesting(undefined);
try {
expect(resolveThemeCssMaxBytes()).toBe(THEME_CSS_MAX_BYTES);
} finally {
setThemeCssMaxBytesForTesting(TEST_MAX_CSS_BYTES);
}
});
it('accepts CSS at exactly the 8MB limit', async () => {
it('accepts CSS at exactly the real 8MB limit through the full HTTP stack', async () => {
setThemeCssMaxBytesForTesting(undefined);
const user = await createTestAccount(harness);
const maxCss = 'a'.repeat(MAX_CSS_BYTES);
const maxCss = 'a'.repeat(THEME_CSS_MAX_BYTES);
const theme = await createBuilder<ThemeCreateResponse>(harness, user.token)
.post('/users/@me/themes')
.body({css: maxCss})
@@ -42,9 +55,38 @@ describe('Theme CSS size limits', () => {
.execute();
expect(theme.id).toBeDefined();
});
it('accepts CSS just under the 8MB limit', async () => {
it('rejects CSS that exceeds the real 8MB limit through the full HTTP stack', async () => {
setThemeCssMaxBytesForTesting(undefined);
const user = await createTestAccount(harness);
const nearMaxCss = 'a'.repeat(MAX_CSS_BYTES - 1);
const oversizedCss = 'a'.repeat(THEME_CSS_MAX_BYTES + 1);
await createBuilder(harness, user.token)
.post('/users/@me/themes')
.body({css: oversizedCss})
.expect(HTTP_STATUS.BAD_REQUEST, 'FILE_SIZE_TOO_LARGE')
.execute();
});
it('rejects CSS that exceeds the limit', async () => {
const user = await createTestAccount(harness);
const oversizedCss = 'a'.repeat(TEST_MAX_CSS_BYTES + 1);
await createBuilder(harness, user.token)
.post('/users/@me/themes')
.body({css: oversizedCss})
.expect(HTTP_STATUS.BAD_REQUEST, 'FILE_SIZE_TOO_LARGE')
.execute();
});
it('accepts CSS at exactly the limit', async () => {
const user = await createTestAccount(harness);
const maxCss = 'a'.repeat(TEST_MAX_CSS_BYTES);
const theme = await createBuilder<ThemeCreateResponse>(harness, user.token)
.post('/users/@me/themes')
.body({css: maxCss})
.expect(HTTP_STATUS.CREATED)
.execute();
expect(theme.id).toBeDefined();
});
it('accepts CSS just under the limit', async () => {
const user = await createTestAccount(harness);
const nearMaxCss = 'a'.repeat(TEST_MAX_CSS_BYTES - 1);
const theme = await createBuilder<ThemeCreateResponse>(harness, user.token)
.post('/users/@me/themes')
.body({css: nearMaxCss})
@@ -52,12 +94,14 @@ describe('Theme CSS size limits', () => {
.execute();
expect(theme.id).toBeDefined();
});
it('rejects CSS exceeding limit with multibyte unicode characters', async () => {
it('rejects CSS whose byte length exceeds the limit even when the character count does not', async () => {
const user = await createTestAccount(harness);
const unicodeChar = '\u{1F600}';
const bytesPerChar = Buffer.from(unicodeChar, 'utf-8').length;
const charsNeeded = Math.ceil((MAX_CSS_BYTES + 1) / bytesPerChar);
const charsNeeded = Math.ceil((TEST_MAX_CSS_BYTES + 1) / bytesPerChar);
const oversizedUnicodeCss = unicodeChar.repeat(charsNeeded);
expect(oversizedUnicodeCss.length).toBeLessThanOrEqual(TEST_MAX_CSS_BYTES);
expect(Buffer.byteLength(oversizedUnicodeCss, 'utf-8')).toBeGreaterThan(TEST_MAX_CSS_BYTES);
await createBuilder(harness, user.token)
.post('/users/@me/themes')
.body({css: oversizedUnicodeCss})
+48 -20
View File
@@ -16,38 +16,44 @@ function parseParallelInteger(value: string | undefined, fallback: number): numb
}
function resolveDefaultParallelWorkers(): number {
const halfCoreCountMinusOne = Math.floor(availableParallelism() / 2) - 1;
return Math.max(2, halfCoreCountMinusOne);
return Math.max(2, availableParallelism() - 1);
}
const DEFAULT_PARALLEL_WORKERS = resolveDefaultParallelWorkers();
const configuredMaxWorkers = parseParallelInteger(process.env.API_TEST_MAX_WORKERS, DEFAULT_PARALLEL_WORKERS);
const configuredMaxConcurrency = parseParallelInteger(process.env.API_TEST_MAX_CONCURRENCY, configuredMaxWorkers);
const MODULE_REGISTRY_TEST_FILES = [
'src/api/gif/GifRequestCountry.test.ts',
'src/api/risk/__tests__/AccountPolicyService.test.ts',
];
const sharedExclude = [
...configDefaults.exclude,
'pkgs/**',
'../fluxer_desktop/**',
'**/target/**',
'**/*Integration.test.ts',
'**/*ExttestIntegration.test.ts',
];
const sharedTestConfig = {
globals: true,
environment: 'node' as const,
setupFiles: ['./src/api/test/Setup.ts'],
pool: 'threads' as const,
testTimeout: 40000,
hookTimeout: 20000,
maxConcurrency: configuredMaxConcurrency,
};
export default defineConfig({
root: process.cwd(),
plugins: [tsconfigPaths()],
cacheDir: './node_modules/.vitest',
test: {
globals: true,
environment: 'node',
setupFiles: ['./src/api/test/Setup.ts'],
include: ['src/**/*.{test,spec}.{ts,tsx}'],
exclude: [
...configDefaults.exclude,
'pkgs/**',
'../fluxer_desktop/**',
'**/target/**',
'**/*Integration.test.ts',
'**/*ExttestIntegration.test.ts',
],
pool: 'threads',
fileParallelism: true,
maxWorkers: configuredMaxWorkers,
maxConcurrency: configuredMaxConcurrency,
isolate: true,
testTimeout: 40000,
hookTimeout: 20000,
fileParallelism: true,
reporters: ['default', 'json'],
outputFile: './test-results.json',
coverage: {
@@ -63,5 +69,27 @@ export default defineConfig({
'**/TestHelpers.tsx',
],
},
projects: [
{
plugins: [tsconfigPaths()],
test: {
...sharedTestConfig,
name: 'api',
include: ['src/**/*.{test,spec}.{ts,tsx}'],
exclude: [...sharedExclude, ...MODULE_REGISTRY_TEST_FILES],
isolate: false,
},
},
{
plugins: [tsconfigPaths()],
test: {
...sharedTestConfig,
name: 'api-module-registry',
include: MODULE_REGISTRY_TEST_FILES,
exclude: sharedExclude,
isolate: true,
},
},
],
},
});