fix(api): add custom headers to email fallback SMTP (#3507)

deploy/self-hosting/.env.example

@@ -334,6 +334,9 @@ FLUXER_EMAIL_SMTP_TLS_MODE=
 #FLUXER_EMAIL_FALLBACK_SMTP_USERNAME=
 #FLUXER_EMAIL_FALLBACK_SMTP_PASSWORD=
 #FLUXER_EMAIL_FALLBACK_SMTP_TLS_MODE=
+# Extra headers on every fallback message, as a JSON object of header name to
+# string value. Use single quotes so the shell keeps the JSON intact.
+#FLUXER_EMAIL_FALLBACK_SMTP_HEADERS='{"X-Example-Tracking":"off"}'
 #FLUXER_EMAIL_WEBHOOK_SECRET=
 
 FLUXER_DISCOVERY_ENABLED=true

deploy/self-hosting/docker-compose.yml

@@ -132,6 +132,7 @@ x-fluxer-env: &fluxer-env
   FLUXER_EMAIL_FALLBACK_SMTP_USERNAME: ${FLUXER_EMAIL_FALLBACK_SMTP_USERNAME:-}
   FLUXER_EMAIL_FALLBACK_SMTP_PASSWORD: ${FLUXER_EMAIL_FALLBACK_SMTP_PASSWORD:-}
   FLUXER_EMAIL_FALLBACK_SMTP_TLS_MODE: ${FLUXER_EMAIL_FALLBACK_SMTP_TLS_MODE:-}
+  FLUXER_EMAIL_FALLBACK_SMTP_HEADERS: ${FLUXER_EMAIL_FALLBACK_SMTP_HEADERS:-}
 
   FLUXER_STRIPE_ENABLED: ${FLUXER_STRIPE_ENABLED:-}
   FLUXER_STRIPE_SECRET_KEY: ${FLUXER_STRIPE_SECRET_KEY:-}

fluxer_api/pkgs/email/src/SmtpEmailProvider.test.ts

@@ -25,4 +25,33 @@ describe('SmtpEmailProvider', () => {
 		new SmtpEmailProvider({host: 'smtp.example.com', port, username: 'user', password: 'pass', tlsMode});
 		expect(createTransport).toHaveBeenCalledWith(expect.objectContaining(expected));
 	});
+
+	const message = {
+		to: '[email protected]',
+		from: {email: '[email protected]', name: 'Example'},
+		subject: 'Hi',
+		text: 'Body',
+	};
+
+	it('sends the configured headers verbatim', async () => {
+		const provider = new SmtpEmailProvider({
+			host: 'smtp.example.com',
+			port: 587,
+			username: 'user',
+			password: 'pass',
+			headers: {'X-Tracking': '{"enable":0}'},
+		});
+		await provider.sendEmail(message);
+		const {sendMail} = createTransport.mock.results[0].value;
+		expect(sendMail).toHaveBeenCalledWith(
+			expect.objectContaining({headers: {'X-Tracking': {prepared: true, value: '{"enable":0}'}}}),
+		);
+	});
+
+	it('sends no extra headers when none are configured', async () => {
+		const provider = new SmtpEmailProvider({host: 'smtp.example.com', port: 587, username: 'user', password: 'pass'});
+		await provider.sendEmail(message);
+		const {sendMail} = createTransport.mock.results[0].value;
+		expect(sendMail.mock.calls[0][0]).not.toHaveProperty('headers');
+	});
 });

fluxer_api/pkgs/email/src/SmtpEmailProvider.ts

@@ -13,6 +13,7 @@ interface SmtpEmailConfig {
 	username: string;
 	password: string;
 	tlsMode?: SmtpTlsMode | null;
+	headers?: Record<string, string>;
 	connectionTimeoutMs?: number;
 	greetingTimeoutMs?: number;
 	socketTimeoutMs?: number;
@@ -23,11 +24,17 @@ export class SmtpEmailProvider implements IEmailProvider {
 	private readonly host: string;
 	private readonly port: number;
 	private readonly tlsMode: SmtpTlsMode;
+	private readonly headers: Record<string, {prepared: true; value: string}> | undefined;
 
 	constructor(config: SmtpEmailConfig) {
 		this.host = config.host;
 		this.port = config.port;
 		this.tlsMode = resolveSmtpTlsMode(config.tlsMode, config.port);
+		this.headers = config.headers
+			? Object.fromEntries(
+					Object.entries(config.headers).map(([name, value]) => [name, {prepared: true as const, value}]),
+				)
+			: undefined;
 		this.transporter = nodemailer.createTransport({
 			host: config.host,
 			port: config.port,
@@ -55,6 +62,7 @@ export class SmtpEmailProvider implements IEmailProvider {
 				to: message.to,
 				from: `${message.from.name} <${message.from.email}>`,
 				...(message.replyTo ? {replyTo: message.replyTo} : {}),
+				...(this.headers ? {headers: this.headers} : {}),
 				subject: message.subject,
 				text: message.text,
 			});

fluxer_api/src/api/Config.ts

@@ -307,7 +307,10 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
 			fallback: master.integrations.email.fallback
 				? {
 						domains: master.integrations.email.fallback.domains,
-						smtp: toSmtpConfig(master.integrations.email.fallback.smtp),
+						smtp: {
+							...toSmtpConfig(master.integrations.email.fallback.smtp),
+							headers: master.integrations.email.fallback.smtp.headers,
+						},
 					}
 				: undefined,
 		},

fluxer_api/src/api/config/APIConfig.ts

@@ -173,7 +173,7 @@ export interface APIConfig {
 		smtp?: SmtpConfig;
 		fallback?: {
 			domains: Array<string>;
-			smtp: SmtpConfig;
+			smtp: SmtpConfig & {headers?: Record<string, string>};
 		};
 	};
 	blocklistFeeds: {

fluxer_api/src/api/email/EmailProviderFactory.test.ts

@@ -4,7 +4,8 @@ import type {APIConfig} from '@app/api/config/APIConfig';
 import {createEmailProvider} from '@app/api/email/EmailProviderFactory';
 import {DomainRoutedEmailProvider} from '@pkgs/email/src/DomainRoutedEmailProvider';
 import {SmtpEmailProvider} from '@pkgs/email/src/SmtpEmailProvider';
-import {describe, expect, it} from 'vitest';
+import nodemailer, {type Transporter} from 'nodemailer';
+import {describe, expect, it, vi} from 'vitest';
 
 const smtp = {host: 'smtp.example.com', port: 587, username: 'user', password: 'pass', tlsMode: null};
 
@@ -33,6 +34,18 @@ describe('createEmailProvider', () => {
 		expect(provider).toBeInstanceOf(DomainRoutedEmailProvider);
 	});
 
+	it('adds the fallback headers only to mail through the fallback', async () => {
+		const sendMail = vi.fn();
+		vi.spyOn(nodemailer, 'createTransport').mockReturnValue({sendMail} as unknown as Transporter);
+		const headers = {'X-Tracking': '{"enable":0}'};
+		const provider = createEmailProvider(emailConfig({fallback: {domains: ['gmail.com'], smtp: {...smtp, headers}}}));
+		const from = {email: '[email protected]', name: 'Example'};
+		await provider?.sendEmail({to: '[email protected]', from, subject: 'Hi', text: 'Body'});
+		await provider?.sendEmail({to: '[email protected]', from, subject: 'Hi', text: 'Body'});
+		expect(sendMail.mock.calls[0][0].headers).toEqual({'X-Tracking': {prepared: true, value: '{"enable":0}'}});
+		expect(sendMail.mock.calls[1][0]).not.toHaveProperty('headers');
+	});
+
 	it.each([{enabled: false}, {provider: 'none' as const}, {smtp: undefined}])(
 		'returns no provider for %j even with a fallback',
 		(overrides) => {

fluxer_docs/src/content/docs/operator/configuration.mdx

@@ -865,6 +865,10 @@ Default empty. The fallback SMTP password. Required when the domains are set.
 
 Default empty. How the fallback connection is encrypted, with the same four values as `FLUXER_EMAIL_SMTP_TLS_MODE`. An empty value picks `implicit` on port 465 and `starttls` on every other port. A failed send through the fallback is logged as `SMTP send failed` with the fallback host.
 
+#### `FLUXER_EMAIL_FALLBACK_SMTP_HEADERS`
+
+Default empty. Extra headers added to every message sent through the fallback provider, as a JSON object of header name to string value, such as `{"X-Example-Tracking":"off"}` for a provider that reads a header to turn off link tracking. Values are sent exactly as given, never encoded or folded. A name must be a valid header field name, a value must be printable ASCII without line breaks, and each header has to fit on one 998 character line. `From`, `To`, `Cc`, `Bcc`, `Subject`, `Reply-To`, `Message-ID`, `Date`, `Content-Type` and `MIME-Version` cannot be set here. Anything else fails startup with the reason. `.env.example` lists it commented out.
+
 Only `api` and `worker` build the mail service, so recreating those two is enough after a change.
 
 ## CAPTCHA

packages/config/src/ConfigLoader.ts

@@ -330,6 +330,41 @@ function validateReplyToEmail(value: string): void {
 const EMAIL_DOMAIN_PATTERN =
 	/^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/u;
 const DEFAULT_SMTP_PORT = 587;
+const SMTP_HEADER_NAME_PATTERN = /^[\x21-\x39\x3b-\x7e]+$/u;
+const SMTP_HEADER_VALUE_PATTERN = /^[\x20-\x7e\t]*$/u;
+const SMTP_HEADER_MAX_LINE_LENGTH = 998;
+const CORE_SMTP_HEADER_NAMES = new Set([
+	'from',
+	'to',
+	'cc',
+	'bcc',
+	'subject',
+	'reply-to',
+	'message-id',
+	'date',
+	'content-type',
+	'mime-version',
+]);
+
+function validateSmtpHeaders(headers: unknown, envName: string): void {
+	if (headers === undefined) return;
+	if (!isConfigObject(headers)) throw new Error(`${envName} must be a JSON object of header names to strings`);
+	for (const [name, value] of Object.entries(headers)) {
+		if (!SMTP_HEADER_NAME_PATTERN.test(name)) {
+			throw new Error(`${envName} header name ${JSON.stringify(name)} is not a valid header field name`);
+		}
+		if (CORE_SMTP_HEADER_NAMES.has(name.toLowerCase())) {
+			throw new Error(`${envName} must not set the ${name} header`);
+		}
+		if (typeof value !== 'string') throw new Error(`${envName} value of ${name} must be a string`);
+		if (!SMTP_HEADER_VALUE_PATTERN.test(value)) {
+			throw new Error(`${envName} value of ${name} must be printable ASCII without line breaks`);
+		}
+		if (name.length + 2 + value.length > SMTP_HEADER_MAX_LINE_LENGTH) {
+			throw new Error(`${envName} header ${name} must fit on one ${SMTP_HEADER_MAX_LINE_LENGTH} character line`);
+		}
+	}
+}
 
 function normalizeEmailFallbackConfig(config: MasterConfig): void {
 	const email = config.integrations.email;
@@ -342,6 +377,7 @@ function normalizeEmailFallbackConfig(config: MasterConfig): void {
 	if (!smtp.password?.trim()) throw missing('FLUXER_EMAIL_FALLBACK_SMTP_PASSWORD');
 	const port: unknown = smtp.port ?? DEFAULT_SMTP_PORT;
 	assertIntegerInRange(port, 'FLUXER_EMAIL_FALLBACK_SMTP_PORT', 1, 65_535);
+	validateSmtpHeaders(smtp.headers, 'FLUXER_EMAIL_FALLBACK_SMTP_HEADERS');
 	const normalizedDomains = domains.map((domain) => domain.trim().toLowerCase());
 	for (const domain of normalizedDomains) {
 		if (!EMAIL_DOMAIN_PATTERN.test(domain)) {

packages/config/src/MasterConfig.ts

@@ -197,7 +197,7 @@ export interface MasterConfig {
 			smtp?: SmtpConfig;
 			fallback?: {
 				domains: Array<string>;
-				smtp: SmtpConfig;
+				smtp: SmtpConfig & {headers?: Record<string, string>};
 			};
 		};
 		voice: {

packages/config/src/__tests__/ConfigLoader.test.ts

@@ -540,6 +540,30 @@ describe('ConfigLoader', () => {
 		},
 	);
 
+	test('loads the email fallback headers', async () => {
+		stubMinimalEnv({...emailFallbackEnv, FLUXER_EMAIL_FALLBACK_SMTP_HEADERS: '{"X-Tracking":"{\\"enable\\":0}"}'});
+		const config = await loadConfig();
+		expect(config.integrations.email.fallback?.smtp.headers).toEqual({'X-Tracking': '{"enable":0}'});
+	});
+
+	test.each([
+		{headers: '["X-Tracking"]', error: 'must be a JSON object'},
+		{headers: '{"X-Tracking":{"enable":0}}', error: 'value of X-Tracking must be a string'},
+		{headers: '{"X-Tracking":0}', error: 'value of X-Tracking must be a string'},
+		{headers: '{"X Tracking":"off"}', error: 'header name "X Tracking" is not a valid header field name'},
+		{headers: '{"X-Tracking:":"off"}', error: 'header name "X-Tracking:" is not a valid header field name'},
+		{headers: '{"X-Tracking\\r\\nBcc":"off"}', error: 'is not a valid header field name'},
+		{headers: '{"X-Tracking":"off\\r\\nBcc: [email protected]"}', error: 'must be printable ASCII without line breaks'},
+		{headers: '{"X-Tracking":"off\\n"}', error: 'must be printable ASCII without line breaks'},
+		{headers: `{"X-Tracking":"${'a'.repeat(990)}"}`, error: 'must fit on one 998 character line'},
+		...['From', 'to', 'Cc', 'BCC', 'Subject', 'Reply-To', 'Message-ID', 'Date', 'content-type', 'MIME-Version'].map(
+			(name) => ({headers: JSON.stringify({[name]: 'x'}), error: `must not set the ${name} header`}),
+		),
+	])('rejects the email fallback headers $headers', async ({headers, error}) => {
+		stubMinimalEnv({...emailFallbackEnv, FLUXER_EMAIL_FALLBACK_SMTP_HEADERS: headers});
+		await expect(loadConfig()).rejects.toThrow(error);
+	});
+
 	test('rejects an out of range email fallback port', async () => {
 		stubMinimalEnv({...emailFallbackEnv, FLUXER_EMAIL_FALLBACK_SMTP_PORT: '70000'});
 		await expect(loadConfig()).rejects.toThrow(

packages/config/src/__tests__/EnvironmentOverrides.test.ts

@@ -48,6 +48,7 @@ describe('buildNamedFluxerEnvOverrides', () => {
 				FLUXER_EMAIL_REPLY_TO_EMAIL: blank,
 				FLUXER_EMAIL_FALLBACK_DOMAINS: blank,
 				FLUXER_EMAIL_FALLBACK_SMTP_HOST: blank,
+				FLUXER_EMAIL_FALLBACK_SMTP_HEADERS: blank,
 			}),
 		).toEqual({});
 	});
@@ -61,6 +62,7 @@ describe('buildNamedFluxerEnvOverrides', () => {
 				FLUXER_EMAIL_FALLBACK_SMTP_USERNAME: 'apikey',
 				FLUXER_EMAIL_FALLBACK_SMTP_PASSWORD: 'secret',
 				FLUXER_EMAIL_FALLBACK_SMTP_TLS_MODE: 'STARTTLS',
+				FLUXER_EMAIL_FALLBACK_SMTP_HEADERS: '{"X-Tracking":"off"}',
 			}),
 		).toEqual({
 			integrations: {
@@ -73,6 +75,7 @@ describe('buildNamedFluxerEnvOverrides', () => {
 							username: 'apikey',
 							password: 'secret',
 							tls_mode: 'starttls',
+							headers: {'X-Tracking': 'off'},
 						},
 					},
 				},

packages/config/src/config_loader/EnvironmentOverrides.ts

@@ -174,6 +174,10 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {
 		path: ['integrations', 'email', 'fallback', 'smtp', 'tls_mode'],
 		parse: parseSmtpTlsMode,
 	},
+	FLUXER_EMAIL_FALLBACK_SMTP_HEADERS: {
+		path: ['integrations', 'email', 'fallback', 'smtp', 'headers'],
+		parse: parseJsonObject,
+	},
 	FLUXER_LIVEKIT_ENABLED: {path: ['integrations', 'voice', 'enabled'], parse: parseBoolean},
 	FLUXER_LIVEKIT_API_KEY: {path: ['integrations', 'voice', 'api_key']},
 	FLUXER_LIVEKIT_API_SECRET: {path: ['integrations', 'voice', 'api_secret']},