fix(api): resolve s3 copy source under endpoint path (#3422)

fluxer_api/src/api/infrastructure/StorageService.test.ts

@@ -225,3 +225,55 @@ describe('StorageService.listObjects', () => {
 		);
 	});
 });
+
+function copySourceFor(endpoint: string, forcePathStyle: boolean): Promise<unknown> {
+	const service = new StorageService({
+		endpoint,
+		forcePathStyle,
+		region: 'us-east-1',
+		accessKeyId: 'fluxer',
+		secretAccessKey: 'fluxer-secret',
+	});
+	let copySource: unknown;
+	Object.assign(service, {
+		client: {
+			send: async (command: {input: {CopySource?: unknown}}) => {
+				copySource = command.input.CopySource;
+				return {};
+			},
+		},
+	});
+	return service
+		.copyObject({
+			sourceBucket: 'fluxer-uploads',
+			sourceKey: 'a b/c+d.pdf',
+			destinationBucket: 'fluxer-cdn',
+			destinationKey: 'attachments/1/2/c.pdf',
+		})
+		.then(() => copySource);
+}
+
+describe('StorageService.copyObject copy source', () => {
+	it.each([
+		['http://seaweedfs:8333', true],
+		['http://seaweedfs:8333/', true],
+		['https://s3.example.test', false],
+	])(
+		'names only the bucket and key for an endpoint without a path (%s, path style %s)',
+		async (endpoint, pathStyle) => {
+			await expect(copySourceFor(endpoint, pathStyle)).resolves.toBe('fluxer-uploads/a%20b/c%2Bd.pdf');
+		},
+	);
+
+	it('puts the endpoint path in front of the bucket with path-style addressing', async () => {
+		await expect(copySourceFor('https://s3.example.test/real%20bucket/sub/', true)).resolves.toBe(
+			'real%20bucket/sub/fluxer-uploads/a%20b/c%2Bd.pdf',
+		);
+	});
+
+	it('puts the endpoint path between the bucket and the key with virtual-hosted addressing', async () => {
+		await expect(copySourceFor('https://s3.example.test/real%20bucket/sub', false)).resolves.toBe(
+			'fluxer-uploads/real%20bucket/sub/a%20b/c%2Bd.pdf',
+		);
+	});
+});

fluxer_api/src/api/infrastructure/StorageService.ts

@@ -115,6 +115,13 @@ const REJECTED_SERVER_SIDE_COPY_ERRORS = new Set([
 	'MethodNotAllowed',
 ]);
 
+function encodedEndpointPath(endpoint: string): Array<string> {
+	return (URL.parse(endpoint)?.pathname ?? '')
+		.split('/')
+		.filter((segment) => segment !== '')
+		.map((segment) => encodeURIComponent(decodeURIComponent(segment)));
+}
+
 function isRejectedServerSideCopy(error: unknown): boolean {
 	return (
 		error instanceof S3ServiceException &&
@@ -126,6 +133,7 @@ export class StorageService implements IStorageService {
 	private readonly client: S3Client;
 	private readonly presignClient: S3Client;
 	private readonly provider: S3ProviderSettings;
+	private readonly endpointPath: Array<string>;
 
 	constructor(provider?: S3ProviderSettings) {
 		this.provider = provider ?? {
@@ -136,6 +144,7 @@ export class StorageService implements IStorageService {
 			accessKeyId: Config.s3.accessKeyId,
 			secretAccessKey: Config.s3.secretAccessKey,
 		};
+		this.endpointPath = encodedEndpointPath(this.provider.endpoint);
 		this.client = buildPooledS3Client({
 			endpoint: this.provider.endpoint,
 			region: this.provider.region,
@@ -492,7 +501,7 @@ export class StorageService implements IStorageService {
 				new CopyObjectCommand({
 					Bucket: destinationBucket,
 					Key: destinationKey,
-					CopySource: `${encodeURIComponent(sourceBucket)}/${sourceKey.split('/').map(encodeURIComponent).join('/')}`,
+					CopySource: this.copySource(sourceBucket, sourceKey),
 					ContentType: newContentType,
 					MetadataDirective: newContentType ? 'REPLACE' : undefined,
 				}),
@@ -508,6 +517,14 @@ export class StorageService implements IStorageService {
 		}
 	}
 
+	private copySource(bucket: string, key: string): string {
+		const encodedBucket = encodeURIComponent(bucket);
+		const location = this.provider.forcePathStyle
+			? [...this.endpointPath, encodedBucket]
+			: [encodedBucket, ...this.endpointPath];
+		return [...location, ...key.split('/').map(encodeURIComponent)].join('/');
+	}
+
 	private async copyObjectThroughApi(
 		{
 			sourceBucket,

fluxer_api/src/api/infrastructure/StorageServiceEndpointPath.test.ts

@@ -0,0 +1,129 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {StorageService} from '@app/api/infrastructure/StorageService';
+import {server} from '@app/api/test/msw/server';
+import {HttpResponse, http} from 'msw';
+import {beforeEach, describe, expect, it} from 'vitest';
+
+const HOST = 's3.single-bucket.test';
+const REAL_BUCKET = 'realbucket';
+
+function errorResponse(code: string, status: number): HttpResponse<string> {
+	return new HttpResponse(
+		`<?xml version="1.0" encoding="UTF-8"?><Error><Code>${code}</Code><Message></Message><RequestId>tx0</RequestId></Error>`,
+		{status, headers: {'Content-Type': 'application/xml'}},
+	);
+}
+
+function splitBucket(path: string): {bucket: string; key: string} {
+	const [bucket = '', ...key] = path.split('/');
+	return {bucket, key: key.join('/')};
+}
+
+function singleBucketStore() {
+	const objects = new Map<string, {body: Uint8Array; contentType: string}>();
+	const resolve = (url: URL): {bucket: string; key: string} => {
+		const path = decodeURIComponent(url.pathname.slice(1));
+		if (url.hostname === HOST) {
+			return splitBucket(path);
+		}
+		return {bucket: url.hostname.slice(0, -HOST.length - 1), key: path};
+	};
+	server.use(
+		http.all(/^https:\/\/([a-z0-9-]+\.)?s3\.single-bucket\.test\//u, async ({request}) => {
+			const target = resolve(new URL(request.url));
+			if (target.bucket !== REAL_BUCKET) {
+				return errorResponse('NoSuchBucket', 404);
+			}
+			if (request.method === 'GET') {
+				const object = objects.get(target.key);
+				if (!object) {
+					return errorResponse('NoSuchKey', 404);
+				}
+				return new HttpResponse(object.body, {
+					status: 200,
+					headers: {'Content-Type': object.contentType, 'Content-Length': String(object.body.length)},
+				});
+			}
+			if (request.method !== 'PUT') {
+				return errorResponse('NotImplemented', 501);
+			}
+			const copySource = request.headers.get('x-amz-copy-source');
+			if (!copySource) {
+				objects.set(target.key, {
+					body: new Uint8Array(await request.arrayBuffer()),
+					contentType: request.headers.get('content-type') ?? 'application/octet-stream',
+				});
+				return new HttpResponse(null, {status: 200, headers: {ETag: '"x"'}});
+			}
+			const source = splitBucket(decodeURIComponent(copySource.replace(/^\//u, '')));
+			if (source.bucket !== REAL_BUCKET) {
+				return errorResponse('NoSuchBucket', 404);
+			}
+			const object = objects.get(source.key);
+			if (!object) {
+				return errorResponse('NoSuchKey', 404);
+			}
+			objects.set(target.key, {...object, contentType: request.headers.get('content-type') ?? object.contentType});
+			return new HttpResponse(
+				'<?xml version="1.0" encoding="UTF-8"?><CopyObjectResult><ETag>"x"</ETag></CopyObjectResult>',
+				{status: 200, headers: {'Content-Type': 'application/xml'}},
+			);
+		}),
+	);
+	return objects;
+}
+
+function storage(endpoint: string, forcePathStyle: boolean): StorageService {
+	return new StorageService({
+		endpoint,
+		forcePathStyle,
+		region: 'us-east-1',
+		accessKeyId: 'TEST',
+		secretAccessKey: 'TEST',
+	});
+}
+
+const PDF = new TextEncoder().encode('%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\n%%EOF\n');
+
+describe('StorageService with every bucket inside one real bucket', () => {
+	let objects: ReturnType<typeof singleBucketStore>;
+
+	beforeEach(() => {
+		objects = singleBucketStore();
+	});
+
+	it('copies a non-media attachment when the endpoint names the bucket with path-style addressing', async () => {
+		const service = storage(`https://${HOST}/${REAL_BUCKET}`, true);
+		await service.uploadObject({bucket: 'fluxer-uploads', key: 'upload 1', body: PDF});
+
+		await expect(
+			service.copyObjectWithMetadataStripping({
+				sourceBucket: 'fluxer-uploads',
+				sourceKey: 'upload 1',
+				destinationBucket: 'fluxer-cdn',
+				destinationKey: 'attachments/1/2/file.pdf',
+				contentType: 'application/pdf',
+			}),
+		).resolves.toBeNull();
+
+		expect(objects.get('fluxer-cdn/attachments/1/2/file.pdf')?.contentType).toBe('application/pdf');
+		await expect(service.readObject('fluxer-cdn', 'attachments/1/2/file.pdf')).resolves.toEqual(PDF);
+	});
+
+	it('copies within a bucket whose virtual host keeps the endpoint path as a key prefix', async () => {
+		const service = storage(`https://${HOST}/prefix`, false);
+		await service.uploadObject({bucket: REAL_BUCKET, key: 'upload-2', body: PDF});
+
+		await service.copyObject({
+			sourceBucket: REAL_BUCKET,
+			sourceKey: 'upload-2',
+			destinationBucket: REAL_BUCKET,
+			destinationKey: 'attachments/1/3/file.pdf',
+			newContentType: 'application/pdf',
+		});
+
+		expect(objects.get('prefix/attachments/1/3/file.pdf')?.contentType).toBe('application/pdf');
+		await expect(service.readObject(REAL_BUCKET, 'attachments/1/3/file.pdf')).resolves.toEqual(PDF);
+	});
+});