From 8bda15159674d2a95560c85408ac3fcbd7fecab3 Mon Sep 17 00:00:00 2001 From: Hampus Date: Mon, 12 Oct 2026 16:02:52 +0200 Subject: [PATCH] fix(api): resolve s3 copy source under endpoint path (#3422) --- .../api/infrastructure/StorageService.test.ts | 52 +++++++ .../src/api/infrastructure/StorageService.ts | 19 ++- .../StorageServiceEndpointPath.test.ts | 129 ++++++++++++++++++ .../content/docs/operator/configuration.mdx | 2 + 4 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 fluxer_api/src/api/infrastructure/StorageServiceEndpointPath.test.ts diff --git a/fluxer_api/src/api/infrastructure/StorageService.test.ts b/fluxer_api/src/api/infrastructure/StorageService.test.ts index e4280f013..0d60e16f6 100644 --- a/fluxer_api/src/api/infrastructure/StorageService.test.ts +++ b/fluxer_api/src/api/infrastructure/StorageService.test.ts @@ -225,3 +225,55 @@ describe('StorageService.listObjects', () => { ); }); }); + +function copySourceFor(endpoint: string, forcePathStyle: boolean): Promise { + const service = new StorageService({ + endpoint, + forcePathStyle, + region: 'us-east-1', + accessKeyId: 'fluxer', + secretAccessKey: 'fluxer-secret', + }); + let copySource: unknown; + Object.assign(service, { + client: { + send: async (command: {input: {CopySource?: unknown}}) => { + copySource = command.input.CopySource; + return {}; + }, + }, + }); + return service + .copyObject({ + sourceBucket: 'fluxer-uploads', + sourceKey: 'a b/c+d.pdf', + destinationBucket: 'fluxer-cdn', + destinationKey: 'attachments/1/2/c.pdf', + }) + .then(() => copySource); +} + +describe('StorageService.copyObject copy source', () => { + it.each([ + ['http://seaweedfs:8333', true], + ['http://seaweedfs:8333/', true], + ['https://s3.example.test', false], + ])( + 'names only the bucket and key for an endpoint without a path (%s, path style %s)', + async (endpoint, pathStyle) => { + await expect(copySourceFor(endpoint, pathStyle)).resolves.toBe('fluxer-uploads/a%20b/c%2Bd.pdf'); + }, + ); + + it('puts the endpoint path in front of the bucket with path-style addressing', async () => { + await expect(copySourceFor('https://s3.example.test/real%20bucket/sub/', true)).resolves.toBe( + 'real%20bucket/sub/fluxer-uploads/a%20b/c%2Bd.pdf', + ); + }); + + it('puts the endpoint path between the bucket and the key with virtual-hosted addressing', async () => { + await expect(copySourceFor('https://s3.example.test/real%20bucket/sub', false)).resolves.toBe( + 'fluxer-uploads/real%20bucket/sub/a%20b/c%2Bd.pdf', + ); + }); +}); diff --git a/fluxer_api/src/api/infrastructure/StorageService.ts b/fluxer_api/src/api/infrastructure/StorageService.ts index d1b3434d7..2c94cfd1f 100644 --- a/fluxer_api/src/api/infrastructure/StorageService.ts +++ b/fluxer_api/src/api/infrastructure/StorageService.ts @@ -115,6 +115,13 @@ const REJECTED_SERVER_SIDE_COPY_ERRORS = new Set([ 'MethodNotAllowed', ]); +function encodedEndpointPath(endpoint: string): Array { + return (URL.parse(endpoint)?.pathname ?? '') + .split('/') + .filter((segment) => segment !== '') + .map((segment) => encodeURIComponent(decodeURIComponent(segment))); +} + function isRejectedServerSideCopy(error: unknown): boolean { return ( error instanceof S3ServiceException && @@ -126,6 +133,7 @@ export class StorageService implements IStorageService { private readonly client: S3Client; private readonly presignClient: S3Client; private readonly provider: S3ProviderSettings; + private readonly endpointPath: Array; constructor(provider?: S3ProviderSettings) { this.provider = provider ?? { @@ -136,6 +144,7 @@ export class StorageService implements IStorageService { accessKeyId: Config.s3.accessKeyId, secretAccessKey: Config.s3.secretAccessKey, }; + this.endpointPath = encodedEndpointPath(this.provider.endpoint); this.client = buildPooledS3Client({ endpoint: this.provider.endpoint, region: this.provider.region, @@ -492,7 +501,7 @@ export class StorageService implements IStorageService { new CopyObjectCommand({ Bucket: destinationBucket, Key: destinationKey, - CopySource: `${encodeURIComponent(sourceBucket)}/${sourceKey.split('/').map(encodeURIComponent).join('/')}`, + CopySource: this.copySource(sourceBucket, sourceKey), ContentType: newContentType, MetadataDirective: newContentType ? 'REPLACE' : undefined, }), @@ -508,6 +517,14 @@ export class StorageService implements IStorageService { } } + private copySource(bucket: string, key: string): string { + const encodedBucket = encodeURIComponent(bucket); + const location = this.provider.forcePathStyle + ? [...this.endpointPath, encodedBucket] + : [encodedBucket, ...this.endpointPath]; + return [...location, ...key.split('/').map(encodeURIComponent)].join('/'); + } + private async copyObjectThroughApi( { sourceBucket, diff --git a/fluxer_api/src/api/infrastructure/StorageServiceEndpointPath.test.ts b/fluxer_api/src/api/infrastructure/StorageServiceEndpointPath.test.ts new file mode 100644 index 000000000..0749db71b --- /dev/null +++ b/fluxer_api/src/api/infrastructure/StorageServiceEndpointPath.test.ts @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {StorageService} from '@app/api/infrastructure/StorageService'; +import {server} from '@app/api/test/msw/server'; +import {HttpResponse, http} from 'msw'; +import {beforeEach, describe, expect, it} from 'vitest'; + +const HOST = 's3.single-bucket.test'; +const REAL_BUCKET = 'realbucket'; + +function errorResponse(code: string, status: number): HttpResponse { + return new HttpResponse( + `${code}tx0`, + {status, headers: {'Content-Type': 'application/xml'}}, + ); +} + +function splitBucket(path: string): {bucket: string; key: string} { + const [bucket = '', ...key] = path.split('/'); + return {bucket, key: key.join('/')}; +} + +function singleBucketStore() { + const objects = new Map(); + const resolve = (url: URL): {bucket: string; key: string} => { + const path = decodeURIComponent(url.pathname.slice(1)); + if (url.hostname === HOST) { + return splitBucket(path); + } + return {bucket: url.hostname.slice(0, -HOST.length - 1), key: path}; + }; + server.use( + http.all(/^https:\/\/([a-z0-9-]+\.)?s3\.single-bucket\.test\//u, async ({request}) => { + const target = resolve(new URL(request.url)); + if (target.bucket !== REAL_BUCKET) { + return errorResponse('NoSuchBucket', 404); + } + if (request.method === 'GET') { + const object = objects.get(target.key); + if (!object) { + return errorResponse('NoSuchKey', 404); + } + return new HttpResponse(object.body, { + status: 200, + headers: {'Content-Type': object.contentType, 'Content-Length': String(object.body.length)}, + }); + } + if (request.method !== 'PUT') { + return errorResponse('NotImplemented', 501); + } + const copySource = request.headers.get('x-amz-copy-source'); + if (!copySource) { + objects.set(target.key, { + body: new Uint8Array(await request.arrayBuffer()), + contentType: request.headers.get('content-type') ?? 'application/octet-stream', + }); + return new HttpResponse(null, {status: 200, headers: {ETag: '"x"'}}); + } + const source = splitBucket(decodeURIComponent(copySource.replace(/^\//u, ''))); + if (source.bucket !== REAL_BUCKET) { + return errorResponse('NoSuchBucket', 404); + } + const object = objects.get(source.key); + if (!object) { + return errorResponse('NoSuchKey', 404); + } + objects.set(target.key, {...object, contentType: request.headers.get('content-type') ?? object.contentType}); + return new HttpResponse( + '"x"', + {status: 200, headers: {'Content-Type': 'application/xml'}}, + ); + }), + ); + return objects; +} + +function storage(endpoint: string, forcePathStyle: boolean): StorageService { + return new StorageService({ + endpoint, + forcePathStyle, + region: 'us-east-1', + accessKeyId: 'TEST', + secretAccessKey: 'TEST', + }); +} + +const PDF = new TextEncoder().encode('%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\n%%EOF\n'); + +describe('StorageService with every bucket inside one real bucket', () => { + let objects: ReturnType; + + beforeEach(() => { + objects = singleBucketStore(); + }); + + it('copies a non-media attachment when the endpoint names the bucket with path-style addressing', async () => { + const service = storage(`https://${HOST}/${REAL_BUCKET}`, true); + await service.uploadObject({bucket: 'fluxer-uploads', key: 'upload 1', body: PDF}); + + await expect( + service.copyObjectWithMetadataStripping({ + sourceBucket: 'fluxer-uploads', + sourceKey: 'upload 1', + destinationBucket: 'fluxer-cdn', + destinationKey: 'attachments/1/2/file.pdf', + contentType: 'application/pdf', + }), + ).resolves.toBeNull(); + + expect(objects.get('fluxer-cdn/attachments/1/2/file.pdf')?.contentType).toBe('application/pdf'); + await expect(service.readObject('fluxer-cdn', 'attachments/1/2/file.pdf')).resolves.toEqual(PDF); + }); + + it('copies within a bucket whose virtual host keeps the endpoint path as a key prefix', async () => { + const service = storage(`https://${HOST}/prefix`, false); + await service.uploadObject({bucket: REAL_BUCKET, key: 'upload-2', body: PDF}); + + await service.copyObject({ + sourceBucket: REAL_BUCKET, + sourceKey: 'upload-2', + destinationBucket: REAL_BUCKET, + destinationKey: 'attachments/1/3/file.pdf', + newContentType: 'application/pdf', + }); + + expect(objects.get('prefix/attachments/1/3/file.pdf')?.contentType).toBe('application/pdf'); + await expect(service.readObject(REAL_BUCKET, 'attachments/1/3/file.pdf')).resolves.toEqual(PDF); + }); +}); diff --git a/fluxer_docs/src/content/docs/operator/configuration.mdx b/fluxer_docs/src/content/docs/operator/configuration.mdx index dde6b8baf..cf414dab4 100644 --- a/fluxer_docs/src/content/docs/operator/configuration.mdx +++ b/fluxer_docs/src/content/docs/operator/configuration.mdx @@ -514,6 +514,8 @@ Every uploaded file lands in an S3-compatible object store, which the stack prov Default `http://localhost:3900`. The S3 API address. `media-proxy` defaults to empty instead. Compose forwards it from `.env` with a default of `http://seaweedfs:8333`, the bundled store. +An endpoint with a bucket in its path, such as `https://s3.example.com/my-bucket` with `FLUXER_S3_FORCE_PATH_STYLE=true`, keeps every Fluxer bucket inside that one bucket. Each bucket name becomes a key prefix there, so uploads land under `fluxer-uploads/` in `my-bucket`. + #### `FLUXER_S3_PUBLIC_ENDPOINT` No default. The host substituted into presigned URLs, which are the only place it appears. Compose forwards it from `.env`, and uses `FLUXER_S3_ENDPOINT` when it is unset, except on `app-proxy`, which gets only a value `.env` sets and adds it to its Content Security Policy. `admin` reads it to find the origin that report downloads are presigned against. See `FLUXER_S3_BUCKET_REPORTS`.