fix(api): bound storage listings, ranges and search paging (#2344)

fluxer_api/src/api/infrastructure/StorageService.ts

@@ -32,7 +32,12 @@ import {seconds} from 'itty-time';
 import {temporaryFile} from 'tempy';
 import {Config} from '../Config';
 import {Logger} from '../Logger';
-import type {IStorageService, ProcessedStorageObjectMetadata} from './IStorageService';
+import {
+	type IStorageService,
+	type ProcessedStorageObjectMetadata,
+	StorageObjectListingOverflowError,
+	StorageObjectRangeNotSatisfiableError,
+} from './IStorageService';
 import {processMediaFile} from './StorageObjectHelpers';
 
 const STREAM_UPLOAD_PART_BYTES = 8 * 1024 * 1024;
@@ -73,18 +78,13 @@ async function streamToUint8Array(body: Readable, maxBytes?: number): Promise<Ui
 	for await (const chunk of body) {
 		const buf = chunk instanceof Buffer ? chunk : Buffer.from(chunk as Uint8Array);
 		if (maxBytes !== undefined && total + buf.length > maxBytes) {
-			const remaining = maxBytes - total;
-			if (remaining > 0) {
-				chunks.push(buf.subarray(0, remaining));
-				total += remaining;
-			}
-			break;
+			body.destroy();
+			throw new Error(
+				`Stream exceeds maximum buffer size of ${maxBytes} bytes (got at least ${total + buf.length} bytes)`,
+			);
 		}
 		chunks.push(buf);
 		total += buf.length;
-		if (maxBytes !== undefined && total >= maxBytes) {
-			break;
-		}
 	}
 	const out = new Uint8Array(total);
 	let offset = 0;
@@ -97,10 +97,15 @@ async function streamToUint8Array(body: Readable, maxBytes?: number): Promise<Ui
 
 function extractStreamFromGet(out: GetObjectCommandOutput): Readable {
 	const body = out.Body;
-	if (body instanceof Readable) {
-		return body instanceof PassThrough ? body : body.pipe(new PassThrough());
+	if (!(body instanceof Readable)) {
+		throw new Error('Unexpected S3 response body type (not a Node Readable)');
 	}
-	throw new Error('Unexpected S3 response body type (not a Node Readable)');
+	if (body instanceof PassThrough) {
+		return body;
+	}
+	const wrapped = new PassThrough();
+	pipeline(body, wrapped, () => undefined);
+	return wrapped;
 }
 
 export class StorageService implements IStorageService {
@@ -362,7 +367,12 @@ export class StorageService implements IStorageService {
 
 	async readObject(bucket: string, key: string, maxBytes?: number): Promise<Uint8Array> {
 		const out = await this.client.send(new GetObjectCommand({Bucket: bucket, Key: key}));
-		return streamToUint8Array(extractStreamFromGet(out), maxBytes);
+		const body = extractStreamFromGet(out);
+		if (maxBytes !== undefined && out.ContentLength !== undefined && out.ContentLength > maxBytes) {
+			body.destroy();
+			throw new Error(`Stream exceeds maximum buffer size of ${maxBytes} bytes (got ${out.ContentLength} bytes)`);
+		}
+		return streamToUint8Array(body, maxBytes);
 	}
 
 	async streamObject(params: {bucket: string; key: string; range?: string}): Promise<{
@@ -395,6 +405,13 @@ export class StorageService implements IStorageService {
 			if (error instanceof S3ServiceException && (error.name === 'NoSuchKey' || error.name === 'NotFound')) {
 				return null;
 			}
+			if (
+				params.range !== undefined &&
+				error instanceof S3ServiceException &&
+				(error.name === 'InvalidRange' || error.$metadata?.httpStatusCode === 416)
+			) {
+				throw new StorageObjectRangeNotSatisfiableError(params.bucket, params.key, params.range);
+			}
 			throw error;
 		}
 	}
@@ -563,31 +580,49 @@ export class StorageService implements IStorageService {
 		return this.client.send(new HeadObjectCommand({Bucket: params.bucket, Key: params.key}));
 	}
 
-	async listObjects(params: {bucket: string; prefix: string}): Promise<
+	async listObjects(params: {bucket: string; prefix: string; maxObjects?: number}): Promise<
 		ReadonlyArray<{
 			key: string;
 			lastModified?: Date;
 		}>
 	> {
+		if (params.maxObjects !== undefined && (!Number.isSafeInteger(params.maxObjects) || params.maxObjects <= 0)) {
+			throw new RangeError('maxObjects must be a positive safe integer');
+		}
 		const result: Array<{
 			key: string;
 			lastModified?: Date;
 		}> = [];
+		let listedObjects = 0;
 		let continuationToken: string | undefined;
 		do {
+			const remaining = params.maxObjects === undefined ? undefined : params.maxObjects - listedObjects;
 			const command = new ListObjectsV2Command({
 				Bucket: params.bucket,
 				Prefix: params.prefix,
 				ContinuationToken: continuationToken,
+				MaxKeys: remaining === undefined ? undefined : Math.min(remaining, 1000),
 			});
 			const response = await this.client.send(command);
 			if (response.Contents) {
 				for (const obj of response.Contents) {
+					listedObjects += 1;
 					if (obj.Key) {
 						result.push({key: obj.Key, lastModified: obj.LastModified});
 					}
 				}
 			}
+			if (
+				params.maxObjects !== undefined &&
+				(listedObjects > params.maxObjects || (listedObjects === params.maxObjects && response.IsTruncated))
+			) {
+				throw new StorageObjectListingOverflowError(params.bucket, params.prefix, params.maxObjects);
+			}
+			if (response.IsTruncated && !response.NextContinuationToken) {
+				throw new Error(
+					`Truncated object listing omitted its continuation token for ${params.bucket}/${params.prefix}`,
+				);
+			}
 			continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined;
 		} while (continuationToken);
 		return result;