fix(media): stream large files and honour size settings (#3387)

fluxer_media_proxy/Cargo.toml

@@ -33,6 +33,7 @@ http-body = "1.1.0"
 http-body-util = "0.1.5"
 hyper = {version = "1.11.1", default-features = false}
 libc = "0.2.189"
+memmap2 = "0.9.11"
 moka = {version = "0.12.16", features = ["sync"]}
 parking_lot = "0.12.5"
 percent-encoding = "2.3.2"

fluxer_media_proxy/src/media_process/av_metadata.rs

@@ -120,9 +120,6 @@ pub fn probe_av_metadata(
     if input.is_empty() {
         return Err(MediaError::MediaDecodeFailed);
     }
-    if input.len() > media_limits.max_media_proxy_bytes() {
-        return Err(MediaError::StreamTooLong);
-    }
     if preview_max_dimension > NSFW_PREVIEW_MAX_DIMENSION {
         return Err(MediaError::InvalidImageDimensions);
     }

fluxer_media_proxy/src/media_process/metadata.rs

@@ -137,15 +137,15 @@ fn metadata_blocking(request: MetadataBlockingRequest<'_>) -> Result<PreparedMet
         input,
         options,
     } = request;
-    if input.len() > constants::MAX_MEDIA_PROXY_BYTES {
-        return Err(MediaError::StreamTooLong);
-    }
     let sniffed = mime::sniff(input);
     if !mime::is_supported_media_mime(sniffed.mime) {
         return Err(MediaError::UnsupportedMediaType);
     }
     let initial_category = mime::category(sniffed.mime).ok_or(MediaError::UnsupportedMediaType)?;
     let is_image = initial_category == mime::Category::Image;
+    if is_image && input.len() > constants::MAX_MEDIA_PROXY_BYTES {
+        return Err(MediaError::StreamTooLong);
+    }
     let dims = if is_image {
         let dims = probe_image_dims(media_limits, input)?;
         validate_jxl_decode_pixels(sniffed.mime, dims)?;

fluxer_media_proxy/src/media_process/mod.rs

@@ -37,7 +37,7 @@ pub use metadata::{
     MetadataOptions, PreparedMetadata, finish_metadata, metadata_json_with_options,
     prepare_metadata,
 };
-pub use native_cost::{AV_NATIVE_COST_BYTES, image_transform_cost, metadata_cost};
+pub use native_cost::{AV_NATIVE_COST_BYTES, av_native_cost, image_transform_cost, metadata_cost};
 pub use nsfw_processing::encode_static_image_for_nsfw;
 pub use transform::transform_image;
 pub use video_thumbnail::{

fluxer_media_proxy/src/media_process/native_cost.rs

@@ -5,8 +5,8 @@ use super::image_probe::probe_image_dims;
 use super::loaded_image::static_output_pixels;
 use super::native_runtime::ensure_vips_init;
 use crate::{
-    image_quality::ImageQuality, image_transform::ImageOptions, media_limits::MediaLimits, mime,
-    output_format::OutputFormat,
+    constants, image_quality::ImageQuality, image_transform::ImageOptions,
+    media_limits::MediaLimits, mime, output_format::OutputFormat,
 };
 
 pub const AV_NATIVE_COST_BYTES: usize = 256 << 20;
@@ -53,6 +53,12 @@ pub fn image_transform_cost(
     Ok(input.len().saturating_add(pixel_cost))
 }
 
+pub fn av_native_cost(input_len: usize) -> usize {
+    input_len
+        .min(constants::MAX_MEDIA_PROXY_BYTES)
+        .saturating_add(AV_NATIVE_COST_BYTES)
+}
+
 pub fn metadata_cost(input: &[u8], media_limits: &MediaLimits) -> Result<usize, MediaError> {
     let sniffed_mime = mime::sniff(input).mime;
     match mime::category(sniffed_mime) {
@@ -64,9 +70,7 @@ pub fn metadata_cost(input: &[u8], media_limits: &MediaLimits) -> Result<usize,
                 .len()
                 .saturating_add(source_pixels * decode_bytes_per_pixel(sniffed_mime)))
         }
-        Some(mime::Category::Video | mime::Category::Audio) => {
-            Ok(input.len().saturating_add(AV_NATIVE_COST_BYTES))
-        }
+        Some(mime::Category::Video | mime::Category::Audio) => Ok(av_native_cost(input.len())),
         _ => Ok(input.len()),
     }
 }

fluxer_media_proxy/src/media_process/nsfw_processing/mod.rs

@@ -111,7 +111,6 @@ pub(super) async fn classify_nsfw_buffers(
 
 fn nsfw_video_scan_buffers(source: &NSFWScanSource<'_>) -> Result<Vec<Vec<u8>>, MediaError> {
     extract_video_frames_for_nsfw(VideoNSFWFramesRequest {
-        media_limits: source.media_limits,
         input: source.input,
         duration_seconds: source.duration_seconds,
         deadline_ms: source.deadline_ms,
@@ -363,7 +362,6 @@ pub(super) fn nsfw_frame_seed(input: &[u8]) -> u64 {
 }
 
 pub(super) struct VideoNSFWFramesRequest<'a> {
-    pub(super) media_limits: &'a MediaLimits,
     pub(super) input: &'a [u8],
     pub(super) duration_seconds: Option<f64>,
     pub(super) deadline_ms: Option<i64>,
@@ -374,13 +372,12 @@ pub(super) fn extract_video_frames_for_nsfw(
 ) -> Result<Vec<Vec<u8>>, MediaError> {
     use rand::SeedableRng as _;
     let VideoNSFWFramesRequest {
-        media_limits,
         input,
         duration_seconds,
         deadline_ms,
     } = request;
-    if input.is_empty() || input.len() > media_limits.max_media_proxy_bytes() {
-        return Err(MediaError::StreamTooLong);
+    if input.is_empty() {
+        return Err(MediaError::MediaDecodeFailed);
     }
     let runtime = vips_runtime()?;
     let seed = nsfw_frame_seed(input);

fluxer_media_proxy/src/media_process/nsfw_processing/tests.rs

@@ -56,7 +56,6 @@ fn video_nsfw_frames_are_downscaled_to_the_frame_cap() {
         .probe
         .duration_seconds;
     let frames = extract_video_frames_for_nsfw(VideoNSFWFramesRequest {
-        media_limits: &media_limits,
         input: &mp4,
         duration_seconds: duration,
         deadline_ms: None,
@@ -215,7 +214,6 @@ fn a_sample_beyond_the_packet_budget_keeps_the_frames_that_decoded() {
     };
     let media_limits = limits();
     let frames = extract_video_frames_for_nsfw(VideoNSFWFramesRequest {
-        media_limits: &media_limits,
         input: &mp4,
         duration_seconds: Some(12.0),
         deadline_ms: None,

fluxer_media_proxy/src/media_process/tests/metadata.rs

@@ -1,22 +1,33 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+use super::super::nsfw_processing::{VideoNSFWFramesRequest, extract_video_frames_for_nsfw};
 use super::super::placeholder::{encode_thumbhash, optional_thumbhash};
 use super::super::{
-    MediaError, MetadataOptions, NSFW_PREVIEW_MAX_DIMENSION, metadata_json_with_options,
-    probe_av_metadata,
+    AV_NATIVE_COST_BYTES, MediaError, MetadataOptions, NSFW_PREVIEW_MAX_DIMENSION,
+    extract_video_thumbnail, metadata_cost, metadata_json_with_options, probe_av_metadata,
 };
 use super::fixtures::{metadata_value, test_media_limits};
 use crate::{
+    constants,
     metrics::transform::TransformMetrics,
     mime,
     nsfw::{NSFWClient, NSFWPolicy},
+    output_format::OutputFormat,
     test_fixtures::{
         fixture_audio_mp3_with_png_cover_art, fixture_audio_mp4_with_attached_picture,
         fixture_audio_only_mp4, fixture_h264_mp4, fixture_mkv_with_png_video_stream,
         fixture_mp4_with_undecodable_video, synthetic_png, synthetic_wav,
     },
 };
-use std::sync::Arc;
+use std::{io::Write as _, sync::Arc};
+
+fn mapped_past_the_buffer_bound(head: &[u8]) -> memmap2::Mmap {
+    let mut file = tempfile::tempfile().expect("temp file");
+    file.write_all(head).expect("written head");
+    file.set_len(constants::MAX_MEDIA_PROXY_BYTES as u64 + 1)
+        .expect("sparse tail");
+    unsafe { memmap2::Mmap::map(&file).expect("mapped file") }
+}
 
 #[test]
 fn metadata_json_includes_dimensions_and_placeholder() {
@@ -258,3 +269,54 @@ fn metadata_refuses_jxl_above_the_decode_pixel_cap() {
         });
     assert_eq!(MediaError::InvalidImageDimensions, err);
 }
+
+#[test]
+fn metadata_refuses_an_image_past_the_buffer_bound() {
+    let png = mapped_past_the_buffer_bound(&synthetic_png(16, 16));
+    assert_eq!("image/png", mime::sniff(&png).mime);
+    let err = tokio::runtime::Builder::new_current_thread()
+        .build()
+        .unwrap()
+        .block_on(async {
+            metadata_json_with_options(
+                &png,
+                "huge.png",
+                MetadataOptions::default(),
+                &test_media_limits(),
+                &NSFWClient::disabled(),
+                &TransformMetrics::new(),
+            )
+            .await
+            .unwrap_err()
+        });
+    assert_eq!(MediaError::StreamTooLong, err);
+}
+
+#[test]
+fn native_cost_of_a_video_past_the_buffer_bound_leaves_out_the_mapped_tail() {
+    let mp4 = mapped_past_the_buffer_bound(&fixture_h264_mp4());
+    assert_eq!(
+        constants::MAX_MEDIA_PROXY_BYTES + AV_NATIVE_COST_BYTES,
+        metadata_cost(&mp4, &test_media_limits()).expect("video cost")
+    );
+}
+
+#[test]
+fn video_reads_accept_an_input_past_the_buffer_bound() {
+    let mp4 = mapped_past_the_buffer_bound(&fixture_h264_mp4());
+    let limits = test_media_limits();
+
+    let probe = probe_av_metadata(&mp4, NSFW_PREVIEW_MAX_DIMENSION, &limits, None)
+        .expect("probed video")
+        .probe;
+    assert!(probe.has_video);
+    let poster = extract_video_thumbnail(&mp4, OutputFormat::WebP, &limits).expect("poster frame");
+    assert_eq!("image/webp", mime::sniff(&poster.bytes).mime);
+    let frames = extract_video_frames_for_nsfw(VideoNSFWFramesRequest {
+        input: &mp4,
+        duration_seconds: probe.duration_seconds,
+        deadline_ms: None,
+    })
+    .expect("sampled frames");
+    assert!(!frames.is_empty());
+}

fluxer_media_proxy/src/media_process/tests/nsfw.rs

@@ -4,7 +4,7 @@ use super::super::nsfw_processing::{
     VideoNSFWFramesRequest, compute_frame_sample_timestamps, extract_video_frames_for_nsfw,
     nsfw_frame_seed,
 };
-use super::super::probe_av_metadata;
+use super::super::{MediaError, probe_av_metadata};
 use super::fixtures::test_media_limits;
 
 #[test]
@@ -58,7 +58,6 @@ fn extract_video_frames_for_nsfw_returns_multiple_frames() {
         .probe
         .duration_seconds;
     let frames = extract_video_frames_for_nsfw(VideoNSFWFramesRequest {
-        media_limits: &limits,
         input: &bytes,
         duration_seconds: duration,
         deadline_ms: None,
@@ -74,3 +73,17 @@ fn extract_video_frames_for_nsfw_returns_multiple_frames() {
         assert_eq!(&f[..2], &[0xFF, 0xD8], "not a JPEG");
     }
 }
+
+#[test]
+fn an_empty_input_fails_to_decode_for_the_probe_and_the_frame_sampler() {
+    let frames = extract_video_frames_for_nsfw(VideoNSFWFramesRequest {
+        input: &[],
+        duration_seconds: None,
+        deadline_ms: None,
+    });
+    assert_eq!(Some(MediaError::MediaDecodeFailed), frames.err());
+    assert_eq!(
+        Some(MediaError::MediaDecodeFailed),
+        probe_av_metadata(&[], 0, &test_media_limits(), None).err()
+    );
+}

fluxer_media_proxy/src/media_process/video_thumbnail.rs

@@ -45,9 +45,6 @@ pub fn extract_video_thumbnail_with_options(
     options: VideoThumbnailOptions,
     media_limits: &MediaLimits,
 ) -> Result<ProcessedMedia, MediaError> {
-    if input.len() > media_limits.max_media_proxy_bytes() {
-        return Err(MediaError::StreamTooLong);
-    }
     let VideoThumbnailOptions {
         format,
         width,

fluxer_media_proxy/src/server/media_operations/input.rs

@@ -4,18 +4,24 @@ use super::failure::MediaFailure;
 use crate::{
     byte_budget::BudgetedBytes,
     constants::{self, AssetExtension},
+    mime,
     server::{
         external::fetch_external,
         params::url_filename,
         self_origin::{self, SelfOrigin},
         state::AppState,
     },
-    storage::StorageError,
+    spool::{SpoolError, spool_to_temp},
+    storage::{self, StorageError, StreamObject},
 };
+use axum::body::Body;
 use base64::{Engine as _, engine::general_purpose};
 use bytes::Bytes;
+use futures_util::StreamExt as _;
 use tracing::warn;
 
+pub(in crate::server) const STORED_SNIFF_PREFIX_BYTES: usize = 8192;
+
 pub(in crate::server) enum MediaInput {
     Base64 {
         data: String,
@@ -89,17 +95,20 @@ pub(in crate::server) async fn load_media_input(
             upload_filename,
             filename,
         } => {
-            let object = app
-                .store
-                .read_object(&app.cfg.storage.bucket_uploads, &upload_filename)
-                .await
-                .map_err(|err| {
-                    let failure = MediaFailure::MetadataUploadRead;
-                    warn!(reason = failure.code(), key = upload_filename.as_str(), %err);
-                    failure
-                })?;
+            let data = read_stored_input(
+                app,
+                &app.cfg.storage.bucket_uploads,
+                &upload_filename,
+                limit.decoded_bytes,
+            )
+            .await
+            .map_err(|err| {
+                let failure = MediaFailure::MetadataUploadRead;
+                warn!(reason = failure.code(), key = upload_filename.as_str(), %err);
+                failure
+            })?;
             Ok(LoadedMediaInput {
-                data: object.data,
+                data,
                 filename: filename.unwrap_or(upload_filename),
             })
         }
@@ -108,13 +117,15 @@ pub(in crate::server) async fn load_media_input(
             key,
             filename,
         } => {
-            let object = app.store.read_object(&bucket, &key).await.map_err(|err| {
-                let failure = MediaFailure::MetadataS3Read;
-                warn!(reason = failure.code(), bucket = bucket.as_str(), key = key.as_str(), %err);
-                failure
-            })?;
+            let data = read_stored_input(app, &bucket, &key, limit.decoded_bytes)
+                .await
+                .map_err(|err| {
+                    let failure = MediaFailure::MetadataS3Read;
+                    warn!(reason = failure.code(), bucket = bucket.as_str(), key = key.as_str(), %err);
+                    failure
+                })?;
             Ok(LoadedMediaInput {
-                data: object.data,
+                data,
                 filename: filename.unwrap_or(key),
             })
         }
@@ -160,6 +171,89 @@ async fn load_external_url(
     })
 }
 
+async fn read_stored_input(
+    app: &AppState,
+    bucket: &str,
+    key: &str,
+    buffer_limit: usize,
+) -> Result<Bytes, StorageError> {
+    let budget = storage::unversioned_read_budget(buffer_limit);
+    match app
+        .store
+        .read_object_limited(bucket, key, buffer_limit, &budget)
+        .await
+    {
+        Err(StorageError::StreamTooLong) => Ok(spool_stored_media(
+            app,
+            bucket,
+            key,
+            &[mime::Category::Video, mime::Category::Audio],
+        )
+        .await?
+        .data),
+        other => Ok(other?.data.as_bytes().clone()),
+    }
+}
+
+pub(in crate::server) async fn spool_stored_media(
+    app: &AppState,
+    bucket: &str,
+    key: &str,
+    categories: &[mime::Category],
+) -> Result<storage::Object, StorageError> {
+    let range = format!("bytes=0-{}", STORED_SNIFF_PREFIX_BYTES - 1);
+    let head = app.store.stream_object(bucket, key, Some(&range)).await?;
+    let prefix = read_sniff_prefix(head.body)
+        .await
+        .map_err(|err| StorageError::ObjectStorage(err.into()))?;
+    let sniffed = mime::sniff(&prefix).mime;
+    if !mime::is_supported_media_mime(sniffed)
+        || !mime::category(sniffed).is_some_and(|category| categories.contains(&category))
+    {
+        return Err(StorageError::StreamTooLong);
+    }
+    let object = app.store.stream_object(bucket, key, None).await?;
+    Ok(storage::Object {
+        data: spool_stream(app, object).await?,
+        content_type: sniffed.to_owned(),
+    })
+}
+
+async fn spool_stream(app: &AppState, object: StreamObject) -> Result<Bytes, StorageError> {
+    let Some(content_length) = object.content_length else {
+        return Err(StorageError::StreamTooLong);
+    };
+    let relay = &app.cfg.upload_relay;
+    let spooled = spool_to_temp(
+        object.body,
+        Some(content_length),
+        relay.spool_max_total_bytes,
+        &relay.spool_dir,
+        relay.spool_chunk_bytes,
+        relay.spool_max_total_bytes,
+    )
+    .await
+    .map_err(|err| match err {
+        SpoolError::PayloadTooLarge => StorageError::StreamTooLong,
+        other => StorageError::ObjectStorage(other.into()),
+    })?;
+    Ok(Bytes::from_owner(spooled.into_mapped()?))
+}
+
+pub(in crate::server) async fn read_sniff_prefix(body: Body) -> Result<Vec<u8>, axum::Error> {
+    let mut prefix = Vec::with_capacity(STORED_SNIFF_PREFIX_BYTES);
+    let mut chunks = body.into_data_stream();
+    while prefix.len() < STORED_SNIFF_PREFIX_BYTES {
+        let Some(chunk) = chunks.next().await else {
+            break;
+        };
+        let chunk = chunk?;
+        let wanted = chunk.len().min(STORED_SNIFF_PREFIX_BYTES - prefix.len());
+        prefix.extend_from_slice(&chunk[..wanted]);
+    }
+    Ok(prefix)
+}
+
 async fn read_own_object(
     app: &AppState,
     bucket: &str,
@@ -182,3 +276,220 @@ async fn read_own_object(
 pub(super) fn retained_input_bytes(data: BudgetedBytes) -> Bytes {
     Bytes::from_owner(data)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        config::Config,
+        storage::tests::{FakeObject, FakeS3, fake_s3},
+    };
+    use axum::http::{Method, StatusCode, header};
+
+    const BUFFER_LIMIT: MediaInputLimit = MediaInputLimit {
+        encoded_bytes: 1024,
+        decoded_bytes: 1024,
+    };
+
+    fn local_storage_app(root: &std::path::Path) -> AppState {
+        let spool_dir = root.join("spool");
+        std::fs::create_dir(&spool_dir).unwrap();
+        let cfg = Config::load_from_iter([
+            ("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret".to_owned()),
+            ("FLUXER_MEDIA_PROXY_STORAGE_BACKEND", "local".to_owned()),
+            (
+                "FLUXER_MEDIA_PROXY_STORAGE_ROOT",
+                root.display().to_string(),
+            ),
+            (
+                "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SPOOL_DIR",
+                spool_dir.display().to_string(),
+            ),
+        ])
+        .expect("test config");
+        AppState::for_tests(cfg)
+    }
+
+    #[tokio::test]
+    async fn a_stored_input_larger_than_the_buffer_limit_is_read_from_a_spooled_copy() {
+        let tmp = tempfile::tempdir().unwrap();
+        let root = tmp.path().canonicalize().unwrap();
+        let app = local_storage_app(&root);
+        let uploads = app.cfg.storage.bucket_uploads.clone();
+        let cdn = app.cfg.storage.bucket_cdn.clone();
+        let mp4 = crate::test_fixtures::fixture_h264_mp4();
+        assert!(mp4.len() > BUFFER_LIMIT.decoded_bytes);
+        for bucket in [&uploads, &cdn] {
+            app.store
+                .write_object(bucket, "big.mp4", &mp4, "video/mp4")
+                .await
+                .unwrap();
+        }
+
+        let upload = load_media_input(
+            &app,
+            MediaInput::Upload {
+                upload_filename: "big.mp4".to_owned(),
+                filename: None,
+            },
+            BUFFER_LIMIT,
+        )
+        .await
+        .unwrap_or_else(|failure| panic!("upload input failed: {}", failure.code()));
+        let stored = load_media_input(
+            &app,
+            MediaInput::Storage {
+                bucket: cdn,
+                key: "big.mp4".to_owned(),
+                filename: Some("named.mp4".to_owned()),
+            },
+            BUFFER_LIMIT,
+        )
+        .await
+        .unwrap_or_else(|failure| panic!("stored input failed: {}", failure.code()));
+
+        assert_eq!(&mp4[..], &upload.data[..]);
+        assert_eq!("big.mp4", upload.filename);
+        assert_eq!(&mp4[..], &stored.data[..]);
+        assert_eq!("named.mp4", stored.filename);
+        let metadata =
+            crate::media_process::probe_av_metadata(&upload.data, 0, &app.media.limits(), None)
+                .unwrap();
+        assert!(metadata.probe.has_video);
+        assert!(
+            std::fs::read_dir(root.join("spool"))
+                .unwrap()
+                .next()
+                .is_none(),
+            "a spooled copy is never visible in the spool directory"
+        );
+    }
+
+    fn upload_input(key: &str) -> MediaInput {
+        MediaInput::Upload {
+            upload_filename: key.to_owned(),
+            filename: None,
+        }
+    }
+
+    fn body_reads(fake: &FakeS3) -> Vec<Option<String>> {
+        fake.requests()
+            .into_iter()
+            .filter(|(method, _, _, _)| *method == Method::GET)
+            .map(|(_, _, headers, _)| {
+                headers
+                    .get(header::RANGE)
+                    .and_then(|value| value.to_str().ok())
+                    .map(str::to_owned)
+            })
+            .collect()
+    }
+
+    #[tokio::test]
+    async fn object_storage_spools_a_video_larger_than_the_buffer_limit() {
+        let fake = fake_s3().await;
+        let mp4 = crate::test_fixtures::fixture_h264_mp4();
+        fake.put_object(
+            "uploads/big.mp4",
+            FakeObject {
+                body: mp4.clone(),
+                content_type: Some("application/octet-stream".to_owned()),
+                ..FakeObject::default()
+            },
+        );
+        let tmp = tempfile::tempdir().unwrap();
+        let app = AppState::for_tests(fake.config(tmp.path()));
+
+        let loaded = load_media_input(&app, upload_input("big.mp4"), BUFFER_LIMIT)
+            .await
+            .unwrap_or_else(|failure| panic!("upload input failed: {}", failure.code()));
+
+        assert_eq!(&mp4[..], &loaded.data[..]);
+        assert_eq!(
+            vec![None, Some("bytes=0-8191".to_owned()), None],
+            body_reads(&fake)
+        );
+    }
+
+    #[tokio::test]
+    async fn an_oversized_input_that_is_not_audio_or_video_is_refused_from_its_prefix() {
+        let fake = fake_s3().await;
+        let mut png = crate::test_fixtures::synthetic_png(8, 8);
+        png.resize(BUFFER_LIMIT.decoded_bytes + 1, 0);
+        for (key, body) in [
+            ("uploads/huge.png", png),
+            (
+                "uploads/notes.mp4",
+                vec![b'a'; BUFFER_LIMIT.decoded_bytes + 1],
+            ),
+        ] {
+            fake.put_object(
+                key,
+                FakeObject {
+                    body,
+                    ..FakeObject::default()
+                },
+            );
+        }
+        let tmp = tempfile::tempdir().unwrap();
+        let app = AppState::for_tests(fake.config(tmp.path()));
+
+        for key in ["huge.png", "notes.mp4"] {
+            let failure = load_media_input(&app, upload_input(key), BUFFER_LIMIT)
+                .await
+                .err()
+                .expect("only audio and video are read past the buffer limit");
+            assert_eq!("metadata_upload_read", failure.code(), "{key}");
+        }
+
+        assert_eq!(
+            vec![
+                None,
+                Some("bytes=0-8191".to_owned()),
+                None,
+                Some("bytes=0-8191".to_owned())
+            ],
+            body_reads(&fake)
+        );
+    }
+
+    #[tokio::test]
+    async fn a_stream_without_a_declared_length_is_not_spooled() {
+        let tmp = tempfile::tempdir().unwrap();
+        let app = local_storage_app(&tmp.path().canonicalize().unwrap());
+        let object = StreamObject {
+            body: Body::from_stream(futures_util::stream::iter([Ok::<_, std::io::Error>(
+                Bytes::from_static(b"payload"),
+            )])),
+            status: StatusCode::OK,
+            content_length: None,
+            content_type: "video/mp4".to_owned(),
+            byte_range: None,
+            total_length: None,
+        };
+
+        let refused = spool_stream(&app, object).await;
+
+        assert!(matches!(refused, Err(StorageError::StreamTooLong)));
+    }
+
+    #[tokio::test]
+    async fn a_missing_stored_input_is_reported_as_a_read_failure() {
+        let tmp = tempfile::tempdir().unwrap();
+        let app = local_storage_app(&tmp.path().canonicalize().unwrap());
+
+        let failure = load_media_input(
+            &app,
+            MediaInput::Upload {
+                upload_filename: "missing.mp4".to_owned(),
+                filename: None,
+            },
+            BUFFER_LIMIT,
+        )
+        .await
+        .err()
+        .expect("a missing upload cannot load");
+
+        assert_eq!("metadata_upload_read", failure.code());
+    }
+}

fluxer_media_proxy/src/server/media_operations/mod.rs

@@ -4,7 +4,10 @@ mod failure;
 mod input;
 
 pub(in crate::server) use failure::MediaFailure;
-pub(in crate::server) use input::{MediaInput, MediaInputLimit, load_media_input};
+pub(in crate::server) use input::{
+    MediaInput, MediaInputLimit, STORED_SNIFF_PREFIX_BYTES, load_media_input, read_sniff_prefix,
+    spool_stored_media,
+};
 
 use crate::{
     constants::{self, AssetExtension},
@@ -34,6 +37,9 @@ pub(in crate::server) async fn resolve_metadata(
     include_data: bool,
 ) -> Result<MetadataOutput, MediaFailure> {
     let mut input = load_media_input(app, input, MediaInputLimit::INTERNAL_REQUEST).await?;
+    if include_data && input.data.len() > constants::MAX_MEDIA_PROXY_BYTES {
+        return Err(MediaFailure::MetadataDecodedTooLarge);
+    }
     if include_data && metadata_input_is_svg(&input) {
         input = rasterize_metadata_svg(app, input).await?;
     }

fluxer_media_proxy/src/server/routes/internal.rs

@@ -10,8 +10,8 @@ use crate::{
     server::{
         format_policy::is_svg_content_type,
         media_operations::{
-            MediaFailure, MediaInput, MediaInputLimit, MetadataOutput, load_media_input,
-            resolve_metadata,
+            MediaFailure, MediaInput, MediaInputLimit, MetadataOutput, STORED_SNIFF_PREFIX_BYTES,
+            load_media_input, read_sniff_prefix, resolve_metadata,
         },
         response::{
             MediaResponse,
@@ -34,12 +34,9 @@ use axum::{
 };
 use base64::{Engine as _, engine::general_purpose};
 use bytes::Bytes;
-use futures_util::StreamExt as _;
 use serde::Deserialize;
 use std::sync::Arc;
 
-const UPLOAD_SNIFF_PREFIX_BYTES: usize = 8192;
-
 #[derive(Debug, Deserialize)]
 struct MetadataRequest {
     version: Option<i64>,
@@ -304,7 +301,7 @@ pub(in crate::server) async fn sniff_handler(
         Ok(req) => req,
         Err(_) => return text(StatusCode::BAD_REQUEST, "Bad Request"),
     };
-    let range = format!("bytes=0-{}", UPLOAD_SNIFF_PREFIX_BYTES - 1);
+    let range = format!("bytes=0-{}", STORED_SNIFF_PREFIX_BYTES - 1);
     let object = match app
         .store
         .stream_object(
@@ -337,20 +334,6 @@ pub(in crate::server) async fn sniff_handler(
     )
 }
 
-async fn read_sniff_prefix(body: Body) -> Result<Vec<u8>, axum::Error> {
-    let mut prefix = Vec::with_capacity(UPLOAD_SNIFF_PREFIX_BYTES);
-    let mut chunks = body.into_data_stream();
-    while prefix.len() < UPLOAD_SNIFF_PREFIX_BYTES {
-        let Some(chunk) = chunks.next().await else {
-            break;
-        };
-        let chunk = chunk?;
-        let wanted = chunk.len().min(UPLOAD_SNIFF_PREFIX_BYTES - prefix.len());
-        prefix.extend_from_slice(&chunk[..wanted]);
-    }
-    Ok(prefix)
-}
-
 fn check_internal_auth(headers: &HeaderMap, secret: &str) -> bool {
     let Some(auth) = headers
         .get(header::AUTHORIZATION)
@@ -885,7 +868,7 @@ mod tests {
         let stream_source = upload.clone();
         let body = Body::from_stream(futures_util::stream::poll_fn(move |_| {
             assert!(
-                yielded < UPLOAD_SNIFF_PREFIX_BYTES,
+                yielded < STORED_SNIFF_PREFIX_BYTES,
                 "the body was polled again after {yielded} bytes were yielded"
             );
             let end = (yielded + 1000).min(stream_source.len());
@@ -896,7 +879,7 @@ mod tests {
 
         let prefix = read_sniff_prefix(body).await.expect("prefix read");
 
-        assert_eq!(&upload[..UPLOAD_SNIFF_PREFIX_BYTES], prefix.as_slice());
+        assert_eq!(&upload[..STORED_SNIFF_PREFIX_BYTES], prefix.as_slice());
     }
 
     #[tokio::test]

fluxer_media_proxy/src/server/stored/mod.rs

@@ -20,6 +20,7 @@ use crate::{
             effective_animated_image_output_format, extension_from_mime, is_svg_content_type,
             same_format_loaded_image_request_can_use_original_with_sniff,
         },
+        media_operations::spool_stored_media,
         params::{
             animated_param, asset_manual_format_override, asset_wants_animated, bool_param,
             filename_from_storage_key,
@@ -31,12 +32,13 @@ use crate::{
         },
         state::AppState,
         transform::{
-            ServeBytesRequest,
+            CachedVideoTransformRequest, ServeBytesRequest,
             cache::{
                 CachedTransformRequest, cached_transform, cached_transform_hit,
                 coalescer_failure_response,
             },
             cache_key::{TransformCacheKeyInput, transform_cache_key},
+            cached_video_transform_response,
             execution::{coalesced_work_result, deadline_instant, run_transform},
             original_image_content_type,
             parameters::TransformRoute,
@@ -336,6 +338,66 @@ async fn read_attachment(
         .store
         .read_object(&app.cfg.storage.bucket_cdn, key)
         .await
+    {
+        Ok(object) => object,
+        Err(StorageError::StreamTooLong) => {
+            return serve_spooled_attachment_video(app, method, key, params, headers).await;
+        }
+        Err(err) => return storage_error_response(key, err),
+    };
+    serve_bytes_or_transform(
+        app.media.transforms(),
+        ServeBytesRequest {
+            method,
+            data: object.data,
+            content_type: object.content_type,
+            cache_identity: key,
+            filename,
+            route: TransformRoute::Attachment,
+            params,
+            headers,
+        },
+    )
+    .await
+}
+
+static SPOOLED_ATTACHMENT_READS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
+
+async fn serve_spooled_attachment_video(
+    app: &Arc<AppState>,
+    method: Method,
+    key: &str,
+    params: &HashMap<String, String>,
+    headers: &HeaderMap,
+) -> Response {
+    let filename = filename_from_storage_key(key);
+    let cached = || {
+        cached_video_transform_response(
+            app.media.transforms(),
+            CachedVideoTransformRequest {
+                method: method.clone(),
+                cache_identity: key,
+                filename,
+                route: TransformRoute::Attachment,
+                params,
+                headers,
+            },
+        )
+    };
+    if let Some(response) = cached() {
+        return response;
+    }
+    let _serial = SPOOLED_ATTACHMENT_READS.lock().await;
+    if let Some(response) = cached() {
+        return response;
+    }
+    let object = match spool_stored_media(
+        app,
+        &app.cfg.storage.bucket_cdn,
+        key,
+        &[mime::Category::Video],
+    )
+    .await
     {
         Ok(object) => object,
         Err(err) => return storage_error_response(key, err),

fluxer_media_proxy/src/server/stored/passthrough.rs

@@ -111,9 +111,6 @@ pub(in crate::server) async fn serve_stored_passthrough_stream(
         Ok(value) => value,
         Err(err) => return storage_error_response(key, err),
     };
-    if total_len > constants::MAX_MEDIA_PROXY_BYTES {
-        return storage_error_response(key, StorageError::StreamTooLong);
-    }
     streaming_media_response(StreamingMediaResponse {
         method,
         object,
@@ -135,9 +132,6 @@ async fn serve_stored_passthrough_head(
         Ok(head) => head,
         Err(err) => return storage_error_response(key, err),
     };
-    if head.content_length > constants::MAX_MEDIA_PROXY_BYTES as u64 {
-        return storage_error_response(key, StorageError::StreamTooLong);
-    }
     let content_type = passthrough_content_type(&head.content_type, key);
     if app.cfg.mode == DeploymentMode::Mp
         && (is_svg_content_type(&content_type)

fluxer_media_proxy/src/server/stored/tests.rs

@@ -285,6 +285,116 @@ async fn a_stored_passthrough_stream_declares_the_length_it_serves() {
     assert_eq!(&b"hello"[..], &response_body(response).await[..]);
 }
 
+#[tokio::test]
+async fn a_stored_passthrough_serves_objects_larger_than_the_buffer_cap() {
+    let tmp = tempfile::tempdir().unwrap();
+    let storage_root = tmp.path().canonicalize().unwrap();
+    let app = test_app_state(asset_test_config(&storage_root));
+    let bucket = app.cfg.storage.bucket_cdn.clone();
+    app.store
+        .write_object(&bucket, "a/big.mp4", b"head", "video/mp4")
+        .await
+        .unwrap();
+    let total_len = constants::MAX_MEDIA_PROXY_BYTES as u64 + 1;
+    std::fs::OpenOptions::new()
+        .write(true)
+        .open(storage_root.join(&bucket).join("a/big.mp4"))
+        .unwrap()
+        .set_len(total_len)
+        .unwrap();
+    let mut range = HeaderMap::new();
+    range.insert(header::RANGE, "bytes=0-3".parse().unwrap());
+
+    let head =
+        passthrough::serve_stored_raw(&app, Method::HEAD, &bucket, "a/big.mp4", &HeaderMap::new())
+            .await;
+    assert_eq!(StatusCode::OK, head.status());
+    assert_eq!(
+        Some(total_len.to_string().as_str()),
+        head.headers()
+            .get(header::CONTENT_LENGTH)
+            .and_then(|value| value.to_str().ok())
+    );
+
+    let get = passthrough::serve_stored_raw(&app, Method::GET, &bucket, "a/big.mp4", &range).await;
+    assert_eq!(StatusCode::PARTIAL_CONTENT, get.status());
+    assert_eq!(
+        Some(format!("bytes 0-3/{total_len}").as_str()),
+        get.headers()
+            .get(header::CONTENT_RANGE)
+            .and_then(|value| value.to_str().ok())
+    );
+    assert_eq!(&b"head"[..], &response_body(get).await[..]);
+}
+
+fn poster_params() -> HashMap<String, String> {
+    HashMap::from([
+        ("format".to_owned(), "webp".to_owned()),
+        ("width".to_owned(), "64".to_owned()),
+    ])
+}
+
+#[tokio::test]
+async fn a_spooled_attachment_video_serves_its_poster_and_then_answers_from_the_cache() {
+    let tmp = tempfile::tempdir().unwrap();
+    let storage_root = tmp.path().canonicalize().unwrap();
+    let app = test_app_state(asset_test_config(&storage_root));
+    let bucket = app.cfg.storage.bucket_cdn.clone();
+    let key = "attachments/1/2/clip.mp4";
+    app.store
+        .write_object(
+            &bucket,
+            key,
+            &crate::test_fixtures::fixture_h264_mp4(),
+            "application/octet-stream",
+        )
+        .await
+        .unwrap();
+    let params = poster_params();
+    let headers = HeaderMap::new();
+
+    let poster = serve_spooled_attachment_video(&app, Method::GET, key, &params, &headers).await;
+    assert_eq!(StatusCode::OK, poster.status());
+    let poster = response_body(poster).await;
+    assert_eq!("image/webp", mime::sniff(&poster).mime);
+
+    std::fs::remove_file(storage_root.join(&bucket).join(key)).unwrap();
+    let cached = serve_spooled_attachment_video(&app, Method::GET, key, &params, &headers).await;
+    assert_eq!(StatusCode::OK, cached.status());
+    assert_eq!(&poster[..], &response_body(cached).await[..]);
+}
+
+#[tokio::test]
+async fn a_spooled_attachment_read_refuses_everything_but_video() {
+    let tmp = tempfile::tempdir().unwrap();
+    let storage_root = tmp.path().canonicalize().unwrap();
+    let app = test_app_state(asset_test_config(&storage_root));
+    let bucket = app.cfg.storage.bucket_cdn.clone();
+    let params = poster_params();
+    for (key, data) in [
+        (
+            "attachments/1/2/huge.png",
+            crate::test_fixtures::synthetic_png(8, 8),
+        ),
+        (
+            "attachments/1/2/song.wav",
+            crate::test_fixtures::synthetic_wav(),
+        ),
+        ("attachments/1/2/disk.iso", vec![0u8; 4096]),
+    ] {
+        app.store
+            .write_object(&bucket, key, &data, "application/octet-stream")
+            .await
+            .unwrap();
+
+        let refused =
+            serve_spooled_attachment_video(&app, Method::GET, key, &params, &HeaderMap::new())
+                .await;
+
+        assert_eq!(StatusCode::PAYLOAD_TOO_LARGE, refused.status(), "{key}");
+    }
+}
+
 fn fake_asset_object(read_status: Option<u16>) -> crate::storage::tests::FakeObject {
     crate::storage::tests::FakeObject {
         body: crate::test_fixtures::synthetic_png(512, 512),

fluxer_media_proxy/src/server/transform/execution.rs

@@ -82,9 +82,7 @@ pub(in crate::server) async fn run_video_transform(
     let deadline = deadline_instant(deadline_ms);
     let media_limits = runtime.limits();
     let transform_metrics = runtime.metrics();
-    let cost_bytes = data
-        .len()
-        .saturating_add(media_process::AV_NATIVE_COST_BYTES);
+    let cost_bytes = media_process::av_native_cost(data.len());
     let timed = runtime
         .tasks()
         .run_native(

fluxer_media_proxy/src/server/transform/mod.rs

@@ -310,6 +310,73 @@ fn original_bytes_response(request: OriginalBytes<'_>) -> Response {
     })
 }
 
+struct VideoTransformPlan {
+    format: OutputFormat,
+    quality: ImageQuality,
+    cache_key: String,
+}
+
+fn video_transform_plan(
+    route: TransformRoute,
+    cache_identity: &str,
+    transform: &ValidatedTransformParameters,
+    requested_format: AssetExtension,
+) -> VideoTransformPlan {
+    let format = OutputFormat::coerce_from_extension(requested_format);
+    let quality = transform.quality.unwrap_or(ImageQuality::High);
+    VideoTransformPlan {
+        format,
+        quality,
+        cache_key: transform_cache_key(TransformCacheKeyInput {
+            route,
+            asset_kind: None,
+            cache_identity,
+            width: transform.width,
+            height: transform.height,
+            format,
+            quality: Some(quality),
+            animated: transform.animated,
+            effort: None,
+            resize_mode: None,
+        }),
+    }
+}
+
+pub(in crate::server) struct CachedVideoTransformRequest<'a> {
+    pub(in crate::server) method: Method,
+    pub(in crate::server) cache_identity: &'a str,
+    pub(in crate::server) filename: &'a str,
+    pub(in crate::server) route: TransformRoute,
+    pub(in crate::server) params: &'a HashMap<String, String>,
+    pub(in crate::server) headers: &'a HeaderMap,
+}
+
+pub(in crate::server) fn cached_video_transform_response(
+    runtime: &TransformRuntime,
+    request: CachedVideoTransformRequest<'_>,
+) -> Option<Response> {
+    let transform =
+        validate_transform_parameters(request.params, &runtime.limits(), request.route).ok()?;
+    let plan = video_transform_plan(
+        request.route,
+        request.cache_identity,
+        &transform,
+        transform.explicit_format?,
+    );
+    let hit = cached_transform_hit(runtime, &plan.cache_key)?;
+    Some(transformed_response(TransformedBytes {
+        method: request.method,
+        data: hit.data.as_bytes().clone(),
+        content_type: hit.format.mime(),
+        filename: request.filename,
+        range_header: request
+            .headers
+            .get(header::RANGE)
+            .and_then(|v| v.to_str().ok()),
+        requested_download: transform.requested_download,
+    }))
+}
+
 struct VideoTransformRequest<'a> {
     method: Method,
     data: Bytes,
@@ -352,20 +419,11 @@ async fn serve_video_transform(
             requested_download: transform.requested_download,
         });
     };
-    let format = OutputFormat::coerce_from_extension(requested_format);
-    let quality = transform.quality.unwrap_or(ImageQuality::High);
-    let cache_key = transform_cache_key(TransformCacheKeyInput {
-        route,
-        asset_kind: None,
-        cache_identity,
-        width: transform.width,
-        height: transform.height,
+    let VideoTransformPlan {
         format,
-        quality: Some(quality),
-        animated: transform.animated,
-        effort: None,
-        resize_mode: None,
-    });
+        quality,
+        cache_key,
+    } = video_transform_plan(route, cache_identity, &transform, requested_format);
     if let Some(hit) = cached_transform_hit(runtime, &cache_key) {
         return transformed_response(TransformedBytes {
             method,

fluxer_media_proxy/src/spool.rs

@@ -2,6 +2,7 @@
 
 use axum::body::Body;
 use http_body_util::BodyExt;
+use memmap2::Mmap;
 use std::{io::SeekFrom, path::Path};
 use thiserror::Error;
 use tokio::{
@@ -43,6 +44,12 @@ impl SpooledBody {
         (file, length)
     }
 
+    pub fn into_mapped(self) -> std::io::Result<MappedSpool> {
+        let file = self.file.as_ref().expect("SpooledBody already consumed");
+        let map = unsafe { Mmap::map(file)? };
+        Ok(MappedSpool { map, _spool: self })
+    }
+
     pub fn len(&self) -> u64 {
         self.length
     }
@@ -52,6 +59,17 @@ impl SpooledBody {
     }
 }
 
+pub struct MappedSpool {
+    map: Mmap,
+    _spool: SpooledBody,
+}
+
+impl AsRef<[u8]> for MappedSpool {
+    fn as_ref(&self) -> &[u8] {
+        &self.map
+    }
+}
+
 impl Drop for SpooledBody {
     fn drop(&mut self) {
         if self.budget_reservation > 0 {

fluxer_media_proxy/src/storage/local.rs

@@ -62,7 +62,6 @@ impl Store {
         &self,
         bucket: &str,
         key: &str,
-        max_bytes: usize,
     ) -> Result<HeadResult, StorageError> {
         let path = self.local_path(bucket, key).await?;
         let metadata = tokio::fs::symlink_metadata(&path)
@@ -71,9 +70,6 @@ impl Store {
         if !metadata.is_file() {
             return Err(StorageError::NotFound);
         }
-        if metadata.len() > max_bytes as u64 {
-            return Err(StorageError::StreamTooLong);
-        }
         Ok(HeadResult {
             content_length: metadata.len(),
             content_type: local_content_type(key),
@@ -86,9 +82,6 @@ impl Store {
     ) -> Result<StreamObject, StorageError> {
         let path = self.local_path(plan.bucket, plan.key).await?;
         let (mut file, metadata) = open_local_read_file(&path).await?;
-        if metadata.len() > plan.max_bytes as u64 {
-            return Err(StorageError::StreamTooLong);
-        }
         let total_len = usize::try_from(metadata.len()).map_err(|_| StorageError::StreamTooLong)?;
         let byte_range = match plan.range {
             StreamRange::Full => None,

fluxer_media_proxy/src/storage/mod.rs

@@ -80,7 +80,6 @@ enum ContentDigestRequest {
 struct ObjectStreamPlan<'a> {
     bucket: &'a str,
     key: &'a str,
-    max_bytes: usize,
     range: StreamRange<'a>,
 }
 

fluxer_media_proxy/src/storage/object_stream.rs

@@ -4,23 +4,13 @@ use super::{
     HeadResult, ObjectStreamPlan, StorageError, Store, StreamObject, StreamRange,
     record_storage_outcome,
 };
-use crate::{config::StorageBackend, constants};
+use crate::config::StorageBackend;
 
 impl Store {
     pub async fn head_object(&self, bucket: &str, key: &str) -> Result<HeadResult, StorageError> {
-        self.head_object_limited(bucket, key, constants::MAX_MEDIA_PROXY_BYTES)
-            .await
-    }
-
-    pub async fn head_object_limited(
-        &self,
-        bucket: &str,
-        key: &str,
-        max_bytes: usize,
-    ) -> Result<HeadResult, StorageError> {
         match self.cfg.storage.backend {
-            StorageBackend::Local => self.head_local(bucket, key, max_bytes).await,
-            StorageBackend::S3 => self.head_s3(bucket, key, max_bytes).await,
+            StorageBackend::Local => self.head_local(bucket, key).await,
+            StorageBackend::S3 => self.head_s3(bucket, key).await,
         }
     }
 
@@ -33,7 +23,6 @@ impl Store {
         self.stream_object_inner(ObjectStreamPlan {
             bucket,
             key,
-            max_bytes: constants::MAX_MEDIA_PROXY_BYTES,
             range: match range_header {
                 Some(header) => StreamRange::Header(header),
                 None => StreamRange::Full,

fluxer_media_proxy/src/storage/s3.rs

@@ -82,7 +82,6 @@ impl Store {
         &self,
         bucket: &str,
         key: &str,
-        max_bytes: usize,
     ) -> Result<HeadResult, StorageError> {
         // A HEAD is a read, so it follows the same endpoint and signing policy as a body read.
         // Sending it to the write origin instead would bypass the configured read endpoint and
@@ -98,9 +97,6 @@ impl Store {
             return Err(StorageError::S3(s3_error_summary(response).await));
         }
         let content_length = http_headers::parse_content_length(response.headers()).unwrap_or(0);
-        if content_length > max_bytes as u64 {
-            return Err(StorageError::StreamTooLong);
-        }
         let content_type = response_content_type(response.headers());
         Ok(HeadResult {
             content_length,
@@ -155,12 +151,6 @@ impl Store {
             return Err(StorageError::S3(s3_error_summary(response).await));
         }
         let content_length = http_headers::parse_content_length(response.headers());
-        if let Some(content_length) = content_length
-            && matches!(plan.range, StreamRange::Full)
-            && content_length > plan.max_bytes as u64
-        {
-            return Err(StorageError::StreamTooLong);
-        }
         let content_type = response_content_type(response.headers());
         let content_range = header_str(response.headers(), header::CONTENT_RANGE)
             .and_then(|value| range::parse_content_range(Some(&value)));