fix(media-proxy): serve originals and speed up slow transforms (#3393)

fluxer_media_proxy/src/media_process/animated_transform.rs

@@ -7,6 +7,7 @@ use super::{
     MediaError, native_animated_frame_limit, native_buffer_to_media_bytes, native_status_error,
 };
 use crate::{
+    constants,
     image_transform::ImageOptions,
     media_limits::MediaLimits,
     metrics::transform::TransformMetrics,
@@ -176,6 +177,15 @@ pub(super) struct GifResizeDims {
     pub(super) height: c_int,
 }
 
+impl GifResizeDims {
+    pub(super) fn fits_work_budget(self, frames: c_int) -> bool {
+        (self.width as usize)
+            .saturating_mul(self.height as usize)
+            .saturating_mul(frames.max(1) as usize)
+            <= constants::MAX_GIF_RESIZE_OUTPUT_PIXELS
+    }
+}
+
 pub(super) fn gif_resize_dims(
     sniffed: mime::SniffInfo,
     options: &ImageOptions,

fluxer_media_proxy/src/media_process/encoding.rs

@@ -156,6 +156,9 @@ pub(super) fn encode_vips_image(
     {
         return encode_animated_apng(image, ph, animation, media_limits, animation_loop_count);
     }
+    let width = unsafe { native::fluxer_vips_image_get_width(image.as_ptr()) };
+    let height = unsafe { native::fluxer_vips_image_get_height(image.as_ptr()) };
+    let output_pixels = u64::try_from(width).unwrap_or(0) * u64::try_from(height).unwrap_or(0);
     if format == OutputFormat::GIF
         && let Some(frame_height) = page_height
         && frame_height > 0
@@ -164,7 +167,13 @@ pub(super) fn encode_vips_image(
             image,
             resolve_animation_loop_count(image, animation_loop_count),
         )?;
-        let suffix = output_suffix(format, quality, Some(frame_height), effort_override)?;
+        let suffix = output_suffix(
+            format,
+            quality,
+            Some(frame_height),
+            effort_override,
+            output_pixels,
+        )?;
         return write_vips_image_to_vec(
             image,
             &suffix,
@@ -172,7 +181,7 @@ pub(super) fn encode_vips_image(
             animation.deadline_unix_ms,
         );
     }
-    let suffix = output_suffix(format, quality, page_height, effort_override)?;
+    let suffix = output_suffix(format, quality, page_height, effort_override, output_pixels)?;
     write_vips_image_to_vec(
         image,
         &suffix,

fluxer_media_proxy/src/media_process/loaded_image.rs

@@ -67,7 +67,7 @@ pub(crate) fn validate_jxl_decode_pixels(
         .saturating_mul(dims.height as usize)
         .saturating_mul(dims.pages.max(1) as usize);
     if pixels > constants::MAX_JXL_DECODE_PIXELS {
-        return Err(MediaError::InvalidImageDimensions);
+        return Err(MediaError::TooLargeToProcess);
     }
     Ok(())
 }
@@ -96,7 +96,7 @@ pub(crate) fn validate_static_output_pixels(
     dims: ImageDimensions,
 ) -> Result<(), MediaError> {
     if static_output_pixels(options, dims) > constants::MAX_STATIC_OUTPUT_PIXELS {
-        return Err(MediaError::InvalidImageDimensions);
+        return Err(MediaError::TooLargeToProcess);
     }
     Ok(())
 }

fluxer_media_proxy/src/media_process/mod.rs

@@ -154,6 +154,8 @@ pub enum MediaError {
     MediaTransformFailed,
     #[error("invalid image dimensions")]
     InvalidImageDimensions,
+    #[error("media too large to process")]
+    TooLargeToProcess,
     #[error("unsupported media type")]
     UnsupportedMediaType,
     #[error("stream too long")]

fluxer_media_proxy/src/media_process/tests/animated_gif.rs

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-use super::super::animated_transform::gif_resize_dims;
+use super::super::animated_transform::{GifResizeDims, gif_resize_dims};
 use super::super::{
     AnimationLimits, AnimationMode, ImageOptions, ImageQuality, MediaError, ResizeMode,
 };
@@ -156,6 +156,27 @@ fn animated_gif_with_trailing_bytes_after_the_trailer_passes_through_byte_identi
     assert_eq!(gif, passed_through.bytes);
 }
 
+#[test]
+fn animated_gif_with_trailing_bytes_after_the_trailer_resizes() {
+    for tail in [&[0u8; 4096][..], b"trailing garbage"] {
+        let mut gif = animated_gif_frames(3, 20);
+        gif.extend_from_slice(tail);
+        let resized = transform_image(
+            &gif,
+            &ImageOptions {
+                width: Some(8),
+                format: OutputFormat::GIF,
+                animation: animated_mode(),
+                ..Default::default()
+            },
+        )
+        .expect("a gif with trailing bytes after the trailer still resizes");
+
+        assert_animated_gif(&resized.bytes, 3);
+        assert_eq!(vec![20u16; 3], gif_frame_delays_cs(&resized.bytes));
+    }
+}
+
 #[test]
 fn animated_gif_over_the_frame_cap_passes_through_byte_identically() {
     let gif = animated_gif_frames(5, 20);
@@ -398,3 +419,42 @@ fn animated_gif_two_bounds_fit_inside_without_distortion() {
     assert_eq!(240, dims.width);
     assert_eq!(180, dims.height);
 }
+
+#[test]
+fn the_gif_resize_work_budget_counts_output_pixels_across_every_frame() {
+    let thumbnail = GifResizeDims {
+        width: 480,
+        height: 270,
+    };
+    assert!(thumbnail.fits_work_budget(129));
+    assert!(!thumbnail.fits_work_budget(130));
+    assert!(!thumbnail.fits_work_budget(739));
+    let still = GifResizeDims {
+        width: 4096,
+        height: 4096,
+    };
+    assert!(still.fits_work_budget(1));
+    assert!(still.fits_work_budget(0));
+    assert!(!still.fits_work_budget(2));
+}
+
+#[test]
+fn an_animated_gif_resize_over_the_work_budget_passes_through_byte_identically() {
+    let mut gif = animated_gif_frames(2, 20);
+    gif[6..8].copy_from_slice(&4096u16.to_le_bytes());
+    gif[8..10].copy_from_slice(&4096u16.to_le_bytes());
+    let passed_through = transform_image(
+        &gif,
+        &ImageOptions {
+            width: Some(4095),
+            format: OutputFormat::GIF,
+            quality: ImageQuality::Lossless,
+            animation: animated_mode(),
+            ..Default::default()
+        },
+    )
+    .expect("a gif too costly to resize still passes through");
+
+    assert_eq!("image/gif", passed_through.content_type);
+    assert_eq!(gif, passed_through.bytes);
+}

fluxer_media_proxy/src/media_process/tests/encoding.rs

@@ -25,12 +25,45 @@ fn webp_suffix(requested: EncodeEffort) -> String {
         ResolvedImageQuality::High,
         None,
         Some(requested),
+        0,
     )
     .expect("webp output suffix")
     .into_string()
     .expect("utf8 output suffix")
 }
 
+#[test]
+fn png_compression_drops_for_outputs_above_a_megapixel() {
+    let png_suffix = |output_pixels| {
+        output_suffix(
+            OutputFormat::PNG,
+            ResolvedImageQuality::High,
+            None,
+            None,
+            output_pixels,
+        )
+        .expect("png output suffix")
+        .into_string()
+        .expect("utf8 output suffix")
+    };
+    assert_eq!(
+        ".png[strip,compression=9,filter=all]",
+        png_suffix(160 * 160)
+    );
+    assert_eq!(
+        ".png[strip,compression=9,filter=all]",
+        png_suffix(1024 * 1024)
+    );
+    assert_eq!(
+        ".png[strip,compression=6,filter=all]",
+        png_suffix(1024 * 1024 + 1)
+    );
+    assert_eq!(
+        ".png[strip,compression=6,filter=all]",
+        png_suffix(3024 * 4032)
+    );
+}
+
 #[test]
 fn animated_webp_default_effort_matches_fast_tier() {
     assert_eq!(2, ResolvedImageQuality::Low.default_effort(true));

fluxer_media_proxy/src/media_process/tests/fixtures.rs

@@ -69,6 +69,12 @@ pub(super) fn png_ihdr_dimensions(ihdr: &[u8]) -> Option<(u32, u32)> {
     ))
 }
 
+pub(super) fn png_zlib_level(bytes: &[u8]) -> Option<u8> {
+    let chunks = parse_png_chunks(bytes).ok()?;
+    let idat = chunks.iter().find(|chunk| chunk.kind == *b"IDAT")?;
+    Some(idat.data.get(1)? >> 6)
+}
+
 pub(super) fn test_media_limits() -> MediaLimits {
     MediaLimits::default_from_config()
 }

fluxer_media_proxy/src/media_process/tests/metadata.rs

@@ -267,7 +267,7 @@ fn metadata_refuses_jxl_above_the_decode_pixel_cap() {
             .await
             .unwrap_err()
         });
-    assert_eq!(MediaError::InvalidImageDimensions, err);
+    assert_eq!(MediaError::TooLargeToProcess, err);
 }
 
 #[test]

fluxer_media_proxy/src/media_process/tests/transform.rs

@@ -4,12 +4,13 @@ use super::super::image_probe::load_image;
 use super::super::native_runtime::ensure_vips_init;
 use super::super::transform::source_supports_pages;
 use super::super::{ImageOptions, MediaError, NativeCost, image_transform_cost};
-use super::fixtures::{animated_mode, metadata_value, transform_image};
+use super::fixtures::{animated_mode, metadata_value, png_zlib_level, transform_image};
 use crate::{
     mime, native,
     output_format::OutputFormat,
     test_fixtures::{
-        JXL_64_LOSSY_ALPHA, JXL_4100_LOSSY_ALPHA, png_dimensions, synthetic_bmp, synthetic_png,
+        JPEG_ROTATED_MISMATCHED_SCAN, JXL_64_LOSSY_ALPHA, JXL_4100_LOSSY_ALPHA, png_dimensions,
+        synthetic_bmp, synthetic_png,
     },
 };
 
@@ -118,7 +119,7 @@ fn refuses_jxl_above_the_decode_pixel_cap_before_decoding() {
             },
         )
         .unwrap_err();
-        assert_eq!(MediaError::InvalidImageDimensions, err);
+        assert_eq!(MediaError::TooLargeToProcess, err);
     }
 }
 
@@ -147,7 +148,7 @@ fn refuses_full_size_svg_raster_above_the_static_output_cap() {
         },
     )
     .unwrap_err();
-    assert_eq!(MediaError::InvalidImageDimensions, err);
+    assert_eq!(MediaError::TooLargeToProcess, err);
     let err = transform_image(
         svg,
         &ImageOptions {
@@ -157,7 +158,7 @@ fn refuses_full_size_svg_raster_above_the_static_output_cap() {
         },
     )
     .unwrap_err();
-    assert_eq!(MediaError::InvalidImageDimensions, err);
+    assert_eq!(MediaError::TooLargeToProcess, err);
 }
 
 #[test]
@@ -213,3 +214,29 @@ fn transform_cost_charges_svg_at_its_output_size() {
     };
     assert!(cost < 1 << 20, "cost {cost}");
 }
+
+#[test]
+fn transforms_a_rotated_jpeg_with_mismatched_scan_headers_at_full_size() {
+    let out = transform_image(
+        JPEG_ROTATED_MISMATCHED_SCAN,
+        &ImageOptions {
+            format: OutputFormat::PNG,
+            ..Default::default()
+        },
+    )
+    .unwrap();
+    assert_eq!(Some((768, 1024)), png_dimensions(&out.bytes));
+}
+
+#[test]
+fn png_outputs_above_a_megapixel_use_default_zlib_compression() {
+    let png_options = ImageOptions {
+        format: OutputFormat::PNG,
+        ..Default::default()
+    };
+    let small = transform_image(&synthetic_bmp(160, 160), &png_options).unwrap();
+    assert_eq!(Some(3), png_zlib_level(&small.bytes));
+    let large = transform_image(&synthetic_bmp(1100, 1000), &png_options).unwrap();
+    assert_eq!(Some((1100, 1000)), png_dimensions(&large.bytes));
+    assert_eq!(Some(2), png_zlib_level(&large.bytes));
+}

fluxer_media_proxy/src/media_process/tests/video.rs

@@ -4,12 +4,13 @@ use super::super::{
     MediaError, VideoThumbnailOptions, extract_video_thumbnail,
     extract_video_thumbnail_with_options,
 };
-use super::fixtures::{decode_rgba, metadata_value, test_media_limits};
+use super::fixtures::{decode_rgba, metadata_value, png_zlib_level, test_media_limits};
 use crate::{
     image_quality::ResolvedImageQuality,
     output_format::OutputFormat,
     test_fixtures::{
-        ffmpeg_gen_media, ffmpeg_gen_mp4, ffmpeg_gen_rotated_mp4, ffmpeg_mirror_mp4, png_dimensions,
+        ffmpeg_gen_media, ffmpeg_gen_mp4, ffmpeg_gen_rotated_mp4, ffmpeg_mirror_mp4,
+        fixture_h264_mp4, png_dimensions,
     },
 };
 
@@ -35,6 +36,14 @@ fn color_tagged_video(primaries: &str, transfer: &str, matrix: &str) -> Option<V
     )
 }
 
+#[test]
+fn a_png_video_thumbnail_uses_default_zlib_compression() {
+    let thumb =
+        extract_video_thumbnail(&fixture_h264_mp4(), OutputFormat::PNG, &test_media_limits())
+            .expect("h264 video should thumbnail");
+    assert_eq!(Some(2), png_zlib_level(&thumb.bytes));
+}
+
 #[test]
 fn video_thumbnail_accepts_smpte170m_sd_primaries() {
     let Some(video) = color_tagged_video("smpte170m", "smpte170m", "smpte170m") else {

fluxer_media_proxy/src/media_process/transform.rs

@@ -161,10 +161,23 @@ pub fn transform_image(
         && !options.wants_cover_crop()
         && sniffed.mime == "image/gif"
     {
-        let bytes = if let Some(dims) = gif_resize_dims(sniffed, options) {
-            resize_animated_gif_with_ffmpeg(input, dims, options, media_limits)?
-        } else {
-            input.to_vec()
+        let bytes = match gif_resize_dims(sniffed, options) {
+            Some(dims) => {
+                let frames = probe_animated(media_limits, input)?.map_or(1, |probe| probe.pages);
+                if dims.fits_work_budget(frames) {
+                    resize_animated_gif_with_ffmpeg(input, dims, options, media_limits)?
+                } else {
+                    tracing::info!(
+                        input_bytes = input.len(),
+                        width = dims.width,
+                        height = dims.height,
+                        frames,
+                        "animated gif resize over the work budget, serving the source"
+                    );
+                    input.to_vec()
+                }
+            }
+            None => input.to_vec(),
         };
         return Ok(ProcessedMedia {
             bytes,
@@ -382,13 +395,9 @@ pub fn transform_image(
     }
 
     let loader_options = if animated && source_supports_pages(sniffed.mime) {
-        if sniffed.mime == "image/jpeg" {
-            "n=-1,access=sequential"
-        } else {
-            "n=-1,access=sequential,fail=true"
-        }
+        "n=-1,access=sequential,fail=true"
     } else if sniffed.mime == "image/jpeg" {
-        "access=sequential"
+        ""
     } else {
         "access=sequential,fail=true"
     };

fluxer_media_proxy/src/media_process/transform_plan.rs

@@ -13,12 +13,14 @@ use std::ffi::CString;
 const AUTO_LOSSLESS_PALETTE_ANIMATION_MAX_BYTES: usize = 4 * 1024 * 1024;
 const AUTO_LOSSLESS_PALETTE_ANIMATION_MAX_PIXELS: usize = 16 * 1024 * 1024;
 const VIPS_WEBP_MAX_EFFORT: u8 = 6;
+const PNG_MAX_COMPRESSION_MAX_PIXELS: u64 = 1024 * 1024;
 
 pub(super) fn output_suffix(
     format: OutputFormat,
     quality: ResolvedImageQuality,
     page_height: Option<c_int>,
     effort_override: Option<EncodeEffort>,
+    output_pixels: u64,
 ) -> Result<CString, MediaError> {
     let q = quality.encoder_quality();
     let animated = page_height.is_some();
@@ -43,12 +45,19 @@ pub(super) fn output_suffix(
                 ),
             }
         }
-        OutputFormat::PNG | OutputFormat::APNG => match page_height {
-            Some(ph) if format == OutputFormat::APNG => {
-                format!(".png[strip,compression=9,filter=all,page-height={ph}]")
+        OutputFormat::PNG | OutputFormat::APNG => {
+            let compression = if output_pixels > PNG_MAX_COMPRESSION_MAX_PIXELS {
+                6
+            } else {
+                9
+            };
+            match page_height {
+                Some(ph) if format == OutputFormat::APNG => {
+                    format!(".png[strip,compression={compression},filter=all,page-height={ph}]")
+                }
+                _ => format!(".png[strip,compression={compression},filter=all]"),
             }
-            _ => ".png[strip,compression=9,filter=all]".to_owned(),
-        },
+        }
         OutputFormat::GIF => match page_height {
             Some(ph) => {
                 format!(".gif[strip,dither=1.0,effort=7,interframe_maxerror=8.0,page-height={ph}]")

fluxer_media_proxy/src/media_process/video_thumbnail.rs

@@ -54,7 +54,7 @@ pub fn extract_video_thumbnail_with_options(
     } = options;
     let runtime = vips_runtime()?;
     let (max_width, max_height) = native_thumbnail_bounds(width, height, media_limits)?;
-    let suffix = output_suffix(format, quality, None, None)?;
+    let suffix = output_suffix(format, quality, None, None, u64::MAX)?;
     let mut out_ptr: *mut c_void = ptr::null_mut();
     let mut out_size: size_t = 0;
     let mut out_capacity: size_t = 0;