From 86e29b61e3dfa9eca64d3365f68ac40ca656f682 Mon Sep 17 00:00:00 2001 From: Hampus Date: Mon, 12 Oct 2026 00:05:14 +0200 Subject: [PATCH] fix(media-proxy): serve large simple jxl, fit full size (#3408) --- fluxer_media_proxy/src/component_tests.rs | 12 +- fluxer_media_proxy/src/constants.rs | 1 + fluxer_media_proxy/src/media_limits.rs | 13 ++ .../src/media_process/image_probe.rs | 41 +++-- .../src/media_process/loaded_image.rs | 8 +- fluxer_media_proxy/src/media_process/mod.rs | 2 + .../src/media_process/native_cost.rs | 37 ++++- .../src/media_process/tests/animated_apng.rs | 4 +- .../src/media_process/tests/heif.rs | 16 +- .../src/media_process/tests/metadata.rs | 56 ++++--- .../src/media_process/tests/transform.rs | 150 ++++++++++++++++-- .../src/media_process/transform.rs | 12 +- fluxer_media_proxy/src/native/mod.rs | 2 + .../src/native_shim/vips_image.c | 8 +- .../src/server/format_policy.rs | 7 +- .../src/server/transform/mod.rs | 3 +- .../src/server/transform/tests.rs | 30 +++- .../src/test_fixtures/images.rs | 4 + .../test_fixtures/jxl_3000_lossy_2_frames.jxl | Bin 0 -> 2353 bytes .../test_fixtures/jxl_4100_lossless_rgb16.jxl | Bin 0 -> 44903 bytes .../jxl_4100_lossy_rgb_depth.jxl | Bin 0 -> 8621 bytes .../test_fixtures/jxl_7000x4000_lossy_rgb.jxl | Bin 0 -> 5777 bytes fluxer_media_proxy/src/test_fixtures/mod.rs | 9 +- fluxer_media_proxy/src/vips_shim.h | 2 +- 24 files changed, 345 insertions(+), 72 deletions(-) create mode 100644 fluxer_media_proxy/src/test_fixtures/jxl_3000_lossy_2_frames.jxl create mode 100644 fluxer_media_proxy/src/test_fixtures/jxl_4100_lossless_rgb16.jxl create mode 100644 fluxer_media_proxy/src/test_fixtures/jxl_4100_lossy_rgb_depth.jxl create mode 100644 fluxer_media_proxy/src/test_fixtures/jxl_7000x4000_lossy_rgb.jxl diff --git a/fluxer_media_proxy/src/component_tests.rs b/fluxer_media_proxy/src/component_tests.rs index 69fc95e58..f485a4cb0 100644 --- a/fluxer_media_proxy/src/component_tests.rs +++ b/fluxer_media_proxy/src/component_tests.rs @@ -171,11 +171,13 @@ fn gif_animation_status(input: &[u8]) -> NativeStatus { }) } -fn probe_animated_status(input: &[u8]) -> (NativeStatus, c_int, c_int, c_int) { +fn probe_animated_status(input: &[u8]) -> (NativeStatus, c_int, c_int, c_int, c_int) { ensure_vips_init().expect("libvips initialises"); let mut width = c_int::MAX; let mut height = c_int::MAX; let mut pages = c_int::MAX; + let mut bands = c_int::MAX; + let mut format = c_int::MAX; let status = NativeStatus::from_code(unsafe { native::fluxer_vips_probe_animated( input.as_ptr().cast(), @@ -183,10 +185,12 @@ fn probe_animated_status(input: &[u8]) -> (NativeStatus, c_int, c_int, c_int) { &mut width, &mut height, &mut pages, + &mut bands, + &mut format, ) }); unsafe { native::fluxer_vips_error_clear() }; - (status, width, height, pages) + (status, width, height, pages, bands) } fn apng_decode_status(input: &[u8]) -> (NativeStatus, *mut native::VipsImage, u32) { @@ -286,16 +290,18 @@ fn every_remaining_native_validator_zeroes_its_out_params_on_failure() { "input {bytes:?}" ); - let (status, width, height, pages) = probe_animated_status(&bytes); + let (status, width, height, pages, bands) = probe_animated_status(&bytes); assert_eq!(decodable, status == NativeStatus::Ok, "input {bytes:?}"); if decodable { assert_eq!(1, width, "input {bytes:?}"); assert_eq!(1, height, "input {bytes:?}"); assert_eq!(1, pages, "input {bytes:?}"); + assert!(bands > 0, "input {bytes:?}"); } else { assert_eq!(0, width, "input {bytes:?}"); assert_eq!(0, height, "input {bytes:?}"); assert_eq!(0, pages, "input {bytes:?}"); + assert_eq!(0, bands, "input {bytes:?}"); } let (status, out, num_plays) = apng_decode_status(&bytes); diff --git a/fluxer_media_proxy/src/constants.rs b/fluxer_media_proxy/src/constants.rs index e509831c2..5b475e08f 100644 --- a/fluxer_media_proxy/src/constants.rs +++ b/fluxer_media_proxy/src/constants.rs @@ -15,6 +15,7 @@ pub const DEFAULT_IMAGE_SIZE: u32 = 128; pub const MAX_ANIMATED_FRAMES_DEFAULT: u32 = 20_000; pub const MAX_ANIMATED_TOTAL_PIXELS_DEFAULT: usize = 4 * MAX_MEDIA_IMAGE_PIXELS_DEFAULT; pub const MAX_JXL_DECODE_PIXELS: usize = 4096 * 4096; +pub const MAX_JXL_SIMPLE_DECODE_PIXELS: usize = 8192 * 6144; pub const MAX_STATIC_OUTPUT_PIXELS: usize = 6144 * 4096; pub const MAX_ANIMATED_OUTPUT_PIXELS: usize = 16 << 20; pub const SVG_RASTER_MAX_DIMENSION: u32 = 4096; diff --git a/fluxer_media_proxy/src/media_limits.rs b/fluxer_media_proxy/src/media_limits.rs index fdebce38e..d6e189d3a 100644 --- a/fluxer_media_proxy/src/media_limits.rs +++ b/fluxer_media_proxy/src/media_limits.rs @@ -12,6 +12,7 @@ pub struct MediaLimits { animated_frames: u32, animated_total_pixels: usize, max_media_proxy_bytes: usize, + native_memory_budget_bytes: usize, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -59,9 +60,17 @@ impl MediaLimits { animated_frames, animated_total_pixels, max_media_proxy_bytes, + native_memory_budget_bytes: usize::MAX, }) } + pub(crate) fn with_native_memory_budget(self, native_memory_budget_bytes: usize) -> Self { + Self { + native_memory_budget_bytes, + ..self + } + } + pub fn default_from_config() -> Self { Self::new(MediaLimitConfiguration { image_dimension: MAX_MEDIA_IMAGE_DIMENSION_DEFAULT, @@ -92,4 +101,8 @@ impl MediaLimits { pub fn max_media_proxy_bytes(&self) -> usize { self.max_media_proxy_bytes } + + pub fn native_memory_budget_bytes(&self) -> usize { + self.native_memory_budget_bytes + } } diff --git a/fluxer_media_proxy/src/media_process/image_probe.rs b/fluxer_media_proxy/src/media_process/image_probe.rs index 790ced846..6fa6a237d 100644 --- a/fluxer_media_proxy/src/media_process/image_probe.rs +++ b/fluxer_media_proxy/src/media_process/image_probe.rs @@ -15,9 +15,24 @@ pub(crate) fn probe_animated( media_limits: &MediaLimits, input: &[u8], ) -> Result, MediaError> { + Ok( + probe_header(media_limits, input)?.map(|dims| AnimatedProbe { + width: dims.width as c_int, + height: dims.height as c_int, + pages: dims.pages as c_int, + }), + ) +} + +fn probe_header( + media_limits: &MediaLimits, + input: &[u8], +) -> Result, MediaError> { let mut width = 0; let mut height = 0; let mut pages = 0; + let mut bands = 0; + let mut format = 0; let rc = unsafe { native::fluxer_vips_probe_animated( input.as_ptr().cast(), @@ -25,6 +40,8 @@ pub(crate) fn probe_animated( &mut width, &mut height, &mut pages, + &mut bands, + &mut format, ) }; match native::NativeStatus::from_code(rc) { @@ -39,14 +56,16 @@ pub(crate) fn probe_animated( .expect("non-success native status must map to an error")); } } - if pages <= 0 || width <= 0 || height <= 0 { + if pages <= 0 || width <= 0 || height <= 0 || bands <= 0 { return Err(MediaError::InvalidImageDimensions); } validate_animation_geometry(media_limits, width as u32, height as u32, pages as u32)?; - Ok(Some(AnimatedProbe { - width, - height, - pages, + Ok(Some(ImageDimensions { + width: width as u32, + height: height as u32, + pages: pages as u32, + bands: bands as u32, + eight_bit: format == unsafe { native::fluxer_vips_format_uchar }, })) } @@ -55,12 +74,8 @@ pub(crate) fn probe_image_dims( input: &[u8], ) -> Result { ensure_vips_init()?; - if let Some(probe) = probe_animated(media_limits, input)? { - return Ok(ImageDimensions { - width: probe.width as u32, - height: probe.height as u32, - pages: probe.pages as u32, - }); + if let Some(dims) = probe_header(media_limits, input)? { + return Ok(dims); } if mime::sniff(input).mime == "image/bmp" && let Some(image) = try_decode_bmp(input, media_limits)? @@ -89,6 +104,10 @@ fn probe_loaded_image_dims( width, height, pages, + bands: unsafe { native::fluxer_vips_image_get_bands(image.as_ptr()) } as u32, + eight_bit: unsafe { + native::fluxer_vips_image_get_format(image.as_ptr()) == native::fluxer_vips_format_uchar + }, }) } diff --git a/fluxer_media_proxy/src/media_process/loaded_image.rs b/fluxer_media_proxy/src/media_process/loaded_image.rs index 3ab65beca..0351ae71f 100644 --- a/fluxer_media_proxy/src/media_process/loaded_image.rs +++ b/fluxer_media_proxy/src/media_process/loaded_image.rs @@ -66,7 +66,13 @@ pub(crate) fn validate_jxl_decode_pixels( let pixels = (dims.width as usize) .saturating_mul(dims.height as usize) .saturating_mul(dims.pages.max(1) as usize); - if pixels > constants::MAX_JXL_DECODE_PIXELS { + let simple = dims.pages == 1 && matches!(dims.bands, 1 | 3) && dims.eight_bit; + let cap = if simple { + constants::MAX_JXL_SIMPLE_DECODE_PIXELS + } else { + constants::MAX_JXL_DECODE_PIXELS + }; + if pixels > cap { return Err(MediaError::TooLargeToProcess); } Ok(()) diff --git a/fluxer_media_proxy/src/media_process/mod.rs b/fluxer_media_proxy/src/media_process/mod.rs index aa79933bc..1ba9ef7f8 100644 --- a/fluxer_media_proxy/src/media_process/mod.rs +++ b/fluxer_media_proxy/src/media_process/mod.rs @@ -138,6 +138,8 @@ pub struct ImageDimensions { pub width: u32, pub height: u32, pub pages: u32, + pub bands: u32, + pub eight_bit: bool, } #[derive(Clone, Copy, Debug, Error, Eq, PartialEq)] diff --git a/fluxer_media_proxy/src/media_process/native_cost.rs b/fluxer_media_proxy/src/media_process/native_cost.rs index c6d246d6d..4abd6eef2 100644 --- a/fluxer_media_proxy/src/media_process/native_cost.rs +++ b/fluxer_media_proxy/src/media_process/native_cost.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use super::MediaError; use super::image_probe::probe_image_dims; use super::loaded_image::static_output_pixels; use super::native_runtime::ensure_vips_init; +use super::{ImageDimensions, MediaError}; use crate::{ constants, image_quality::ImageQuality, image_transform::ImageOptions, media_limits::MediaLimits, mime, output_format::OutputFormat, @@ -48,7 +48,8 @@ pub fn image_transform_cost( let sniffed_mime = mime::sniff(input).mime; let dims = probe_image_dims(media_limits, input)?; let source_pixels = dims.width as usize * dims.height as usize; - let output_pixels = static_output_pixels(options, dims); + let options = fit_jxl_output(input.len(), sniffed_mime, options, dims, media_limits); + let output_pixels = static_output_pixels(&options, dims); let encode = encode_bytes_per_pixel(options.format, options.quality); let pixel_cost = if options.is_animated() && dims.pages > 1 { source_pixels.max(output_pixels) * RGBA_BYTES_PER_PIXEL * ANIMATED_CANVAS_BUFFERS @@ -59,6 +60,38 @@ pub fn image_transform_cost( Ok(NativeCost::Bytes(input.len().saturating_add(pixel_cost))) } +pub(super) fn fit_jxl_output( + input_len: usize, + sniffed_mime: &str, + options: &ImageOptions, + dims: ImageDimensions, + media_limits: &MediaLimits, +) -> ImageOptions { + let mut fitted = *options; + if sniffed_mime != "image/jxl" || options.is_animated() { + return fitted; + } + let source_pixels = dims.width as usize * dims.height as usize; + let affordable_pixels = media_limits + .native_memory_budget_bytes() + .saturating_sub(input_len) + .saturating_sub(source_pixels * decode_bytes_per_pixel(sniffed_mime)) + / encode_bytes_per_pixel(options.format, options.quality); + let max_pixels = affordable_pixels.min(constants::MAX_STATIC_OUTPUT_PIXELS); + if max_pixels == 0 || static_output_pixels(options, dims) <= max_pixels { + return fitted; + } + let (width, height) = match (options.wants_cover_crop(), options.width, options.height) { + (true, Some(width), Some(height)) => (width, height), + _ => (dims.width, dims.height), + }; + let (width, height) = (f64::from(width), f64::from(height)); + let shrink = (max_pixels as f64 / (width * height)).sqrt(); + fitted.width = Some(((width * shrink).floor() as u32).max(1)); + fitted.height = Some(((height * shrink).floor() as u32).max(1)); + fitted +} + pub fn av_native_cost(input_len: usize) -> NativeCost { NativeCost::UpToBudget( input_len diff --git a/fluxer_media_proxy/src/media_process/tests/animated_apng.rs b/fluxer_media_proxy/src/media_process/tests/animated_apng.rs index 9098d095c..6a2d8faa7 100644 --- a/fluxer_media_proxy/src/media_process/tests/animated_apng.rs +++ b/fluxer_media_proxy/src/media_process/tests/animated_apng.rs @@ -57,7 +57,9 @@ fn animated_apng_input_transforms_through_ffmpeg_decode_path() { ImageDimensions { width: 8, height: 8, - pages: 1 + pages: 1, + bands: 3, + eight_bit: true, }, probe_image_dims(&test_media_limits(), &static_webp.bytes).unwrap() ); diff --git a/fluxer_media_proxy/src/media_process/tests/heif.rs b/fluxer_media_proxy/src/media_process/tests/heif.rs index 8ca855a47..4033535d6 100644 --- a/fluxer_media_proxy/src/media_process/tests/heif.rs +++ b/fluxer_media_proxy/src/media_process/tests/heif.rs @@ -37,7 +37,9 @@ fn animated_heic_sequence_decodes_every_frame_and_encodes_an_animation() { ImageDimensions { width: 64, height: 64, - pages: 4 + pages: 4, + bands: 3, + eight_bit: true, }, probe_image_dims(&test_media_limits(), &animated.bytes).expect("probes webp") ); @@ -85,7 +87,9 @@ fn still_heic_decodes_as_a_single_frame_when_animation_is_requested() { ImageDimensions { width: 64, height: 64, - pages: 1 + pages: 1, + bands: 3, + eight_bit: true, }, probe_image_dims(&test_media_limits(), &still.bytes).expect("probes webp") ); @@ -113,7 +117,9 @@ fn avif_direct_decode_applies_resize_and_crop() { ImageDimensions { width: 32, height: 24, - pages: 1 + pages: 1, + bands: 3, + eight_bit: true, }, probe_image_dims(&test_media_limits(), &resized.bytes).unwrap() ); @@ -135,7 +141,9 @@ fn avif_direct_decode_applies_resize_and_crop() { ImageDimensions { width: 32, height: 32, - pages: 1 + pages: 1, + bands: 3, + eight_bit: true, }, probe_image_dims(&test_media_limits(), &cropped.bytes).unwrap() ); diff --git a/fluxer_media_proxy/src/media_process/tests/metadata.rs b/fluxer_media_proxy/src/media_process/tests/metadata.rs index b3b910806..431c0e95a 100644 --- a/fluxer_media_proxy/src/media_process/tests/metadata.rs +++ b/fluxer_media_proxy/src/media_process/tests/metadata.rs @@ -14,9 +14,11 @@ use crate::{ nsfw::{NSFWClient, NSFWPolicy}, output_format::OutputFormat, test_fixtures::{ - fixture_audio_mp3_with_png_cover_art, fixture_audio_mp4_with_attached_picture, - fixture_audio_only_mp4, fixture_h264_mp4, fixture_mkv_with_png_video_stream, - fixture_mp4_with_undecodable_video, synthetic_png, synthetic_wav, + JXL_3000_LOSSY_2_FRAMES, JXL_4100_LOSSLESS_RGB16, JXL_4100_LOSSY_ALPHA, + JXL_4100_LOSSY_RGB_DEPTH, JXL_7000X4000_LOSSY_RGB, fixture_audio_mp3_with_png_cover_art, + fixture_audio_mp4_with_attached_picture, fixture_audio_only_mp4, fixture_h264_mp4, + fixture_mkv_with_png_video_stream, fixture_mp4_with_undecodable_video, synthetic_png, + synthetic_wav, }, }; use std::{io::Write as _, sync::Arc}; @@ -252,22 +254,38 @@ fn metadata_succeeds_without_a_placeholder_when_thumbhash_generation_fails() { #[test] fn metadata_refuses_jxl_above_the_decode_pixel_cap() { - let err = tokio::runtime::Builder::new_current_thread() - .build() - .unwrap() - .block_on(async { - metadata_json_with_options( - crate::test_fixtures::JXL_4100_LOSSY_ALPHA, - "big.jxl", - MetadataOptions::default(), - &test_media_limits(), - &NSFWClient::disabled(), - &TransformMetrics::new(), - ) - .await - .unwrap_err() - }); - assert_eq!(MediaError::TooLargeToProcess, err); + for jxl in [ + JXL_4100_LOSSY_ALPHA, + JXL_4100_LOSSY_RGB_DEPTH, + JXL_4100_LOSSLESS_RGB16, + JXL_3000_LOSSY_2_FRAMES, + ] { + let err = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(async { + metadata_json_with_options( + jxl, + "big.jxl", + MetadataOptions::default(), + &test_media_limits(), + &NSFWClient::disabled(), + &TransformMetrics::new(), + ) + .await + .unwrap_err() + }); + assert_eq!(MediaError::TooLargeToProcess, err); + } +} + +#[test] +fn metadata_admits_8_bit_rgb_jxl_above_the_decode_pixel_cap() { + let value = metadata_value(JXL_7000X4000_LOSSY_RGB, "big.jxl"); + assert_eq!("image/jxl", value["content_type"]); + assert_eq!(7000, value["width"]); + assert_eq!(4000, value["height"]); + assert!(value["placeholder"].as_str().is_some_and(|p| !p.is_empty())); } #[test] diff --git a/fluxer_media_proxy/src/media_process/tests/transform.rs b/fluxer_media_proxy/src/media_process/tests/transform.rs index 4812a4a00..3f106423d 100644 --- a/fluxer_media_proxy/src/media_process/tests/transform.rs +++ b/fluxer_media_proxy/src/media_process/tests/transform.rs @@ -1,16 +1,22 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use super::super::image_probe::load_image; +use super::super::image_probe::{load_image, probe_image_dims}; +use super::super::loaded_image::validate_jxl_decode_pixels; use super::super::native_runtime::ensure_vips_init; use super::super::transform::source_supports_pages; -use super::super::{ImageOptions, MediaError, NativeCost, image_transform_cost}; -use super::fixtures::{animated_mode, metadata_value, png_zlib_level, transform_image}; +use super::super::{ImageDimensions, ImageOptions, MediaError, NativeCost, image_transform_cost}; +use super::fixtures::{ + animated_mode, metadata_value, png_zlib_level, test_media_limits, transform_image, +}; use crate::{ + constants, + metrics::transform::TransformMetrics, mime, native, output_format::OutputFormat, test_fixtures::{ - JPEG_ROTATED_MISMATCHED_SCAN, JXL_64_LOSSY_ALPHA, JXL_4100_LOSSY_ALPHA, png_dimensions, - synthetic_bmp, synthetic_png, + JPEG_ROTATED_MISMATCHED_SCAN, JXL_64_LOSSY_ALPHA, JXL_3000_LOSSY_2_FRAMES, + JXL_4100_LOSSLESS_RGB16, JXL_4100_LOSSY_ALPHA, JXL_4100_LOSSY_RGB_DEPTH, + JXL_7000X4000_LOSSY_RGB, png_dimensions, synthetic_bmp, synthetic_png, }, }; @@ -109,20 +115,132 @@ fn source_supports_pages_matches_libvips_loader_list() { #[test] fn refuses_jxl_above_the_decode_pixel_cap_before_decoding() { - for width in [None, Some(128)] { - let err = transform_image( - JXL_4100_LOSSY_ALPHA, - &ImageOptions { - width, - format: OutputFormat::WebP, - ..Default::default() - }, - ) - .unwrap_err(); - assert_eq!(MediaError::TooLargeToProcess, err); + for jxl in [ + JXL_4100_LOSSY_ALPHA, + JXL_4100_LOSSY_RGB_DEPTH, + JXL_4100_LOSSLESS_RGB16, + JXL_3000_LOSSY_2_FRAMES, + ] { + for width in [None, Some(128)] { + let err = transform_image( + jxl, + &ImageOptions { + width, + format: OutputFormat::WebP, + ..Default::default() + }, + ) + .unwrap_err(); + assert_eq!(MediaError::TooLargeToProcess, err); + } } } +#[test] +fn jxl_decode_cap_depends_on_the_header_layout() { + let dims = |width, height, pages, bands, eight_bit| ImageDimensions { + width, + height, + pages, + bands, + eight_bit, + }; + let simple_cap = (8192, 6144); + let complex_cap = (4096, 4096); + for bands in [1, 3] { + let (width, height) = simple_cap; + assert_eq!( + Ok(()), + validate_jxl_decode_pixels("image/jxl", dims(width, height, 1, bands, true)) + ); + assert_eq!( + Err(MediaError::TooLargeToProcess), + validate_jxl_decode_pixels("image/jxl", dims(width, height + 1, 1, bands, true)) + ); + } + for (pages, bands, eight_bit) in [(1, 2, true), (1, 4, true), (1, 3, false), (2, 3, true)] { + let (width, height) = complex_cap; + assert_eq!( + Ok(()), + validate_jxl_decode_pixels( + "image/jxl", + dims(width, height / pages, pages, bands, eight_bit) + ) + ); + assert_eq!( + Err(MediaError::TooLargeToProcess), + validate_jxl_decode_pixels( + "image/jxl", + dims(width, height / pages + 1, pages, bands, eight_bit) + ) + ); + } + assert_eq!( + Ok(()), + validate_jxl_decode_pixels("image/png", dims(16384, 16384, 1, 4, false)) + ); +} + +#[test] +fn thumbnails_8_bit_rgb_jxl_above_the_decode_pixel_cap() { + let out = transform_image( + JXL_7000X4000_LOSSY_RGB, + &ImageOptions { + width: Some(550), + format: OutputFormat::WebP, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!("image/webp", out.content_type); + let dims = probe_image_dims(&test_media_limits(), &out.bytes).unwrap(); + assert_eq!((550, 314), (dims.width, dims.height)); +} + +#[test] +fn full_size_jxl_above_the_static_output_cap_is_downscaled_to_fit() { + let options = ImageOptions { + format: OutputFormat::WebP, + ..Default::default() + }; + let out = transform_image(JXL_7000X4000_LOSSY_RGB, &options).unwrap(); + assert_eq!("image/webp", out.content_type); + let dims = probe_image_dims(&test_media_limits(), &out.bytes).unwrap(); + assert_eq!((6636, 3792), (dims.width, dims.height)); + assert!(dims.width as usize * dims.height as usize <= constants::MAX_STATIC_OUTPUT_PIXELS); + + let cost = + image_transform_cost(JXL_7000X4000_LOSSY_RGB, &options, &test_media_limits()).unwrap(); + assert_eq!( + NativeCost::Bytes(JXL_7000X4000_LOSSY_RGB.len() + 7000 * 4000 * 40 + 6636 * 3792 * 28), + cost + ); +} + +#[test] +fn full_size_jxl_is_downscaled_to_fit_the_native_memory_budget() { + let options = ImageOptions { + format: OutputFormat::WebP, + ..Default::default() + }; + let decode = JXL_7000X4000_LOSSY_RGB.len() + 7000 * 4000 * 40; + let budget = decode + 10_000_000 * 28; + let media_limits = test_media_limits().with_native_memory_budget(budget); + let out = super::super::transform_image( + JXL_7000X4000_LOSSY_RGB, + &options, + &media_limits, + &TransformMetrics::new(), + ) + .unwrap(); + let dims = probe_image_dims(&test_media_limits(), &out.bytes).unwrap(); + assert_eq!((4183, 2390), (dims.width, dims.height)); + assert_eq!( + NativeCost::Bytes(decode + 4183 * 2390 * 28), + image_transform_cost(JXL_7000X4000_LOSSY_RGB, &options, &media_limits).unwrap() + ); +} + #[test] fn transforms_jxl_within_the_decode_pixel_cap() { let out = transform_image( diff --git a/fluxer_media_proxy/src/media_process/transform.rs b/fluxer_media_proxy/src/media_process/transform.rs index ff797a812..b0efbf764 100644 --- a/fluxer_media_proxy/src/media_process/transform.rs +++ b/fluxer_media_proxy/src/media_process/transform.rs @@ -16,6 +16,7 @@ use super::loaded_image::{ static_output_pixels, validate_dimensions_u32, validate_jxl_decode_pixels, validate_static_output_pixels, validate_vips_image, }; +use super::native_cost::fit_jxl_output; use super::native_runtime::{clear_vips_error, ensure_vips_init, last_vips_error}; use super::transform_plan::{ AutoQualitySource, output_is_sdr, resolve_auto_quality, should_transform_animated_webp_direct, @@ -80,17 +81,18 @@ fn heif_source_may_be_a_sequence(sniffed: mime::SniffInfo) -> bool { } } -fn ensure_decode_cost_admissible( +fn admitted_transform_options( input: &[u8], sniffed_mime: &str, options: &ImageOptions, media_limits: &MediaLimits, -) -> Result<(), MediaError> { +) -> Result { let animated = options.is_animated(); if animated && sniffed_mime != "image/jxl" { - return Ok(()); + return Ok(*options); } let dims = probe_image_dims(media_limits, input)?; + let options = &fit_jxl_output(input.len(), sniffed_mime, options, dims, media_limits); let source_pixels = dims.width as usize * dims.height as usize; if source_pixels >= constants::LARGE_SOURCE_LOG_PIXELS { tracing::info!( @@ -125,7 +127,7 @@ fn ensure_decode_cost_admissible( "native transform refused by decode cost limits" ); } - admitted + admitted.map(|()| *options) } fn ensure_animated_output_admissible( @@ -175,7 +177,7 @@ pub fn transform_image( ensure_vips_init()?; let sniffed = mime::sniff(input); let animated = options.is_animated(); - ensure_decode_cost_admissible(input, sniffed.mime, options, media_limits)?; + let options = &admitted_transform_options(input, sniffed.mime, options, media_limits)?; let format = effective_transform_format(sniffed.mime, options.format, animated); let animated_avif = sniffed.mime == "image/avif" && sniffed.animated; let full_canvas_animation = animated diff --git a/fluxer_media_proxy/src/native/mod.rs b/fluxer_media_proxy/src/native/mod.rs index e397ab305..608e88b6b 100644 --- a/fluxer_media_proxy/src/native/mod.rs +++ b/fluxer_media_proxy/src/native/mod.rs @@ -130,6 +130,8 @@ unsafe extern "C" { width: *mut c_int, height: *mut c_int, pages: *mut c_int, + bands: *mut c_int, + format: *mut c_int, ) -> c_int; pub fn fluxer_apng_probe( buf: *const c_void, diff --git a/fluxer_media_proxy/src/native_shim/vips_image.c b/fluxer_media_proxy/src/native_shim/vips_image.c index 9a48c2199..9610d1955 100644 --- a/fluxer_media_proxy/src/native_shim/vips_image.c +++ b/fluxer_media_proxy/src/native_shim/vips_image.c @@ -41,13 +41,15 @@ void fluxer_vips_tune_for_server(int per_pipeline_threads) { vips_leak_set(FALSE); } -int fluxer_vips_probe_animated(const void *buf, size_t len, int *width, int *height, int *pages) { - if (width == NULL || height == NULL || pages == NULL) { +int fluxer_vips_probe_animated(const void *buf, size_t len, int *width, int *height, int *pages, int *bands, int *format) { + if (width == NULL || height == NULL || pages == NULL || bands == NULL || format == NULL) { return FLUXER_NATIVE_STATUS_CODEC_FAILURE; } *width = 0; *height = 0; *pages = 0; + *bands = 0; + *format = VIPS_FORMAT_NOTSET; if (buf == NULL || len == 0) return FLUXER_NATIVE_STATUS_CODEC_FAILURE; VipsImage *header = vips_image_new_from_buffer(buf, len, "n=1", NULL); if (header == NULL) { @@ -82,6 +84,8 @@ int fluxer_vips_probe_animated(const void *buf, size_t len, int *width, int *hei *width = display_width; *height = display_height; *pages = n_pages; + *bands = vips_image_get_bands(display_header); + *format = vips_image_get_format(display_header); g_object_unref(display_header); g_object_unref(header); return FLUXER_NATIVE_STATUS_OK; diff --git a/fluxer_media_proxy/src/server/format_policy.rs b/fluxer_media_proxy/src/server/format_policy.rs index eb5d054cc..de966d733 100644 --- a/fluxer_media_proxy/src/server/format_policy.rs +++ b/fluxer_media_proxy/src/server/format_policy.rs @@ -42,7 +42,12 @@ pub(in crate::server) fn transform_static_quality_default( ) -> ImageQuality { if matches!( source_ext, - Some(AssetExtension::Jpeg | AssetExtension::Heic | AssetExtension::Heif) + Some( + AssetExtension::Jpeg + | AssetExtension::Heic + | AssetExtension::Heif + | AssetExtension::Jxl + ) ) { ImageQuality::High } else { diff --git a/fluxer_media_proxy/src/server/transform/mod.rs b/fluxer_media_proxy/src/server/transform/mod.rs index 6b0b97238..b67064dc9 100644 --- a/fluxer_media_proxy/src/server/transform/mod.rs +++ b/fluxer_media_proxy/src/server/transform/mod.rs @@ -93,12 +93,13 @@ impl TransformRuntime { cfg: &Config, metrics: &Arc, ) -> anyhow::Result { - let limits = MediaLimits::default_from_config(); let memory_limit_bytes = process_memory_limit_bytes(); let memory_budget_bytes = native_memory_budget_bytes( memory_limit_bytes, cfg.media.transform_cache_capacity_bytes, ); + let limits = + MediaLimits::default_from_config().with_native_memory_budget(memory_budget_bytes); tracing::info!( memory_limit_bytes, transform_cache_bytes = cfg.media.transform_cache_capacity_bytes, diff --git a/fluxer_media_proxy/src/server/transform/tests.rs b/fluxer_media_proxy/src/server/transform/tests.rs index bebeca729..3b17e4a64 100644 --- a/fluxer_media_proxy/src/server/transform/tests.rs +++ b/fluxer_media_proxy/src/server/transform/tests.rs @@ -4,7 +4,10 @@ use super::*; use crate::{ constants::AssetExtension, server::format_policy::{default_transform_quality, transform_static_quality_default}, - test_fixtures::{JXL_4100_LOSSY_ALPHA, fixture_h264_mp4, fixture_jpeg, synthetic_png}, + test_fixtures::{ + JXL_64_LOSSY_ALPHA, JXL_4100_LOSSY_ALPHA, JXL_7000X4000_LOSSY_RGB, fixture_h264_mp4, + fixture_jpeg, synthetic_png, + }, }; use axum::body::to_bytes; @@ -95,6 +98,7 @@ fn lossy_source_transform_defaults_to_lossy_output() { AssetExtension::Jpeg, AssetExtension::Heic, AssetExtension::Heif, + AssetExtension::Jxl, ] { assert_eq!( ImageQuality::High, @@ -150,6 +154,16 @@ async fn image_transform_route_sends_lossy_sources_to_a_lossy_encode() { transform_qualities_reaching_the_encoder(fixture_jpeg(), "image/jpeg", "photo.jpg").await, "a jpeg attachment must not be re-encoded losslessly" ); + assert_eq!( + vec![ImageQuality::High], + transform_qualities_reaching_the_encoder( + JXL_64_LOSSY_ALPHA.to_vec(), + "image/jxl", + "photo.jxl" + ) + .await, + "a jxl attachment must not be re-encoded losslessly" + ); assert_eq!( vec![ImageQuality::Lossless], transform_qualities_reaching_the_encoder(synthetic_png(48, 48), "image/png", "art.png") @@ -307,6 +321,20 @@ async fn a_full_size_transform_too_large_to_process_never_serves_an_undisplayabl assert_eq!(StatusCode::BAD_REQUEST, response.status()); } +#[tokio::test] +async fn a_full_size_jxl_above_the_static_output_cap_is_downscaled() { + let response = + attachment_transform(JXL_7000X4000_LOSSY_RGB, "big.jxl", &[("format", "webp")]).await; + assert_eq!(StatusCode::OK, response.status()); + assert_eq!( + Some("image/webp"), + response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + ); +} + #[test] fn the_budget_leaves_room_for_the_cache_and_headroom() { assert_eq!( diff --git a/fluxer_media_proxy/src/test_fixtures/images.rs b/fluxer_media_proxy/src/test_fixtures/images.rs index b87958279..b8b798e6c 100644 --- a/fluxer_media_proxy/src/test_fixtures/images.rs +++ b/fluxer_media_proxy/src/test_fixtures/images.rs @@ -32,6 +32,10 @@ pub fn apng_header(frames: u32) -> Vec { pub const JXL_4100_LOSSY_ALPHA: &[u8] = include_bytes!("jxl_4100_lossy_alpha.jxl"); pub const JXL_64_LOSSY_ALPHA: &[u8] = include_bytes!("jxl_64_lossy_alpha.jxl"); +pub const JXL_7000X4000_LOSSY_RGB: &[u8] = include_bytes!("jxl_7000x4000_lossy_rgb.jxl"); +pub const JXL_4100_LOSSLESS_RGB16: &[u8] = include_bytes!("jxl_4100_lossless_rgb16.jxl"); +pub const JXL_4100_LOSSY_RGB_DEPTH: &[u8] = include_bytes!("jxl_4100_lossy_rgb_depth.jxl"); +pub const JXL_3000_LOSSY_2_FRAMES: &[u8] = include_bytes!("jxl_3000_lossy_2_frames.jxl"); pub const ANIMATED_WEBP_1024_17_FRAMES: &[u8] = include_bytes!("animated_webp_1024_17_frames.webp"); pub const JPEG_ROTATED_MISMATCHED_SCAN: &[u8] = include_bytes!("jpeg_rotated_mismatched_scan.jpg"); diff --git a/fluxer_media_proxy/src/test_fixtures/jxl_3000_lossy_2_frames.jxl b/fluxer_media_proxy/src/test_fixtures/jxl_3000_lossy_2_frames.jxl new file mode 100644 index 0000000000000000000000000000000000000000..914124b601d18982933ab546de7b7ff26c38327f GIT binary patch literal 2353 zcmey*wJTPTM?sxIc!nJVgF^sA&w>6E$FkP-8*h`m?2wZA>~u;GFGOy`6bSPB{DE}dEV?n{A|y#X7;rU|vravT^K7-q3> zzH#K4XU2A3I=LsCQL|*mp`h*6$Kof*sIUFM&cpxQT&XSQ^4fxbj_hM!etuT@%O3mJ z59%FMIz?RWW^CK{TQneS-;ea-2MaF0e9OwvU}s%h#=_JHG?=Hugj2}+|9?(T7K7Do zZyhG7c#fh041xI{*IOM6-X=)fC(-f3BT1QzIxPaxpNrsW~geDnm6KxRT0Sp3q3wP4jt z8o^xl56ob08mBBcE_|mLfyJfRE`Y^Nz?xr18!CXfmukO3?9SZP0#?R+!wjs9ZPF4G zDL4)B41E@8?AgZ;vM|#hk*FQfww;$co`JCBy&#neI0E literal 0 HcmV?d00001 diff --git a/fluxer_media_proxy/src/test_fixtures/jxl_4100_lossless_rgb16.jxl b/fluxer_media_proxy/src/test_fixtures/jxl_4100_lossless_rgb16.jxl new file mode 100644 index 0000000000000000000000000000000000000000..dd74db6e21d90a5093288bdc67ea6c4f69c73663 GIT binary patch literal 44903 zcmeI&L1>a;9LMqJHF5gpNJ^%Sg)??hLDB-DLmOV4vSQtK5R$gVSev;DG;kpu8m(3% z3WG3W+hJYW2nvZ9u7gFy+!&1VWR8Ny4zjvLZDmn6y9B}ej<3A%zUZqwJiM>Z@BRG$ zLI`81tE0JI9+8C*=2Pb{#QGBh&DCS^dvB=j2dW>=9H}03f0r9m^Sj!H?0StPex3I7 z)ITrheNCFtd|Jbzed62_)^aVT?dW@7BjTzac+F?FI`pU6$#k#=}rN~yIy zA1n=DtZSPaW7odM=K{m;Y)?K#V)o5U>(yYS^<->0pY`;0FXpqQ?oPuif7rfyw!^Wy zv2uHA()B!3F6iaT>FC49pBFw(=?X_*Z;hm%9UpC){nit{?&u9a^&eb#aA4V~VVuk- z*C+Q8|4aOTvL3N+l0&vfUfLgQX^)glT908pV2l>6KHZ?PXtgAd>BLgnwdih>DrH}2 z$(^#107-x(KoTGckOW8qBmt5Dw?_h0ORgiSa>sGUVgA%6sP#~jqQ*zfl-jN`Yvm+W zk}64+q)JjHsghJlsw7pCDoK?-Rr+b^PgYB{g>lmDL$?pzK6Lxg?W2TlA50Zf#Z)m> zOchhbR54Xd6;s7jF;z?zQ^hK=N~{vA#452$C0M0e)+0<6Q^iyOchhb zR54Xd6;oBqGc4o8ez9Nd7yHG2m0-V^DyE95Vyc)bri!Uzs+cOKim76%m@1}_DyE95Vyc)bri!Uzs+cOKs+MP1#)OckrdDzQqe605{2m0*=>S&uMPOchhbR54Xd z6;s7jF;z@e?WX#-OzYR>hA+F!=loLTkuT&K8nfv-JP3 g5cAH4n|+e^MZ?{Qci+XZBr2UkM0|~-w`@ZE0B1$cU;qFB literal 0 HcmV?d00001 diff --git a/fluxer_media_proxy/src/test_fixtures/jxl_4100_lossy_rgb_depth.jxl b/fluxer_media_proxy/src/test_fixtures/jxl_4100_lossy_rgb_depth.jxl new file mode 100644 index 0000000000000000000000000000000000000000..7736232706dd9d8556565178c95bfb80a18cc7fa GIT binary patch literal 8621 zcmey*CDp(vtRN!J#NgAQ!LWp3S^_6u0_U^`4SX*Su&z1C;&T8iV#XoPVuM{h(2!9r zSVNM$5M#DEfDJh?Z=J8e(6P)Z_1LiwYmUy2+S?%8@VNNr+;}z*h9Lh?LkFe_E|WAw zwNw~8Cr$O7pzv^>YD01ALuMw{312Ob?mOsS@#n>tpUxjxSR|4@b*6n@#NMmi+TR{~ z*zmx8rgOpzEQJhnm(Hww_oYC~-hho^(}db*I%b%IIcXFPUwVzV;$EzSp9 z4J(u8EiXBGy?letzwL?D+o~qFFPysY-aO{-`ZFc<<{YW;t#&;BWJB<7%UPw@W=`H) zU-U|)Mo&)em(6^;qD?XZ_fsxR`DCrbK3T!BiG@>0#bbiP$A1iL4;UB%m=z|utPD7) zGVSmG3yDh^xV67D2n`A$MND@M3Lzz^yUzZ<@NxM$kwFnwSR64Z!U~HcMuQKO!>|Ml zaYCcPHyV7fVw$)VLrqnqsc$s(jix>VO(kk(IBJ@OmU)#i+gF1^NCnefgF;9ZwcR%; z!U~Hc21Qt5al~lwfvOHH0YjY7Xz+~&AFSpjF2zt&)oAJ)O?{)O54qhpbZcHU)OHNr Wxaz3((v7PDvRAS53TqV?0|Nkp#oo{W literal 0 HcmV?d00001 diff --git a/fluxer_media_proxy/src/test_fixtures/jxl_7000x4000_lossy_rgb.jxl b/fluxer_media_proxy/src/test_fixtures/jxl_7000x4000_lossy_rgb.jxl new file mode 100644 index 0000000000000000000000000000000000000000..ea4cf0a3b66fbdae88b16fe4eecc7700025ee6a9 GIT binary patch literal 5777 zcmey*^{eJZR&&cECkEjTCWan{X$hRuTHIz%@tirMO@_sRVHAyqz@QC*t@9NaI+i)5 z9y|77&C%IWdmCgM9vAxty&j|_-=czUnr#@t6Vx92S^60*U z?iGJteEI48frUjP=~HLg=SA$j%B}tFv4;&0+-Eu`yueb(Fn8(9%6DH1wCoMo7&c9) zeWs(HJEvWg?O*wS<{6>18aD-YMm5Xq@EdV| zLJCcPeVB18y75G;g2Kk>E=T6Gu6&gB<)Pu$?MxF7?s9+bYx$quWs-;EmEN@1yW<&C zpLW-qvGZBYy%)#8@M81jbaw>-4hDt;4IYyfHh%s;e}W6c^KX0%DxQ-@(Ex@(wygcH zRB6W1bTyg|sTu;K>1qHo^)R-Y_UUTMA&nC|61t}C+_X2AjZvLxirYNhk;t}57GrKxgtUrdvN;ZKj{{LtE^Pl<0fA$am z`5%0jXQ%`6|1oR)V-NVtpYThbp=tf}HD}G3MiUujX=F5!jV3ahXY$cRHn53oiW z<0-d*{ra7ot|ltq6Z*ECGjHFH5Dff4Rnz|_Nw^Fcbmt)}VEJ;1G) zcLHFCPKsj#Yn;#u*2r==8mwc&k_@0DAkCkj{p_IX=QqE?4|#^a_6!$J`mbf+=V4W0 z=n6<^POQ~qIFPBlc+PrvGxt?%9$0Hkd#TB9qW`o>z4lKyV*y{Lv)`e8vk%@W-j@D( zVS@bGa#o$b0OwzN`%avnp0MV17SE(Eo6DcKr` nXxw9efI)_VA&UVD7@>d(3YdXlILNU;0V@=+K><4u3