From cbb770bff8472aaede8aab474d06ebd9759db250 Mon Sep 17 00:00:00 2001 From: khannurien Date: Sat, 10 Oct 2026 19:40:14 +0000 Subject: [PATCH] =?UTF-8?q?v4:=20a=20still=20is=20scaled=20from=20its=20fi?= =?UTF-8?q?rst=20frame=20alone,=20a=20second=20image=20in=20its=20file=20m?= =?UTF-8?q?ade=20the=20copy=20flash=20=E2=80=94=20an=20avatar=20came=20out?= =?UTF-8?q?=20as=20a=20WebP=20of=20two=20frames=20looping=20at=2040=20ms?= =?UTF-8?q?=20each,=20the=20picture=20then=20a=20blank.=20A=20still's=20fi?= =?UTF-8?q?le=20may=20hold=20more=20than=20one=20image,=20a=20phone's=20JP?= =?UTF-8?q?EG=20carrying=20a=20gain=20map=20or=20a=20depth=20map=20after?= =?UTF-8?q?=20the=20photo,=20and=20ffmpeg=20reads=20each=20as=20a=20frame:?= =?UTF-8?q?=20only=20the=20JPEG=20and=20PNG=20copies=20were=20held=20to=20?= =?UTF-8?q?one,=20the=20WebP=20keeping=20them=20all=20as=20it=20does=20a?= =?UTF-8?q?=20GIF's.=20scalableImage=20(lib/upload.ts)=20now=20says=20whet?= =?UTF-8?q?her=20the=20image=20is=20an=20animation,=20which=20a=20GIF=20al?= =?UTF-8?q?one=20is,=20and=20scaleImage=20(lib/ffmpeg.ts)=20writes=20one?= =?UTF-8?q?=20frame=20of=20anything=20else,=20in=20every=20format;=20the?= =?UTF-8?q?=20image=20proxy=20scales=20through=20the=20same=20call.=20The?= =?UTF-8?q?=20copies=20already=20made=20stay,=20a=20copy=20being=20current?= =?UTF-8?q?=20while=20it=20has=20its=20upload's=20date:=20uploads/scaled?= =?UTF-8?q?=20is=20to=20be=20deleted=20once=20on=20an=20instance=20that=20?= =?UTF-8?q?ran=20the=20previous=20version,=20everything=20in=20it=20being?= =?UTF-8?q?=20made=20again=20on=20first=20request.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- api/lib/ffmpeg.ts | 33 +++++++++++++++++++++++++-------- api/lib/upload.ts | 17 ++++++++--------- api/remote/image-proxy.ts | 3 +-- api/test/lib/upload.test.ts | 22 ++++++++++++++++------ api/test/routes/avatars.test.ts | 18 ++++++++++++++++++ 5 files changed, 68 insertions(+), 25 deletions(-) diff --git a/api/lib/ffmpeg.ts b/api/lib/ffmpeg.ts index 225a6c81..b14bc93d 100644 --- a/api/lib/ffmpeg.ts +++ b/api/lib/ffmpeg.ts @@ -200,28 +200,44 @@ const ORIENTATION_FILTERS: Record = { /** What ffmpeg is told to write each {@link ScaledFormat} with. */ const ENCODERS: Record = { webp: ["-c:v", "libwebp", "-q:v", "80", "-loop", "0", "-f", "webp"], - // A still: the first frame of an animation. - jpg: ["-frames:v", "1", "-c:v", "mjpeg", "-q:v", "4", "-f", "image2"], - png: ["-frames:v", "1", "-c:v", "png", "-f", "image2"], + jpg: ["-c:v", "mjpeg", "-q:v", "4", "-f", "image2"], + png: ["-c:v", "png", "-f", "image2"], }; +/** + * What {@link scaleImage} is told of the image it scales, which + * `scalableImage` (lib/upload.ts) reads from its first bytes. + */ +export interface ImageTraits { + /** The EXIF orientation to turn the image by, 1 when it is upright. */ + orientation: number; + /** + * Whether its frames are an animation, as a GIF's are. A still may hold + * more than one image too, a phone's JPEG carrying a gain map or a depth + * map after the photo, and ffmpeg reads each as a frame. + */ + animated: boolean; +} + /** * Writes a copy of an image at most `width` wide, never wider than the image * itself, as a WebP unless `format` names another. WebP keeps a PNG's * transparency and a GIF's animation, at a fraction of either's bytes. - * `orientation` is the EXIF orientation of a JPEG, as `scalableImage` - * (lib/upload.ts) reads it. */ export function scaleImage( srcPath: string, outPath: string, width: ImageWidth, - orientation = 1, + { orientation, animated }: ImageTraits, format: ScaledFormat = "webp", ): Promise { const upright = ORIENTATION_FILTERS[orientation]; // A JPEG's colour planes halve its height, which they need even. const height = format === "jpg" ? -2 : -1; + // Only an animation keeps its frames, and only as a WebP: of anything else + // the first is the image, and a copy of them all would flash through what + // follows it. + const frames = animated && format === "webp" ? [] : ["-frames:v", "1"]; return runFfmpeg([ "-noautorotate", "-format_whitelist", @@ -231,6 +247,7 @@ export function scaleImage( "-vf", `${upright ? `${upright},` : ""}scale='min(${width},iw)':${height}`, "-an", + ...frames, ...ENCODERS[format], ], outPath); } @@ -243,7 +260,7 @@ export function scaleImage( export async function scaleImageBytes( bytes: Uint8Array, width: ImageWidth, - orientation = 1, + traits: ImageTraits, ): Promise { if (!await ffmpegAvailable()) return null; const srcPath = `${SCALED_DIR}/tmp/${crypto.randomUUID()}`; @@ -251,7 +268,7 @@ export async function scaleImageBytes( try { await Deno.mkdir(dirname(srcPath), { recursive: true }); await Deno.writeFile(srcPath, bytes); - await scaleImage(srcPath, outPath, width, orientation); + await scaleImage(srcPath, outPath, width, traits); return await Deno.readFile(outPath); } catch { return null; diff --git a/api/lib/upload.ts b/api/lib/upload.ts index cb90b9cc..7ab044b8 100644 --- a/api/lib/upload.ts +++ b/api/lib/upload.ts @@ -14,6 +14,7 @@ import { MAX_IMAGE_SIZE_BYTES } from "../config.ts"; import { type ImageWidth, parseImageWidth } from "../../shared/image.ts"; import { ensureGenerated, + type ImageTraits, type ScaledFormat, scaledImagePath, scaleImage, @@ -113,9 +114,7 @@ function jpegOrientation(data: Uint8Array): number { const IMAGE_HEAD_BYTES = 64 * 1024 + 32; /** Returned by {@link scalableImage}, for `scaleImage`. */ -export interface ScalableImage { - /** The EXIF orientation to turn the image by, 1 when it is upright. */ - orientation: number; +export interface ScalableImage extends ImageTraits { /** * The format of its copy for a reader without WebP: a JPEG, or a PNG for * an image that may be transparent, which a JPEG would fill with black. @@ -133,16 +132,16 @@ export function scalableImage(head: Uint8Array): ScalableImage | null { // A file dump may be a BMP ("BM"), which no upload route here accepts. if (!mime) { return head[0] === 0x42 && head[1] === 0x4D - ? { orientation: 1, compat: "jpg" } + ? { orientation: 1, animated: false, compat: "jpg" } : null; } // "RIFF....WEBPVP8X", whose flags then say whether frames follow. - const animated = mime === "image/webp" && head[15] === 0x58 && + const animatedWebp = mime === "image/webp" && head[15] === 0x58 && (head[20] & 0x02) !== 0; - if (animated) return null; + if (animatedWebp) return null; return mime === "image/jpeg" - ? { orientation: jpegOrientation(head), compat: "jpg" } - : { orientation: 1, compat: "png" }; + ? { orientation: jpegOrientation(head), animated: false, compat: "jpg" } + : { orientation: 1, animated: mime === "image/gif", compat: "png" }; } /** Reads the first {@link IMAGE_HEAD_BYTES} of the file at `path`. */ @@ -195,7 +194,7 @@ async function scaledCopy( const made = await ensureGenerated(scaledPath, async () => { how ??= await describe(path); if (!how) throw new Error("Not an image that can be scaled"); - await scaleImage(path, scaledPath, width, how.orientation, format); + await scaleImage(path, scaledPath, width, how, format); }, path); return made ? { path: scaledPath, mime: SCALED_MIMES[format] } : null; } diff --git a/api/remote/image-proxy.ts b/api/remote/image-proxy.ts index 673f21c1..16afbaf8 100644 --- a/api/remote/image-proxy.ts +++ b/api/remote/image-proxy.ts @@ -111,8 +111,7 @@ export async function fetchProxiedImage( const original = await fetchOriginal(url); if (!original) return null; const how = scalableImage(original.bytes); - const bytes = how && - await scaleImageBytes(original.bytes, width, how.orientation); + const bytes = how && await scaleImageBytes(original.bytes, width, how); if (!bytes) return original; writeImageCache(key, bytes, "image/webp"); return { bytes, contentType: "image/webp" }; diff --git a/api/test/lib/upload.test.ts b/api/test/lib/upload.test.ts index 2b5c89e2..fc1a0980 100644 --- a/api/test/lib/upload.test.ts +++ b/api/test/lib/upload.test.ts @@ -94,17 +94,27 @@ Deno.test("scalableImage: an animated WebP and what isn't an image are served as assertEquals(scalableImage(webp(0x02)), null); // A reader without WebP gets a PNG of what may be transparent, a JPEG of // the rest. - assertEquals(scalableImage(webp(0x10)), { orientation: 1, compat: "png" }); + const still = { orientation: 1, animated: false }; + assertEquals(scalableImage(webp(0x10)), { ...still, compat: "png" }); assertEquals( scalableImage(new TextEncoder().encode("RIFF\0\0\0\0WEBPVP8 ")), - { orientation: 1, compat: "png" }, + { ...still, compat: "png" }, ); - assertEquals(scalableImage(PNG), { orientation: 1, compat: "png" }); - assertEquals(scalableImage(jpegWithOrientation(1, true))?.compat, "jpg"); - assertEquals(scalableImage(new TextEncoder().encode("BM-a-bitmap")), { - orientation: 1, + assertEquals(scalableImage(PNG), { ...still, compat: "png" }); + assertEquals(scalableImage(jpegWithOrientation(1, true)), { + ...still, compat: "jpg", }); + assertEquals(scalableImage(new TextEncoder().encode("BM-a-bitmap")), { + ...still, + compat: "jpg", + }); + // Only a GIF's frames are an animation to keep. + assertEquals(scalableImage(new TextEncoder().encode("GIF89a")), { + orientation: 1, + animated: true, + compat: "png", + }); assertEquals(scalableImage(new TextEncoder().encode("")), null); assertEquals(scalableImage(new Uint8Array()), null); }); diff --git a/api/test/routes/avatars.test.ts b/api/test/routes/avatars.test.ts index 0b5d161e..78f9834a 100644 --- a/api/test/routes/avatars.test.ts +++ b/api/test/routes/avatars.test.ts @@ -38,6 +38,24 @@ test("avatars: upload, serve, and 404 when there is none", async () => { ); }); +test("avatars: a still holding a second image is scaled to its first alone", async () => { + if (!await hasFfmpeg()) return; + const user = await createUser(); + // As a phone's photo carries a gain map after it, which ffmpeg reads as a + // second frame. + const photo = await makeImage(600, 400, "red"); + const extra = await makeImage(150, 100, "white"); + const both = new Uint8Array(photo.length + extra.length); + both.set(photo); + both.set(extra, photo.length); + await put("/api/me/avatar", { auth: user.auth, form: formWithFile(both) }); + + const small = await get(`/api/users/${user.id}/avatar?w=128`); + assertEquals(small.headers.get("content-type"), "image/webp"); + assertEquals(webpWidth(small.bytes), 128); + assertFalse(new TextDecoder().decode(small.bytes).includes("ANMF")); +}); + test("avatars: `?w=` answers a scaled copy, made again from a new avatar", async () => { if (!await hasFfmpeg()) return; const user = await createUser();