Improve handling of animated images, add support for AVIF animations (#30932)

* Only set MSC4230 is_animated flag if we are able to tell if the media is animated

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Set blob type correctly to not need to weave the mimetype around

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Use ImageDecoder to determine whether media is animated or not, adding support for AVIF and other formats

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix test

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add test

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix test

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

src/ContentMessages.ts

@@ -158,14 +158,17 @@ async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imag
     }
 
     // We don't await this immediately so it can happen in the background
-    const isAnimatedPromise = blobIsAnimated(imageFile.type, imageFile);
+    const isAnimatedPromise = blobIsAnimated(imageFile);
 
     const imageElement = await loadImageElement(imageFile);
 
     const result = await createThumbnail(imageElement.img, imageElement.width, imageElement.height, thumbnailType);
     const imageInfo = result.info;
 
-    imageInfo["org.matrix.msc4230.is_animated"] = await isAnimatedPromise;
+    const isAnimated = await isAnimatedPromise;
+    if (isAnimated !== undefined) {
+        imageInfo["org.matrix.msc4230.is_animated"] = await isAnimatedPromise;
+    }
 
     // For lesser supported image types, always include the thumbnail even if it is larger
     if (!ALWAYS_INCLUDE_THUMBNAIL.includes(imageFile.type)) {

src/components/views/messages/MImageBody.tsx

@@ -311,10 +311,7 @@ export class MImageBodyInner extends React.Component<IProps, IState> {
                     // then we need to check if the image is animated by downloading it.
                     if (
                         content.info?.["org.matrix.msc4230.is_animated"] === false ||
-                        !(await blobIsAnimated(
+                        (await blobIsAnimated(await this.props.mediaEventHelper!.sourceBlob.value)) === false
-                            content.info?.mimetype,
-                            await this.props.mediaEventHelper!.sourceBlob.value,
-                        ))
                     ) {
                         isAnimated = false;
                     }

src/utils/Image.ts

@@ -9,7 +9,6 @@
 import { arrayHasDiff } from "./arrays";
 
 export function mayBeAnimated(mimeType?: string): boolean {
-    // AVIF animation support at the time of writing is only available in Chrome hence not having `blobIsAnimated` check
     return ["image/gif", "image/webp", "image/png", "image/apng", "image/avif"].includes(mimeType!);
 }
 
@@ -26,8 +25,28 @@ function arrayBufferReadStr(arr: ArrayBuffer, start: number, len: number): strin
     return String.fromCharCode.apply(null, Array.from(arrayBufferRead(arr, start, len)));
 }
 
-export async function blobIsAnimated(mimeType: string | undefined, blob: Blob): Promise<boolean> {
+/**
-    switch (mimeType) {
+ * Check if a Blob contains an animated image.
+ * @param blob The Blob to check.
+ * @returns True if the image is animated, false if not, or undefined if it could not be determined.
+ */
+export async function blobIsAnimated(blob: Blob): Promise<boolean | undefined> {
+    try {
+        // Try parse the image using ImageDecoder as this is the most coherent way of asserting whether a piece of media
+        // is or is not animated. Limited availability at time of writing, notably Safari lacks support.
+        // https://developer.mozilla.org/en-US/docs/Web/API/ImageDecoder
+        const data = await blob.arrayBuffer();
+        const decoder = new ImageDecoder({ data, type: blob.type });
+        await decoder.tracks.ready;
+        if ([...decoder.tracks].some((track) => track.animated)) {
+            return true;
+        }
+    } catch (e) {
+        console.warn("ImageDecoder not supported or failed to decode image", e);
+        // Not supported by this browser, fall through to manual checks
+    }
+
+    switch (blob.type) {
         case "image/webp": {
             // Only extended file format WEBP images support animation, so grab the expected data range and verify header.
             // Based on https://developers.google.com/speed/webp/docs/riff_container#extended_file_format
@@ -42,7 +61,7 @@ export async function blobIsAnimated(mimeType: string | undefined, blob: Blob):
                 const animationFlagMask = 1 << 1;
                 return (flags & animationFlagMask) != 0;
             }
-            break;
+            return false;
         }
 
         case "image/gif": {
@@ -100,9 +119,7 @@ export async function blobIsAnimated(mimeType: string | undefined, blob: Blob):
                 }
                 i += length + 4;
             }
-            break;
+            return false;
         }
     }
-
-    return false;
 }

src/utils/MediaEventHelper.ts

@@ -72,18 +72,25 @@ export class MediaEventHelper implements IDestroyable {
     };
 
     private fetchSource = (): Promise<Blob> => {
+        const content = this.event.getContent<MediaEventContent>();
         if (this.media.isEncrypted) {
-            const content = this.event.getContent<MediaEventContent>();
             return decryptFile(content.file!, content.info);
         }
-        return this.media.downloadSource().then((r) => r.blob());
+
+        return (
+            this.media
+                .downloadSource()
+                .then((r) => r.blob())
+                // Set the mime type from the event info on the blob
+                .then((blob) => blob.slice(0, blob.size, content.info?.mimetype ?? blob.type))
+        );
     };
 
     private fetchThumbnail = (): Promise<Blob | null> => {
         if (!this.media.hasThumbnail) return Promise.resolve(null);
 
+        const content = this.event.getContent<ImageContent>();
         if (this.media.isEncrypted) {
-            const content = this.event.getContent<ImageContent>();
             if (content.info?.thumbnail_file) {
                 return decryptFile(content.info.thumbnail_file, content.info.thumbnail_info);
             } else {
@@ -96,7 +103,12 @@ export class MediaEventHelper implements IDestroyable {
         const thumbnailHttp = this.media.thumbnailHttp;
         if (!thumbnailHttp) return Promise.resolve(null);
 
-        return fetch(thumbnailHttp).then((r) => r.blob());
+        return (
+            fetch(thumbnailHttp)
+                .then((r) => r.blob())
+                // Set the mime type from the event info on the blob
+                .then((blob) => blob.slice(0, blob.size, content.info?.thumbnail_info?.mimetype ?? blob.type))
+        );
     };
 
     public static isEligible(event: MatrixEvent): boolean {

test/unit-tests/Image-test.ts

@@ -32,42 +32,55 @@ describe("Image", () => {
 
     describe("blobIsAnimated", () => {
         it("Animated GIF", async () => {
-            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.gif"))]);
+            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.gif"))], {
-            expect(await blobIsAnimated("image/gif", img)).toBeTruthy();
+                type: "image/gif",
+            });
+            expect(await blobIsAnimated(img)).toBeTruthy();
         });
 
         it("Static GIF", async () => {
-            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.gif"))]);
+            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.gif"))], {
-            expect(await blobIsAnimated("image/gif", img)).toBeFalsy();
+                type: "image/gif",
+            });
+            expect(await blobIsAnimated(img)).toBeFalsy();
         });
 
         it("Animated WEBP", async () => {
-            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.webp"))]);
+            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.webp"))], {
-            expect(await blobIsAnimated("image/webp", img)).toBeTruthy();
+                type: "image/webp",
+            });
+            expect(await blobIsAnimated(img)).toBeTruthy();
         });
 
         it("Static WEBP", async () => {
-            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.webp"))]);
+            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.webp"))], {
-            expect(await blobIsAnimated("image/webp", img)).toBeFalsy();
+                type: "image/webp",
+            });
+            expect(await blobIsAnimated(img)).toBeFalsy();
         });
 
         it("Static WEBP in extended file format", async () => {
-            const img = new Blob([
+            const img = new Blob(
-                fs.readFileSync(path.resolve(__dirname, "images", "static-logo-extended-file-format.webp")),
+                [fs.readFileSync(path.resolve(__dirname, "images", "static-logo-extended-file-format.webp"))],
-            ]);
+                { type: "image/webp" },
-            expect(await blobIsAnimated("image/webp", img)).toBeFalsy();
+            );
+            expect(await blobIsAnimated(img)).toBeFalsy();
         });
 
         it("Animated PNG", async () => {
             const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.apng"))]);
-            expect(await blobIsAnimated("image/png", img)).toBeTruthy();
+            const pngBlob = img.slice(0, img.size, "image/png");
-            expect(await blobIsAnimated("image/apng", img)).toBeTruthy();
+            const apngBlob = img.slice(0, img.size, "image/apng");
+            expect(await blobIsAnimated(pngBlob)).toBeTruthy();
+            expect(await blobIsAnimated(apngBlob)).toBeTruthy();
         });
 
         it("Static PNG", async () => {
             const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.png"))]);
-            expect(await blobIsAnimated("image/png", img)).toBeFalsy();
+            const pngBlob = img.slice(0, img.size, "image/png");
-            expect(await blobIsAnimated("image/apng", img)).toBeFalsy();
+            const apngBlob = img.slice(0, img.size, "image/apng");
+            expect(await blobIsAnimated(pngBlob)).toBeFalsy();
+            expect(await blobIsAnimated(apngBlob)).toBeFalsy();
         });
     });
 });

test/unit-tests/utils/MediaEventHelper-test.ts

@@ -0,0 +1,40 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { MatrixEvent } from "matrix-js-sdk/src/matrix";
+
+import { MediaEventHelper } from "../../../src/utils/MediaEventHelper.ts";
+import { stubClient } from "../../test-utils";
+
+describe("MediaEventHelper", () => {
+    it("should set the mime type on the blob based on the event metadata", async () => {
+        stubClient();
+
+        const event = new MatrixEvent({
+            type: "m.room.message",
+            content: {
+                msgtype: "m.image",
+                body: "image.png",
+                info: {
+                    mimetype: "image/png",
+                    size: 1234,
+                    w: 100,
+                    h: 100,
+                    thumbnail_info: {
+                        mimetype: "image/png",
+                    },
+                    thumbnail_url: "mxc://matrix.org/thumbnail",
+                },
+                url: "mxc://matrix.org/abcdef",
+            },
+        });
+        const helper = new MediaEventHelper(event);
+
+        const blob = await helper.thumbnailBlob.value;
+        expect(blob?.type).toBe(event.getContent().info.thumbnail_info?.mimetype);
+    });
+});