Improve handling of animated images, add support for AVIF animations (#30932)

* Only set MSC4230 is_animated flag if we are able to tell if the media is animated

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Set blob type correctly to not need to weave the mimetype around

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Use ImageDecoder to determine whether media is animated or not, adding support for AVIF and other formats

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix test

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add test

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix test

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

src/utils/Image.ts

@@ -9,7 +9,6 @@
 import { arrayHasDiff } from "./arrays";
 
 export function mayBeAnimated(mimeType?: string): boolean {
-    // AVIF animation support at the time of writing is only available in Chrome hence not having `blobIsAnimated` check
     return ["image/gif", "image/webp", "image/png", "image/apng", "image/avif"].includes(mimeType!);
 }
 
@@ -26,8 +25,28 @@ function arrayBufferReadStr(arr: ArrayBuffer, start: number, len: number): strin
     return String.fromCharCode.apply(null, Array.from(arrayBufferRead(arr, start, len)));
 }
 
-export async function blobIsAnimated(mimeType: string | undefined, blob: Blob): Promise<boolean> {
-    switch (mimeType) {
+/**
+ * Check if a Blob contains an animated image.
+ * @param blob The Blob to check.
+ * @returns True if the image is animated, false if not, or undefined if it could not be determined.
+ */
+export async function blobIsAnimated(blob: Blob): Promise<boolean | undefined> {
+    try {
+        // Try parse the image using ImageDecoder as this is the most coherent way of asserting whether a piece of media
+        // is or is not animated. Limited availability at time of writing, notably Safari lacks support.
+        // https://developer.mozilla.org/en-US/docs/Web/API/ImageDecoder
+        const data = await blob.arrayBuffer();
+        const decoder = new ImageDecoder({ data, type: blob.type });
+        await decoder.tracks.ready;
+        if ([...decoder.tracks].some((track) => track.animated)) {
+            return true;
+        }
+    } catch (e) {
+        console.warn("ImageDecoder not supported or failed to decode image", e);
+        // Not supported by this browser, fall through to manual checks
+    }
+
+    switch (blob.type) {
         case "image/webp": {
             // Only extended file format WEBP images support animation, so grab the expected data range and verify header.
             // Based on https://developers.google.com/speed/webp/docs/riff_container#extended_file_format
@@ -42,7 +61,7 @@ export async function blobIsAnimated(mimeType: string | undefined, blob: Blob):
                 const animationFlagMask = 1 << 1;
                 return (flags & animationFlagMask) != 0;
             }
-            break;
+            return false;
         }
 
         case "image/gif": {
@@ -100,9 +119,7 @@ export async function blobIsAnimated(mimeType: string | undefined, blob: Blob):
                 }
                 i += length + 4;
             }
-            break;
+            return false;
         }
     }
-
-    return false;
 }

src/utils/MediaEventHelper.ts

@@ -72,18 +72,25 @@ export class MediaEventHelper implements IDestroyable {
     };
 
     private fetchSource = (): Promise<Blob> => {
+        const content = this.event.getContent<MediaEventContent>();
         if (this.media.isEncrypted) {
-            const content = this.event.getContent<MediaEventContent>();
             return decryptFile(content.file!, content.info);
         }
-        return this.media.downloadSource().then((r) => r.blob());
+
+        return (
+            this.media
+                .downloadSource()
+                .then((r) => r.blob())
+                // Set the mime type from the event info on the blob
+                .then((blob) => blob.slice(0, blob.size, content.info?.mimetype ?? blob.type))
+        );
     };
 
     private fetchThumbnail = (): Promise<Blob | null> => {
         if (!this.media.hasThumbnail) return Promise.resolve(null);
 
+        const content = this.event.getContent<ImageContent>();
         if (this.media.isEncrypted) {
-            const content = this.event.getContent<ImageContent>();
             if (content.info?.thumbnail_file) {
                 return decryptFile(content.info.thumbnail_file, content.info.thumbnail_info);
             } else {
@@ -96,7 +103,12 @@ export class MediaEventHelper implements IDestroyable {
         const thumbnailHttp = this.media.thumbnailHttp;
         if (!thumbnailHttp) return Promise.resolve(null);
 
-        return fetch(thumbnailHttp).then((r) => r.blob());
+        return (
+            fetch(thumbnailHttp)
+                .then((r) => r.blob())
+                // Set the mime type from the event info on the blob
+                .then((blob) => blob.slice(0, blob.size, content.info?.thumbnail_info?.mimetype ?? blob.type))
+        );
     };
 
     public static isEligible(event: MatrixEvent): boolean {