Add URL preview above message composer (#33964)

* Modify LinkPreview to export shared atomics * Implement MessageComposerUrlPreviewView * Create UrlPreviewFetcher utility function * Modify view models * Implement in composer * Support running tests in dom-less vitest environment * Add a playwright test * hide another one * fmt * cleanup * test rte too * fixup * Add back docstring * cleanup * off by one * remove description check * Cleanup hacks * Remove another hack * cleanup * one more * fixup window here too * whoops type * Rename to be cleaeer * Trim URLs first * fix bug
2026-06-30 16:58:03 +00:00
parent bbbd050f50
commit be91d19e31
29 changed files with 1386 additions and 600 deletions
@@ -0,0 +1,223 @@
+/*
+ * Copyright 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+import { vi, describe, it, expect, beforeAll, afterAll, type Mock } from "vitest";
+
+import type { IPreviewUrlResponse, MatrixClient } from "matrix-js-sdk/src/matrix";
+import { UrlPreviewFetcher } from "./UrlPreviewFetcher";
+
+const IMAGE_MXC = "mxc://example.org/abc";
+const BASIC_PREVIEW_OGDATA = {
+    "og:title": "This is an example!",
+    "og:description": "This is a description",
+    "og:type": "document",
+    "og:url": "https://example.org",
+    "og:site_name": "Example.org",
+};
+
+function getFetcher(): {
+    fetcher: UrlPreviewFetcher;
+    client: { getUrlPreview: Mock; mxcUrlToHttp: Mock };
+} {
+    const client = {
+        getUrlPreview: vi.fn(),
+        mxcUrlToHttp: vi.fn(),
+    } as unknown as MatrixClient;
+    return {
+        fetcher: new UrlPreviewFetcher(client, 0, false),
+        client: client as unknown as { getUrlPreview: Mock; mxcUrlToHttp: Mock },
+    };
+}
+
+describe("UrlPreviewFetcher", () => {
+    let originalDevicePixelRatio: Window["devicePixelRatio"];
+    beforeAll(() => {
+        originalDevicePixelRatio = window.devicePixelRatio;
+        window.devicePixelRatio = 1;
+    });
+    afterAll(() => {
+        window.devicePixelRatio = originalDevicePixelRatio;
+    });
+    it("should return null when the fetch fails", async () => {
+        const { fetcher, client } = getFetcher();
+        client.getUrlPreview.mockRejectedValue(new Error("Forced test failure"));
+        expect(await fetcher.fetchPreview("https://example.org", true)).toBeNull();
+    });
+
+    it("should return null when title equals the URL and there is no image", async () => {
+        const { fetcher, client } = getFetcher();
+        client.getUrlPreview.mockResolvedValueOnce({
+            "og:title": "https://example.org",
+            "og:type": "document",
+            "og:url": "https://example.org",
+        });
+        expect(await fetcher.fetchPreview("https://example.org", true)).toBeNull();
+    });
+
+    it("should cache results and not re-fetch for the same URL", async () => {
+        const { fetcher, client } = getFetcher();
+        client.getUrlPreview.mockResolvedValue(BASIC_PREVIEW_OGDATA);
+        await fetcher.fetchPreview("https://example.org", true);
+        await fetcher.fetchPreview("https://example.org", true);
+        expect(client.getUrlPreview).toHaveBeenCalledTimes(1);
+    });
+
+    it("should re-fetch after clearCache is called", async () => {
+        const { fetcher, client } = getFetcher();
+        client.getUrlPreview.mockResolvedValue(BASIC_PREVIEW_OGDATA);
+        await fetcher.fetchPreview("https://example.org", true);
+        fetcher.clearCache();
+        await fetcher.fetchPreview("https://example.org", true);
+        expect(client.getUrlPreview).toHaveBeenCalledTimes(2);
+    });
+
+    it("should not process media when loadMedia is false", async () => {
+        const { fetcher, client } = getFetcher();
+        client.getUrlPreview.mockResolvedValueOnce({
+            "og:title": "This is an example!",
+            "og:type": "document",
+            "og:url": "https://example.org",
+            "og:image": IMAGE_MXC,
+            "og:image:height": 128,
+            "og:image:width": 128,
+            "matrix:image:size": 10000,
+        });
+        const preview = await fetcher.fetchPreview("https://example.org", false);
+        expect(preview?.image).toBeUndefined();
+        // eslint-disable-next-line no-restricted-properties
+        expect(client.mxcUrlToHttp).not.toHaveBeenCalled();
+    });
+
+    it("should preview a URL with media", async () => {
+        const { fetcher, client } = getFetcher();
+        client.getUrlPreview.mockResolvedValueOnce({
+            "og:title": "This is an example!",
+            "og:type": "document",
+            "og:url": "https://example.org",
+            "og:image": IMAGE_MXC,
+            "og:image:height": 128,
+            "og:image:width": 128,
+            "matrix:image:size": 10000,
+        });
+        // eslint-disable-next-line no-restricted-properties
+        client.mxcUrlToHttp.mockImplementation((url, width) => {
+            expect(url).toEqual(IMAGE_MXC);
+            if (width) return "https://example.org/image/thumb";
+            return "https://example.org/image/src";
+        });
+        const preview = await fetcher.fetchPreview("https://example.org", true);
+        expect(preview).toMatchSnapshot();
+    });
+
+    it.each<Partial<IPreviewUrlResponse>>([
+        { "matrix:image:size": 8191 },
+        { "og:image:width": 95 },
+        { "og:image:height": 95 },
+    ])("should use a site icon for small images %s", async (extraResp) => {
+        const { fetcher, client } = getFetcher();
+        client.getUrlPreview.mockResolvedValueOnce({
+            "og:title": "This is an example!",
+            "og:type": "document",
+            "og:url": "https://example.org",
+            "og:image": IMAGE_MXC,
+            "og:image:height": 128,
+            "og:image:width": 128,
+            "matrix:image:size": 8193,
+            ...extraResp,
+        });
+        // eslint-disable-next-line no-restricted-properties
+        client.mxcUrlToHttp.mockImplementation((url) => {
+            expect(url).toEqual(IMAGE_MXC);
+            return "https://example.org/image/src";
+        });
+        const preview = await fetcher.fetchPreview("https://example.org", true);
+        expect(preview?.siteIcon).toBeTruthy();
+        expect(preview?.image).toBeUndefined();
+    });
+
+    it.each<string>(["og:video", "og:video:type", "og:audio"])("detects playable links via %s", async (property) => {
+        const { fetcher, client } = getFetcher();
+        // eslint-disable-next-line no-restricted-properties
+        client.mxcUrlToHttp.mockImplementation((url, width) => {
+            if (width) return "https://example.org/image/thumb";
+            return "https://example.org/image/src";
+        });
+        client.getUrlPreview.mockResolvedValueOnce({
+            ...BASIC_PREVIEW_OGDATA,
+            "og:image": IMAGE_MXC,
+            "og:image:height": 128,
+            "og:image:width": 128,
+            "matrix:image:size": 10000,
+            [property]: "anything",
+        });
+        const preview = await fetcher.fetchPreview("https://example.org", true);
+        expect(preview?.image?.playable).toBe(true);
+    });
+
+    describe("calculates author", () => {
+        it("should use the profile:username if provided", async () => {
+            const { fetcher, client } = getFetcher();
+            client.getUrlPreview.mockResolvedValueOnce({ ...BASIC_PREVIEW_OGDATA, "profile:username": "my username" });
+            const preview = await fetcher.fetchPreview("https://example.org", true);
+            expect(preview?.author).toEqual("my username");
+        });
+
+        it("should use author if the og:type is an article", async () => {
+            const { fetcher, client } = getFetcher();
+            client.getUrlPreview.mockResolvedValueOnce({
+                ...BASIC_PREVIEW_OGDATA,
+                "og:type": "article",
+                "article:author": "my name",
+            });
+            const preview = await fetcher.fetchPreview("https://example.org", true);
+            expect(preview?.author).toEqual("my name");
+        });
+
+        it("should NOT use author if the author is a URL", async () => {
+            const { fetcher, client } = getFetcher();
+            client.getUrlPreview.mockResolvedValueOnce({
+                ...BASIC_PREVIEW_OGDATA,
+                "og:type": "article",
+                "article:author": "https://junk.example.org/foo",
+            });
+            const preview = await fetcher.fetchPreview("https://example.org", true);
+            expect(preview?.author).toBeUndefined();
+        });
+    });
+
+    // og:url and og:type are not surfaced in the preview.
+    const baseOg = {
+        "og:url": "https://example.org",
+        "og:type": "document",
+    };
+
+    it.each<IPreviewUrlResponse>([
+        { ...baseOg, "og:title": "Basic title" },
+        { ...baseOg, "og:site_name": "Site name", "og:title": "" },
+        { ...baseOg, "og:description": "A description", "og:title": "" },
+        { ...baseOg, "og:title": "Cool blog", "og:site_name": "Cool site" },
+        {
+            ...baseOg,
+            "og:title": "Media test",
+            // API *may* return a string, so check we parse correctly.
+            "og:image:height": "500" as unknown as number,
+            "og:image:width": 500,
+            "matrix:image:size": 10000,
+            "og:image": IMAGE_MXC,
+        },
+    ])("handles different kinds of opengraph responses %s", async (og) => {
+        const { fetcher, client } = getFetcher();
+        // eslint-disable-next-line no-restricted-properties
+        client.mxcUrlToHttp.mockImplementation((url, width) => {
+            if (width) return "https://example.org/image/thumb";
+            return "https://example.org/image/src";
+        });
+        client.getUrlPreview.mockResolvedValueOnce(og);
+        const preview = await fetcher.fetchPreview("https://example.org", true);
+        expect(preview).toMatchSnapshot();
+    });
+});
@@ -0,0 +1,204 @@
+/*
+ * Copyright 2026 Element Creations Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+import { logger as rootLogger } from "matrix-js-sdk/src/logger";
+import { type IPreviewUrlResponse, type MatrixClient, MatrixError } from "matrix-js-sdk/src/matrix";
+import { decode } from "html-entities";
+
+import type { UrlPreview } from "@element-hq/web-shared-components";
+import { mediaFromMxc } from "../customisations/Media";
+import { thumbHeight } from "../ImageUtils";
+
+const logger = rootLogger.getChild("UrlPreviewFetcher");
+
+export const PREVIEW_WIDTH_PX = 478;
+export const PREVIEW_HEIGHT_PX = 200;
+export const MIN_PREVIEW_PX = 96;
+export const MIN_IMAGE_SIZE_BYTES = 8192;
+
+/**
+ * Handles fetching and parsing URL previews.
+ * Maintains a cache of previously fetched previews; call `clearCache` when
+ * media visibility changes so images are re-fetched with the correct visibility.
+ */
+export class UrlPreviewFetcher {
+    private readonly cache = new Map<string, UrlPreview>();
+
+    public constructor(
+        private readonly client: MatrixClient,
+        private readonly previewRequestTs: number,
+        private readonly showTooltips: boolean,
+    ) {}
+
+    public clearCache(): void {
+        this.cache.clear();
+    }
+
+    /**
+     * Parse a numeric value from OpenGraph. The OpenGraph spec defines all values as strings
+     * although Synapse may return these values as numbers. To be compatible, test strings
+     * and numbers.
+     * @param value The numeric value
+     * @returns A number if the value parsed correctly, or undefined otherwise.
+     */
+    private static getNumberFromOpenGraph(value: number | string | undefined): number | undefined {
+        if (typeof value === "number") {
+            return value;
+        } else if (typeof value === "string" && value) {
+            const i = Number.parseInt(value, 10);
+            if (!Number.isNaN(i)) return i;
+        }
+        return undefined;
+    }
+
+    /**
+     * Calculate the best possible title from an opengraph response.
+     * @param response The opengraph response
+     * @param link The link being used to preview.
+     * @returns The title value.
+     */
+    private static getBaseMetadataFromResponse(
+        response: IPreviewUrlResponse,
+        link: string,
+    ): Pick<UrlPreview, "title" | "description" | "siteName"> {
+        let title =
+            typeof response["og:title"] === "string" && response["og:title"].trim()
+                ? response["og:title"].trim()
+                : undefined;
+        let description =
+            typeof response["og:description"] === "string" && response["og:description"].trim()
+                ? response["og:description"].trim()
+                : undefined;
+        const siteName =
+            typeof response["og:site_name"] === "string" && response["og:site_name"].trim()
+                ? response["og:site_name"].trim()
+                : new URL(link).hostname;
+
+        if (!title && description) {
+            title = description;
+            description = undefined;
+        } else if (!title && siteName) {
+            title = siteName;
+        } else if (!title) {
+            title = link;
+        }
+
+        if (description && description.toLowerCase() === siteName.toLowerCase()) {
+            description = undefined;
+        }
+
+        return { title, description: description && decode(description), siteName };
+    }
+
+    /**
+     * Calculate the best possible author from an opengraph response.
+     * @param response The opengraph response
+     * @returns The author value, or undefined if no valid author could be found.
+     */
+    private static getAuthorFromResponse(response: IPreviewUrlResponse): UrlPreview["author"] {
+        let calculatedAuthor: string | undefined;
+        if (response["og:type"] === "article") {
+            if (typeof response["article:author"] === "string" && response["article:author"]) {
+                calculatedAuthor = response["article:author"];
+            }
+        }
+        if (typeof response["profile:username"] === "string" && response["profile:username"]) {
+            calculatedAuthor = response["profile:username"];
+        }
+        if (calculatedAuthor && URL.canParse(calculatedAuthor)) {
+            return undefined;
+        }
+        return calculatedAuthor;
+    }
+
+    /**
+     * Calculate whether the provided image from the preview response is an full size preview or
+     * a site icon.
+     * @returns `true` if the image should be used as a preview, otherwise `false`
+     */
+    private static isImagePreview(width?: number, height?: number, bytes?: number): boolean {
+        if (width && width < MIN_PREVIEW_PX) return false;
+        if (height && height < MIN_PREVIEW_PX) return false;
+        if (bytes && bytes < MIN_IMAGE_SIZE_BYTES) return false;
+        return true;
+    }
+
+    /**
+     * Fetch a preview for a single URL, returning a cached result if available.
+     * @param link The URL to preview.
+     * @param loadMedia Whether to include the preview image. Pass false when media is hidden.
+     */
+    public async fetchPreview(link: string, loadMedia: boolean): Promise<UrlPreview | null> {
+        const cached = this.cache.get(link);
+        if (cached) return cached;
+
+        let response: IPreviewUrlResponse;
+        try {
+            response = await this.client.getUrlPreview(link, this.previewRequestTs);
+        } catch (error) {
+            if (error instanceof MatrixError && error.httpStatus === 404) {
+                logger.debug("Failed to get URL preview: ", error);
+            } else {
+                logger.error("Failed to get URL preview: ", error);
+            }
+            return null;
+        }
+
+        const { title, description, siteName } = UrlPreviewFetcher.getBaseMetadataFromResponse(response, link);
+        const author = UrlPreviewFetcher.getAuthorFromResponse(response);
+        const hasImage = response["og:image"] && typeof response["og:image"] === "string";
+
+        if (title === link && !hasImage) {
+            return null;
+        }
+
+        let image: UrlPreview["image"];
+        let siteIcon: string | undefined;
+
+        if (typeof response["og:image"] === "string" && loadMedia) {
+            const media = mediaFromMxc(response["og:image"], this.client);
+            const declaredHeight = UrlPreviewFetcher.getNumberFromOpenGraph(response["og:image:height"]);
+            const declaredWidth = UrlPreviewFetcher.getNumberFromOpenGraph(response["og:image:width"]);
+            const imageSize = UrlPreviewFetcher.getNumberFromOpenGraph(response["matrix:image:size"]);
+            const alt = typeof response["og:image:alt"] === "string" ? response["og:image:alt"] : undefined;
+
+            if (UrlPreviewFetcher.isImagePreview(declaredWidth, declaredHeight, imageSize)) {
+                const width = Math.min(declaredWidth ?? PREVIEW_WIDTH_PX, PREVIEW_WIDTH_PX);
+                const height =
+                    thumbHeight(width, declaredHeight, PREVIEW_WIDTH_PX, PREVIEW_WIDTH_PX) ?? PREVIEW_WIDTH_PX;
+                const thumb = media.getThumbnailOfSourceHttp(PREVIEW_WIDTH_PX, PREVIEW_HEIGHT_PX, "scale");
+                const playable = !!response["og:video"] || !!response["og:video:type"] || !!response["og:audio"];
+                if (thumb) {
+                    image = {
+                        imageThumb: thumb,
+                        imageFull: media.srcHttp ?? thumb,
+                        width,
+                        height,
+                        fileSize: UrlPreviewFetcher.getNumberFromOpenGraph(response["matrix:image:size"]),
+                        alt,
+                        playable,
+                    };
+                }
+            } else if (media.srcHttp) {
+                siteIcon = media.srcHttp;
+            }
+        }
+
+        const result = {
+            link,
+            title,
+            author,
+            description,
+            siteName,
+            siteIcon,
+            showTooltipOnLink: !!(link !== title && this.showTooltips),
+            image,
+        } satisfies UrlPreview;
+        this.cache.set(link, result);
+        return result;
+    }
+}
@@ -0,0 +1,95 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`UrlPreviewFetcher > handles different kinds of opengraph responses { 'og:url': 'https://example.org', 'og:type': 'document', 'og:description': 'A description', 'og:title': '' } 1`] = `
+{
+  "author": undefined,
+  "description": undefined,
+  "image": undefined,
+  "link": "https://example.org",
+  "showTooltipOnLink": false,
+  "siteIcon": undefined,
+  "siteName": "example.org",
+  "title": "A description",
+}
+`;
+
+exports[`UrlPreviewFetcher > handles different kinds of opengraph responses { 'og:url': 'https://example.org', 'og:type': 'document', 'og:site_name': 'Site name', 'og:title': '' } 1`] = `
+{
+  "author": undefined,
+  "description": undefined,
+  "image": undefined,
+  "link": "https://example.org",
+  "showTooltipOnLink": false,
+  "siteIcon": undefined,
+  "siteName": "Site name",
+  "title": "Site name",
+}
+`;
+
+exports[`UrlPreviewFetcher > handles different kinds of opengraph responses { 'og:url': 'https://example.org', 'og:type': 'document', 'og:title': 'Basic title' } 1`] = `
+{
+  "author": undefined,
+  "description": undefined,
+  "image": undefined,
+  "link": "https://example.org",
+  "showTooltipOnLink": false,
+  "siteIcon": undefined,
+  "siteName": "example.org",
+  "title": "Basic title",
+}
+`;
+
+exports[`UrlPreviewFetcher > handles different kinds of opengraph responses { 'og:url': 'https://example.org', 'og:type': 'document', 'og:title': 'Cool blog', 'og:site_name': 'Cool site' } 1`] = `
+{
+  "author": undefined,
+  "description": undefined,
+  "image": undefined,
+  "link": "https://example.org",
+  "showTooltipOnLink": false,
+  "siteIcon": undefined,
+  "siteName": "Cool site",
+  "title": "Cool blog",
+}
+`;
+
+exports[`UrlPreviewFetcher > handles different kinds of opengraph responses { 'og:url': 'https://example.org', 'og:type': 'document', 'og:title': 'Media test', 'og:image:height': '500', 'og:image:width': 500, 'matrix:image:size': 10000, 'og:image': 'mxc://example.org/abc' } 1`] = `
+{
+  "author": undefined,
+  "description": undefined,
+  "image": {
+    "alt": undefined,
+    "fileSize": 10000,
+    "height": 478,
+    "imageFull": "https://example.org/image/src",
+    "imageThumb": "https://example.org/image/thumb",
+    "playable": false,
+    "width": 478,
+  },
+  "link": "https://example.org",
+  "showTooltipOnLink": false,
+  "siteIcon": undefined,
+  "siteName": "example.org",
+  "title": "Media test",
+}
+`;
+
+exports[`UrlPreviewFetcher > should preview a URL with media 1`] = `
+{
+  "author": undefined,
+  "description": undefined,
+  "image": {
+    "alt": undefined,
+    "fileSize": 10000,
+    "height": 128,
+    "imageFull": "https://example.org/image/src",
+    "imageThumb": "https://example.org/image/thumb",
+    "playable": false,
+    "width": 128,
+  },
+  "link": "https://example.org",
+  "showTooltipOnLink": false,
+  "siteIcon": undefined,
+  "siteName": "example.org",
+  "title": "This is an example!",
+}
+`;