From 63e65fc60a6639face9384797e7d95c42110ca4a Mon Sep 17 00:00:00 2001 From: Sirius Date: Fri, 17 Jul 2026 11:18:10 +0100 Subject: [PATCH] Use url preview bundle for URL preview in timeline (MSC4095) (#34170) * use url preview bundle preview content in timeline * fixed linting errors * claude wrote a test! * moved reading settings from the view model into the component * claude wrote more tests * applied reviews --- .../views/messages/TextualBodyFactory.tsx | 3 + apps/web/src/utils/UrlPreviewFetcher.test.ts | 126 ++++++++++++ apps/web/src/utils/UrlPreviewFetcher.ts | 49 +++++ .../message-body/UrlPreviewGroupViewModel.ts | 35 +++- .../UrlPreviewGroupViewModel-test.ts | 187 +++++++++++++++++- 5 files changed, 384 insertions(+), 16 deletions(-) diff --git a/apps/web/src/components/views/messages/TextualBodyFactory.tsx b/apps/web/src/components/views/messages/TextualBodyFactory.tsx index 252d2f4e0e..56fb20fc49 100644 --- a/apps/web/src/components/views/messages/TextualBodyFactory.tsx +++ b/apps/web/src/components/views/messages/TextualBodyFactory.tsx @@ -33,6 +33,7 @@ import EditMessageComposer from "../rooms/EditMessageComposer"; import { EditWysiwygComposer } from "../rooms/wysiwyg_composer"; import { UrlPreviewGroupViewModel } from "../../../viewmodels/message-body/UrlPreviewGroupViewModel"; import PlatformPeg from "../../../PlatformPeg"; +import { useSettingValue } from "../../../hooks/useSettings"; const logger = rootLogger.getChild("TextualBodyFactory"); @@ -61,6 +62,7 @@ export function TextualBodyFactory(props: Readonly): JSX.Element { const willHaveWrapper = !!props.replacingEventId || !!props.isSeeingThroughMessageHiddenForModeration || isEmote; const stripReply = !props.mxEvent.replacingEvent() && !!getParentEventId(props.mxEvent); const contentRef = useRef(null); + const urlPreviewBundleEnabled = useSettingValue("feature_msc4095_url_preview_bundle"); const textualBodyVm = useCreateAutoDisposedViewModel( () => @@ -121,6 +123,7 @@ export function TextualBodyFactory(props: Readonly): JSX.Element { }, visible: props.showUrlPreview ?? false, showTooltips: PlatformPeg.get()?.needsUrlTooltips() ?? true, + urlPreviewBundleEnabled, }), ); diff --git a/apps/web/src/utils/UrlPreviewFetcher.test.ts b/apps/web/src/utils/UrlPreviewFetcher.test.ts index c4f0ebb59b..31c1115818 100644 --- a/apps/web/src/utils/UrlPreviewFetcher.test.ts +++ b/apps/web/src/utils/UrlPreviewFetcher.test.ts @@ -9,6 +9,7 @@ import { vi, describe, it, expect, beforeAll, afterAll, type Mock } from "vitest import type { IPreviewUrlResponse, MatrixClient } from "matrix-js-sdk/src/matrix"; import { UrlPreviewFetcher } from "./UrlPreviewFetcher"; +import { type UnstableBundledUrlPreviewSingle } from "../../@types/url-preview"; const IMAGE_MXC = "mxc://example.org/abc"; const BASIC_PREVIEW_OGDATA = { @@ -220,4 +221,129 @@ describe("UrlPreviewFetcher", () => { const preview = await fetcher.fetchPreview("https://example.org", true); expect(preview).toMatchSnapshot(); }); + + describe("previewFromBundle", () => { + const BASIC_BUNDLE: UnstableBundledUrlPreviewSingle = { + "matched_url": "https://example.org/page", + "og:title": "Bundled title", + "og:description": "Bundled description", + "og:url": "https://example.org/canonical", + }; + + const IMAGE_BUNDLE: UnstableBundledUrlPreviewSingle = { + ...BASIC_BUNDLE, + "og:image": IMAGE_MXC, + "og:image:type": "image/png", + "og:image:width": 500, + "og:image:height": 400, + "matrix:image:size": 10000, + }; + + function mockMedia(client: { mxcUrlToHttp: Mock }): void { + // eslint-disable-next-line no-restricted-properties + client.mxcUrlToHttp.mockImplementation((url, width) => { + expect(url).toEqual(IMAGE_MXC); + if (width) return "https://example.org/image/thumb"; + return "https://example.org/image/src"; + }); + } + + it("should map basic bundle fields without an image", () => { + const { fetcher, client } = getFetcher(); + const preview = fetcher.previewFromBundle(BASIC_BUNDLE); + expect(preview).toEqual({ + link: "https://example.org/page", + title: "Bundled title", + siteName: "example.org", + showTooltipOnLink: false, + description: "Bundled description", + ogUrl: "https://example.org/canonical", + }); + // eslint-disable-next-line no-restricted-properties + expect(client.mxcUrlToHttp).not.toHaveBeenCalled(); + }); + + it("should fall back to the matched_url when there is no title", () => { + const { fetcher } = getFetcher(); + const preview = fetcher.previewFromBundle({ matched_url: "https://example.org/page" }); + expect(preview.title).toEqual("https://example.org/page"); + expect(preview.showTooltipOnLink).toBe(false); + }); + + it("should set showTooltipOnLink when tooltips are enabled and title differs from the URL", () => { + const { client } = getFetcher(); + const fetcher = new UrlPreviewFetcher(client as unknown as MatrixClient, 0, true); + const preview = fetcher.previewFromBundle(BASIC_BUNDLE); + expect(preview.showTooltipOnLink).toBe(true); + }); + + // Unlike fetchPreview, the tooltip flag is computed against the raw og:title rather than + // the resolved title, so a missing og:title still shows a tooltip even though the displayed + // title falls back to the matched_url. + it("should set showTooltipOnLink when tooltips are enabled and og:title is absent", () => { + const { client } = getFetcher(); + const fetcher = new UrlPreviewFetcher(client as unknown as MatrixClient, 0, true); + const preview = fetcher.previewFromBundle({ matched_url: "https://example.org/page" }); + expect(preview.showTooltipOnLink).toBe(true); + }); + + it("should not set showTooltipOnLink when tooltips are enabled but og:title equals the URL", () => { + const { client } = getFetcher(); + const fetcher = new UrlPreviewFetcher(client as unknown as MatrixClient, 0, true); + const preview = fetcher.previewFromBundle({ + "matched_url": "https://example.org/page", + "og:title": "https://example.org/page", + }); + expect(preview.showTooltipOnLink).toBe(false); + }); + + it("should include the image when all image fields are present", () => { + const { fetcher, client } = getFetcher(); + mockMedia(client); + const preview = fetcher.previewFromBundle(IMAGE_BUNDLE); + expect(preview.image).toEqual({ + imageThumb: "https://example.org/image/thumb", + imageFull: "https://example.org/image/src", + imageType: "image/png", + mxcImageFull: IMAGE_MXC, + width: 500, + height: 400, + playable: false, + }); + }); + + it.each>([ + { "og:image": undefined }, + { "og:image:type": undefined }, + { "og:image:width": undefined }, + { "og:image:height": undefined }, + // Non-numeric dimensions are ignored (bundle values are trusted as-is). + { "og:image:width": "500" as unknown as number }, + ])("should omit the image when image metadata is incomplete %s", (override) => { + const { fetcher, client } = getFetcher(); + mockMedia(client); + const preview = fetcher.previewFromBundle({ ...IMAGE_BUNDLE, ...override }); + expect(preview.image).toBeUndefined(); + }); + + it("should omit the image when the media mxc URL is malformed", () => { + const { fetcher, client } = getFetcher(); + // A malformed/unresolvable mxc yields no HTTP URL. + // eslint-disable-next-line no-restricted-properties + client.mxcUrlToHttp.mockReturnValue(null); + const preview = fetcher.previewFromBundle(IMAGE_BUNDLE); + expect(preview.image).toBeUndefined(); + // The rest of the preview is still returned. + expect(preview.title).toEqual("Bundled title"); + }); + + it("should compute the siteName from the matched_url hostname", () => { + const { fetcher } = getFetcher(); + const preview = fetcher.previewFromBundle({ + ...BASIC_BUNDLE, + matched_url: "https://sub.example.com:8443/some/path?q=1", + }); + expect(preview.siteName).toEqual("sub.example.com"); + }); + }); }); diff --git a/apps/web/src/utils/UrlPreviewFetcher.ts b/apps/web/src/utils/UrlPreviewFetcher.ts index 17bdb3ed17..708f96d49c 100644 --- a/apps/web/src/utils/UrlPreviewFetcher.ts +++ b/apps/web/src/utils/UrlPreviewFetcher.ts @@ -12,6 +12,7 @@ import { decode } from "html-entities"; import type { UrlPreview } from "@element-hq/web-shared-components"; import { mediaFromMxc } from "../customisations/Media"; import { thumbHeight } from "../ImageUtils"; +import { type UnstableBundledUrlPreviewSingle } from "../../@types/url-preview"; const logger = rootLogger.getChild("UrlPreviewFetcher"); @@ -206,4 +207,52 @@ export class UrlPreviewFetcher { this.cache.set(link, result); return result; } + + /* + * Convert an MSC4095 URL preview bundle item to a UrlPreview + */ + public previewFromBundle(single: UnstableBundledUrlPreviewSingle): UrlPreview { + // missing fields from the bundle because backend does provide it: + // - siteName (can be computed) + // - favicon + // - media is a video or audio? + // TODO in next PR: URL previews in encrypted chat? + const hasImage = + typeof single["og:image"] === "string" && + typeof single["og:image:type"] === "string" && + typeof single["og:image:width"] === "number" && + typeof single["og:image:height"] === "number"; + + const preview: UrlPreview = { + link: single.matched_url, + title: single["og:title"] ?? single.matched_url, + siteName: new URL(single.matched_url).hostname, + showTooltipOnLink: !!(single.matched_url !== single["og:title"] && this.showTooltips), + description: single["og:description"], + ogUrl: single["og:url"], + }; + + if (hasImage) { + const media = mediaFromMxc(single["og:image"], this.client); + const thumb = media.getThumbnailOfSourceHttp(PREVIEW_WIDTH_PX, PREVIEW_HEIGHT_PX, "scale"); + + // cannot rule out the mxc:// url is malformed because + // the sender can specify anything + if (media.srcHttp === null || thumb === null) { + return preview; + } + + preview.image = { + imageThumb: thumb, + imageFull: media.srcHttp, + imageType: single["og:image:type"] as string, + mxcImageFull: single["og:image"] as string, + width: single["og:image:width"] as number, + height: single["og:image:height"] as number, + playable: false, // TODO: do we know? + }; + } + + return preview; + } } diff --git a/apps/web/src/viewmodels/message-body/UrlPreviewGroupViewModel.ts b/apps/web/src/viewmodels/message-body/UrlPreviewGroupViewModel.ts index 5b421d6455..0f6505a2bc 100644 --- a/apps/web/src/viewmodels/message-body/UrlPreviewGroupViewModel.ts +++ b/apps/web/src/viewmodels/message-body/UrlPreviewGroupViewModel.ts @@ -5,7 +5,7 @@ * Please see LICENSE files in the repository root for full details. */ -import { type MatrixClient, type MatrixEvent } from "matrix-js-sdk/src/matrix"; +import { MsgType, type MatrixClient, type MatrixEvent } from "matrix-js-sdk/src/matrix"; import { BaseViewModel, type UrlPreview, @@ -17,6 +17,7 @@ import { type UrlPreviewVisibilityChanged } from "@matrix-org/analytics-events/t import { PosthogAnalytics } from "../../PosthogAnalytics"; import { isPermalinkHost } from "../../utils/permalinks/Permalinks"; import { UrlPreviewFetcher } from "../../utils/UrlPreviewFetcher"; +import { type RoomMessageEventContent } from "../../../@types/url-preview"; // From https://github.com/matrix-org/matrix-spec-proposals/pull/4095 export const BUNDLED_LINK_PREVIEWS = "com.beeper.linkpreviews"; @@ -41,6 +42,7 @@ export interface UrlPreviewGroupViewModelProps { mediaVisible: boolean; showTooltips: boolean; onImageClicked: (preview: UrlPreview) => void; + urlPreviewBundleEnabled: boolean; } export class UrlPreviewGroupViewModel @@ -167,14 +169,29 @@ export class UrlPreviewGroupViewModel } const loadMedia = this.visibility === PreviewVisibility.Visible; - const previews = - this.visibility <= PreviewVisibility.UserHidden - ? [] - : await Promise.all( - this.links - .slice(0, this.limitPreviews ? MAX_PREVIEWS_WHEN_LIMITED : undefined) - .map((link) => this.fetcher.fetchPreview(link, loadMedia)), - ); + let previews: (UrlPreview | null)[] | undefined; + + if (this.visibility <= PreviewVisibility.UserHidden) { + previews = []; + } + + const content = this.props.mxEvent.getContent(); + if (content.msgtype === MsgType.Text && this.props.urlPreviewBundleEnabled) { + const messageContent = content as RoomMessageEventContent; + + if (messageContent[BUNDLED_LINK_PREVIEWS] !== undefined) { + previews = messageContent[BUNDLED_LINK_PREVIEWS] + .slice(0, this.limitPreviews ? MAX_PREVIEWS_WHEN_LIMITED : undefined) + .map((preview) => this.fetcher.previewFromBundle(preview)); + } + } + + previews ??= await Promise.all( + this.links + .slice(0, this.limitPreviews ? MAX_PREVIEWS_WHEN_LIMITED : undefined) + .map((link) => this.fetcher.fetchPreview(link, loadMedia)), + ); + this.snapshot.merge({ previews: previews.filter((p) => !!p), totalPreviewCount: this.links.length, diff --git a/apps/web/test/viewmodels/message-body/UrlPreviewGroupViewModel-test.ts b/apps/web/test/viewmodels/message-body/UrlPreviewGroupViewModel-test.ts index c508410dc2..e3a1e502be 100644 --- a/apps/web/test/viewmodels/message-body/UrlPreviewGroupViewModel-test.ts +++ b/apps/web/test/viewmodels/message-body/UrlPreviewGroupViewModel-test.ts @@ -8,9 +8,10 @@ import { expect } from "@jest/globals"; import type { MockedObject } from "jest-mock-vitest-adapter"; -import type { MatrixClient } from "matrix-js-sdk/src/matrix"; +import { MsgType, type MatrixClient } from "matrix-js-sdk/src/matrix"; import { BUNDLED_LINK_PREVIEWS, + MAX_PREVIEWS_WHEN_LIMITED, UrlPreviewGroupViewModel, } from "../../../src/viewmodels/message-body/UrlPreviewGroupViewModel"; import type { UrlPreview } from "@element-hq/web-shared-components"; @@ -25,9 +26,46 @@ const BASIC_PREVIEW_OGDATA = { "og:site_name": "Example.org", }; -function getViewModel( - { mediaVisible, visible, showPreview } = { mediaVisible: true, visible: true, showPreview: true }, -): { +const BUNDLE_PREVIEW_ONE = { + "matched_url": "https://example.org/1", + "og:title": "Bundled one", + "og:description": "First bundled preview", + "og:url": "https://example.org/1", +}; +const BUNDLE_PREVIEW_TWO = { + "matched_url": "https://example.org/2", + "og:title": "Bundled two", + "og:description": "Second bundled preview", + "og:url": "https://example.org/2", +}; +const BUNDLE_PREVIEW_THREE = { + "matched_url": "https://example.org/3", + "og:title": "Bundled three", + "og:description": "Third bundled preview", + "og:url": "https://example.org/3", +}; +const BUNDLE_PREVIEW_WITH_IMAGE = { + "matched_url": "https://example.org/image", + "og:title": "Bundled with image", + "og:image": IMAGE_MXC, + "og:image:type": "image/png", + "og:image:width": 128, + "og:image:height": 128, +}; + +function getViewModel({ + mediaVisible = true, + visible = true, + showPreview = true, + urlPreviewBundleEnabled = true, + content, +}: { + mediaVisible?: boolean; + visible?: boolean; + showPreview?: boolean; + urlPreviewBundleEnabled?: boolean; + content?: object; +} = {}): { vm: UrlPreviewGroupViewModel; client: MockedObject; onImageClicked: jest.Mock; @@ -49,9 +87,11 @@ function getViewModel( type: "m.room.message", content: { ...(showPreview ? undefined : { [BUNDLED_LINK_PREVIEWS]: [] }), + ...content, }, id: "$id", }), + urlPreviewBundleEnabled, }); return { vm, client, onImageClicked }; } @@ -96,7 +136,12 @@ describe("UrlPreviewGroupViewModel", () => { ]); }); it("should hide preview when invisible", async () => { - const { vm, client } = getViewModel({ visible: false, mediaVisible: true, showPreview: true }); + const { vm, client } = getViewModel({ + visible: false, + mediaVisible: true, + showPreview: true, + urlPreviewBundleEnabled: false, + }); const msg = document.createElement("div"); msg.innerHTML = 'Test'; await vm.updateEventElement(msg); @@ -104,7 +149,12 @@ describe("UrlPreviewGroupViewModel", () => { expect(client.getUrlPreview).not.toHaveBeenCalled(); }); it("should ignore media when mediaVisible is false", async () => { - const { vm, client } = getViewModel({ mediaVisible: false, visible: true, showPreview: true }); + const { vm, client } = getViewModel({ + mediaVisible: false, + visible: true, + showPreview: true, + urlPreviewBundleEnabled: false, + }); client.getUrlPreview.mockResolvedValueOnce({ "og:title": "This is an example!", "og:type": "document", @@ -178,7 +228,12 @@ describe("UrlPreviewGroupViewModel", () => { expect(vm.getSnapshot()).toMatchSnapshot(); }); it("should hide a preview if the message requests it", async () => { - const { vm, client } = getViewModel({ showPreview: false, mediaVisible: true, visible: true }); + const { vm, client } = getViewModel({ + showPreview: false, + mediaVisible: true, + visible: true, + urlPreviewBundleEnabled: false, + }); client.getUrlPreview.mockResolvedValueOnce(BASIC_PREVIEW_OGDATA); const msg = document.createElement("div"); msg.innerHTML = 'Test'; @@ -207,4 +262,122 @@ describe("UrlPreviewGroupViewModel", () => { await vm.updateEventElement(msg); expect(vm.getSnapshot().previews).toHaveLength(item.hasPreview ? 1 : 0); }); + + describe("bundled link previews (MSC4095)", () => { + it("should render bundled previews when the message is text and the bundle is enabled", async () => { + const { vm, client } = getViewModel({ + urlPreviewBundleEnabled: true, + content: { + msgtype: MsgType.Text, + [BUNDLED_LINK_PREVIEWS]: [BUNDLE_PREVIEW_ONE, BUNDLE_PREVIEW_TWO], + }, + }); + const msg = document.createElement("div"); + msg.innerHTML = 'Test1Test2'; + await vm.updateEventElement(msg); + const { previews } = vm.getSnapshot(); + expect(previews).toMatchObject([ + { + link: "https://example.org/1", + title: "Bundled one", + description: "First bundled preview", + siteName: "example.org", + ogUrl: "https://example.org/1", + }, + { + link: "https://example.org/2", + title: "Bundled two", + description: "Second bundled preview", + siteName: "example.org", + ogUrl: "https://example.org/2", + }, + ]); + // Bundled previews are provided inline and must not trigger network fetches. + expect(client.getUrlPreview).not.toHaveBeenCalled(); + }); + + it("should render an image for a bundled preview", async () => { + const { vm, client } = getViewModel({ + urlPreviewBundleEnabled: true, + content: { + msgtype: MsgType.Text, + [BUNDLED_LINK_PREVIEWS]: [BUNDLE_PREVIEW_WITH_IMAGE], + }, + }); + // eslint-disable-next-line no-restricted-properties + client.mxcUrlToHttp.mockReturnValue("https://example.org/image/src"); + const msg = document.createElement("div"); + msg.innerHTML = 'Test'; + await vm.updateEventElement(msg); + const { previews } = vm.getSnapshot(); + expect(previews).toHaveLength(1); + expect(previews[0].image).toMatchObject({ + mxcImageFull: IMAGE_MXC, + imageType: "image/png", + width: 128, + height: 128, + }); + expect(client.getUrlPreview).not.toHaveBeenCalled(); + }); + + it("should limit bundled previews and reveal the rest when the limit is toggled", async () => { + const { vm, client } = getViewModel({ + urlPreviewBundleEnabled: true, + content: { + msgtype: MsgType.Text, + [BUNDLED_LINK_PREVIEWS]: [BUNDLE_PREVIEW_ONE, BUNDLE_PREVIEW_TWO, BUNDLE_PREVIEW_THREE], + }, + }); + const msg = document.createElement("div"); + msg.innerHTML = + 'Test1Test2Test3'; + await vm.updateEventElement(msg); + + let snapshot = vm.getSnapshot(); + expect(snapshot.previews).toHaveLength(MAX_PREVIEWS_WHEN_LIMITED); + expect(snapshot.previewsLimited).toBe(true); + expect(snapshot.overPreviewLimit).toBe(true); + + await vm.onTogglePreviewLimit(); + snapshot = vm.getSnapshot(); + expect(snapshot.previews).toHaveLength(3); + expect(snapshot.previewsLimited).toBe(false); + expect(client.getUrlPreview).not.toHaveBeenCalled(); + }); + + it("should fetch previews instead of using the bundle when the bundle setting is disabled", async () => { + const { vm, client } = getViewModel({ + urlPreviewBundleEnabled: false, + content: { + msgtype: MsgType.Text, + [BUNDLED_LINK_PREVIEWS]: [BUNDLE_PREVIEW_ONE], + }, + }); + client.getUrlPreview.mockResolvedValueOnce(BASIC_PREVIEW_OGDATA); + const msg = document.createElement("div"); + msg.innerHTML = 'Test1'; + await vm.updateEventElement(msg); + const { previews } = vm.getSnapshot(); + expect(client.getUrlPreview).toHaveBeenCalledWith("https://example.org/1", expect.anything()); + // The fetched preview wins over the ignored bundle entry. + expect(previews).toMatchObject([{ title: "This is an example!" }]); + }); + + it("should fetch previews instead of using the bundle when the message is not a text message", async () => { + const { vm, client } = getViewModel({ + urlPreviewBundleEnabled: true, + content: { + msgtype: MsgType.Notice, + [BUNDLED_LINK_PREVIEWS]: [BUNDLE_PREVIEW_ONE], + }, + }); + client.getUrlPreview.mockResolvedValueOnce(BASIC_PREVIEW_OGDATA); + const msg = document.createElement("div"); + msg.innerHTML = 'Test1'; + await vm.updateEventElement(msg); + const { previews } = vm.getSnapshot(); + expect(client.getUrlPreview).toHaveBeenCalledWith("https://example.org/1", expect.anything()); + expect(previews).toMatchObject([{ title: "This is an example!" }]); + }); + }); });