Refactor and move MVideoBody to shared components (#32849)
* init MVideoBody to shared components, including test, stories and view * fix prettier and other warnings * move video message body to shared view + app viewmodel * Fix prettier warnings and masking spinner for tests * stabilize VideoBodyView screenshots with local media asset * Disable spinner from changing image all the time * Added mask over video spinner to prevent issues with new generated images on playwright tests * Update prettier fix * Update snapshot * Add tests to cover different states of Video * Update code to prevent the previous component Hack fix regarding jumps on the timeline. * Update snapshot * Update code to improve code quality for Sonar + update snapshot * adde documentation snippets * refactor: move m.video rendering into body factory * docs: add tsdoc for video body view model * docs: add thumbnail tsdoc for video body view model * docs: add content-url tsdoc for video body view model * docs: add dimensions tsdoc for video body view model * docs: add aspect-ratio tsdoc for video body view model * docs: add tsdoc for video body view state * refactor: replace video body view state enum * refactor: remove duplicate video body state init * refactor: drop unused video body view state attribute * Fix Prettier * Update snapshot screenshot * test: restore video story screenshot mask * chore: refresh PR head * Add mask to screenshot to pass CI tests * test: narrow video story mask hook * Fix easy Sonar warnings in video body components * Move shared message body views into event-tile layout * Move shared message body visual baselines * Revert unrelated shared message body moves
This commit is contained in:
Zack
@@ -11,15 +11,18 @@ import {
|
||||
DecryptionFailureBodyView,
|
||||
FileBodyView,
|
||||
RedactedBodyView,
|
||||
VideoBodyView,
|
||||
useCreateAutoDisposedViewModel,
|
||||
} from "@element-hq/web-shared-components";
|
||||
|
||||
import { type IBodyProps } from "./IBodyProps";
|
||||
import RoomContext from "../../../contexts/RoomContext";
|
||||
import RoomContext, { TimelineRenderingType } from "../../../contexts/RoomContext";
|
||||
import { LocalDeviceVerificationStateContext } from "../../../contexts/LocalDeviceVerificationStateContext";
|
||||
import { useMediaVisible } from "../../../hooks/useMediaVisible";
|
||||
import { DecryptionFailureBodyViewModel } from "../../../viewmodels/room/timeline/event-tile/body/DecryptionFailureBodyViewModel";
|
||||
import { FileBodyViewModel } from "../../../viewmodels/message-body/FileBodyViewModel";
|
||||
import { RedactedBodyViewModel } from "../../../viewmodels/message-body/RedactedBodyViewModel";
|
||||
import { VideoBodyViewModel } from "../../../viewmodels/message-body/VideoBodyViewModel";
|
||||
|
||||
type MBodyComponent = React.ComponentType<IBodyProps>;
|
||||
|
||||
@@ -59,6 +62,78 @@ export function FileBodyFactory({
|
||||
return <FileBodyView vm={vm} refIFrame={refIFrame} refLink={refLink} className="mx_MFileBody" />;
|
||||
}
|
||||
|
||||
export function VideoBodyFactory({
|
||||
mxEvent,
|
||||
mediaEventHelper,
|
||||
forExport,
|
||||
inhibitInteraction,
|
||||
}: Readonly<Pick<IBodyProps, "mxEvent" | "mediaEventHelper" | "forExport" | "inhibitInteraction">>): JSX.Element {
|
||||
const { timelineRenderingType } = useContext(RoomContext);
|
||||
const [mediaVisible, setMediaVisible] = useMediaVisible(mxEvent);
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
|
||||
const vm = useCreateAutoDisposedViewModel(
|
||||
() =>
|
||||
new VideoBodyViewModel({
|
||||
mxEvent,
|
||||
mediaEventHelper,
|
||||
forExport,
|
||||
inhibitInteraction,
|
||||
mediaVisible,
|
||||
onPreviewClick: (): void => setMediaVisible(true),
|
||||
videoRef,
|
||||
}),
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
vm.loadInitialMediaIfVisible();
|
||||
}, [vm]);
|
||||
|
||||
useEffect(() => {
|
||||
vm.setEvent(mxEvent, mediaEventHelper);
|
||||
}, [mxEvent, mediaEventHelper, vm]);
|
||||
|
||||
useEffect(() => {
|
||||
vm.setForExport(forExport);
|
||||
}, [forExport, vm]);
|
||||
|
||||
useEffect(() => {
|
||||
vm.setInhibitInteraction(inhibitInteraction);
|
||||
}, [inhibitInteraction, vm]);
|
||||
|
||||
useEffect(() => {
|
||||
vm.setMediaVisible(mediaVisible);
|
||||
}, [mediaVisible, vm]);
|
||||
|
||||
useEffect(() => {
|
||||
vm.setOnPreviewClick((): void => setMediaVisible(true));
|
||||
}, [setMediaVisible, vm]);
|
||||
|
||||
const showFileBody =
|
||||
!forExport &&
|
||||
timelineRenderingType !== TimelineRenderingType.Room &&
|
||||
timelineRenderingType !== TimelineRenderingType.Pinned &&
|
||||
timelineRenderingType !== TimelineRenderingType.Search;
|
||||
|
||||
return (
|
||||
<VideoBodyView
|
||||
vm={vm}
|
||||
className="mx_MVideoBody"
|
||||
containerClassName="mx_MVideoBody_container"
|
||||
videoRef={videoRef}
|
||||
>
|
||||
{showFileBody ? (
|
||||
<FileBodyFactory
|
||||
mxEvent={mxEvent}
|
||||
mediaEventHelper={mediaEventHelper}
|
||||
forExport={forExport}
|
||||
showFileInfo={false}
|
||||
/>
|
||||
) : null}
|
||||
</VideoBodyView>
|
||||
);
|
||||
}
|
||||
|
||||
export function RedactedBodyFactory({ mxEvent, ref }: Pick<IBodyProps, "mxEvent" | "ref">): JSX.Element {
|
||||
const vm = useCreateAutoDisposedViewModel(() => new RedactedBodyViewModel({ mxEvent }));
|
||||
|
||||
@@ -87,9 +162,11 @@ export function DecryptionFailureBodyFactory({ mxEvent, ref }: Pick<IBodyProps,
|
||||
return <DecryptionFailureBodyView vm={vm} ref={ref} className="mx_DecryptionFailureBody mx_EventTile_content" />;
|
||||
}
|
||||
|
||||
// Message body factory registry.
|
||||
// Start small: only m.file currently routes to the new FileBodyView path.
|
||||
const MESSAGE_BODY_TYPES = new Map<string, MBodyComponent>([[MsgType.File, FileBodyFactory]]);
|
||||
// Message body factory registry for bodies that already route through view-model-backed wrappers.
|
||||
const MESSAGE_BODY_TYPES = new Map<string, MBodyComponent>([
|
||||
[MsgType.File, FileBodyFactory],
|
||||
[MsgType.Video, VideoBodyFactory],
|
||||
]);
|
||||
|
||||
// Render a body using the picked factory.
|
||||
// Falls back to the provided factory when msgtype has no specific handler.
|
||||
|
||||
@@ -1,353 +0,0 @@
|
||||
/*
|
||||
Copyright 2024 New Vector Ltd.
|
||||
Copyright 2015-2021 The Matrix.org Foundation C.I.C.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE files in the repository root for full details.
|
||||
*/
|
||||
|
||||
import React, { type ReactNode } from "react";
|
||||
import { decode } from "blurhash";
|
||||
import { type MediaEventContent } from "matrix-js-sdk/src/types";
|
||||
import { logger } from "matrix-js-sdk/src/logger";
|
||||
|
||||
import { _t } from "../../../languageHandler";
|
||||
import SettingsStore from "../../../settings/SettingsStore";
|
||||
import InlineSpinner from "../elements/InlineSpinner";
|
||||
import { mediaFromContent } from "../../../customisations/Media";
|
||||
import { BLURHASH_FIELD } from "../../../utils/image-media";
|
||||
import { type IBodyProps } from "./IBodyProps";
|
||||
import { type ImageSize, suggestedSize as suggestedVideoSize } from "../../../settings/enums/ImageSize";
|
||||
import RoomContext, { TimelineRenderingType } from "../../../contexts/RoomContext";
|
||||
import MediaProcessingError from "./shared/MediaProcessingError";
|
||||
import { HiddenMediaPlaceholder } from "./HiddenMediaPlaceholder";
|
||||
import { useMediaVisible } from "../../../hooks/useMediaVisible";
|
||||
import { FileBodyFactory, renderMBody } from "./MBodyFactory";
|
||||
|
||||
interface IState {
|
||||
decryptedUrl: string | null;
|
||||
decryptedThumbnailUrl: string | null;
|
||||
decryptedBlob: Blob | null;
|
||||
error?: any;
|
||||
fetchingData: boolean;
|
||||
posterLoading: boolean;
|
||||
blurhashUrl: string | null;
|
||||
}
|
||||
|
||||
interface IProps extends IBodyProps {
|
||||
/**
|
||||
* Should the media be behind a preview.
|
||||
*/
|
||||
mediaVisible: boolean;
|
||||
/**
|
||||
* Set the visibility of the media event.
|
||||
* @param visible Should the event be visible.
|
||||
*/
|
||||
setMediaVisible: (visible: boolean) => void;
|
||||
}
|
||||
|
||||
class MVideoBodyInner extends React.PureComponent<IProps, IState> {
|
||||
public static contextType = RoomContext;
|
||||
declare public context: React.ContextType<typeof RoomContext>;
|
||||
|
||||
private videoRef = React.createRef<HTMLVideoElement>();
|
||||
private sizeWatcher?: string;
|
||||
|
||||
public state = {
|
||||
fetchingData: false,
|
||||
decryptedUrl: null,
|
||||
decryptedThumbnailUrl: null,
|
||||
decryptedBlob: null,
|
||||
error: null,
|
||||
posterLoading: false,
|
||||
blurhashUrl: null,
|
||||
};
|
||||
|
||||
private onClick = (): void => {
|
||||
this.props.setMediaVisible(true);
|
||||
};
|
||||
|
||||
private getContentUrl(): string | undefined {
|
||||
const content = this.props.mxEvent.getContent<MediaEventContent>();
|
||||
// During export, the content url will point to the MSC, which will later point to a local url
|
||||
if (this.props.forExport) return content.file?.url ?? content.url;
|
||||
const media = mediaFromContent(content);
|
||||
if (media.isEncrypted) {
|
||||
return this.state.decryptedUrl ?? undefined;
|
||||
} else {
|
||||
return media.srcHttp ?? undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private hasContentUrl(): boolean {
|
||||
const url = this.getContentUrl();
|
||||
return !!url && !url.startsWith("data:");
|
||||
}
|
||||
|
||||
private getThumbUrl(): string | null {
|
||||
// there's no need of thumbnail when the content is local
|
||||
if (this.props.forExport) return null;
|
||||
|
||||
const content = this.props.mxEvent.getContent<MediaEventContent>();
|
||||
const media = mediaFromContent(content);
|
||||
|
||||
if (media.isEncrypted && this.state.decryptedThumbnailUrl) {
|
||||
return this.state.decryptedThumbnailUrl;
|
||||
} else if (this.state.posterLoading) {
|
||||
return this.state.blurhashUrl;
|
||||
} else if (media.hasThumbnail) {
|
||||
return media.thumbnailHttp;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private loadBlurhash(): void {
|
||||
const info = this.props.mxEvent.getContent()?.info;
|
||||
if (!info[BLURHASH_FIELD]) return;
|
||||
|
||||
const canvas = document.createElement("canvas");
|
||||
|
||||
const { w: width, h: height } = suggestedVideoSize(SettingsStore.getValue("Images.size") as ImageSize, {
|
||||
w: info.w,
|
||||
h: info.h,
|
||||
});
|
||||
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
|
||||
const pixels = decode(info[BLURHASH_FIELD], width, height);
|
||||
const ctx = canvas.getContext("2d")!;
|
||||
const imgData = ctx.createImageData(width, height);
|
||||
imgData.data.set(pixels);
|
||||
ctx.putImageData(imgData, 0, 0);
|
||||
|
||||
this.setState({
|
||||
blurhashUrl: canvas.toDataURL(),
|
||||
posterLoading: true,
|
||||
});
|
||||
|
||||
const content = this.props.mxEvent.getContent<MediaEventContent>();
|
||||
const media = mediaFromContent(content);
|
||||
if (media.hasThumbnail) {
|
||||
const image = new Image();
|
||||
image.onload = () => {
|
||||
this.setState({ posterLoading: false });
|
||||
};
|
||||
image.src = media.thumbnailHttp!;
|
||||
}
|
||||
}
|
||||
|
||||
private async downloadVideo(): Promise<void> {
|
||||
try {
|
||||
this.loadBlurhash();
|
||||
} catch (e) {
|
||||
logger.error("Failed to load blurhash", e);
|
||||
}
|
||||
|
||||
if (this.props.mediaEventHelper?.media.isEncrypted && this.state.decryptedUrl === null) {
|
||||
try {
|
||||
const autoplay = SettingsStore.getValue("autoplayVideo") as boolean;
|
||||
const thumbnailUrl = await this.props.mediaEventHelper.thumbnailUrl.value;
|
||||
if (autoplay) {
|
||||
logger.log("Preloading video");
|
||||
this.setState({
|
||||
decryptedUrl: await this.props.mediaEventHelper.sourceUrl.value,
|
||||
decryptedThumbnailUrl: thumbnailUrl,
|
||||
decryptedBlob: await this.props.mediaEventHelper.sourceBlob.value,
|
||||
});
|
||||
} else {
|
||||
logger.log("NOT preloading video");
|
||||
const content = this.props.mxEvent.getContent<MediaEventContent>();
|
||||
|
||||
let mimetype = content?.info?.mimetype;
|
||||
|
||||
// clobber quicktime muxed files to be considered MP4 so browsers
|
||||
// are willing to play them
|
||||
if (mimetype == "video/quicktime") {
|
||||
mimetype = "video/mp4";
|
||||
}
|
||||
|
||||
this.setState({
|
||||
// For Chrome and Electron, we need to set some non-empty `src` to
|
||||
// enable the play button. Firefox does not seem to care either
|
||||
// way, so it's fine to do for all browsers.
|
||||
decryptedUrl: `data:${mimetype},`,
|
||||
decryptedThumbnailUrl: thumbnailUrl || `data:${mimetype},`,
|
||||
decryptedBlob: null,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn("Unable to decrypt attachment: ", err);
|
||||
// Set a placeholder image when we can't decrypt the image.
|
||||
this.setState({
|
||||
error: err,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async componentDidMount(): Promise<void> {
|
||||
this.sizeWatcher = SettingsStore.watchSetting("Images.size", null, () => {
|
||||
this.forceUpdate(); // we don't really have a reliable thing to update, so just update the whole thing
|
||||
});
|
||||
|
||||
// Do not attempt to load the media if we do not want to show previews here.
|
||||
if (this.props.mediaVisible) {
|
||||
await this.downloadVideo();
|
||||
}
|
||||
}
|
||||
|
||||
public async componentDidUpdate(prevProps: Readonly<IProps>): Promise<void> {
|
||||
if (!prevProps.mediaVisible && this.props.mediaVisible) {
|
||||
await this.downloadVideo();
|
||||
}
|
||||
}
|
||||
|
||||
public componentWillUnmount(): void {
|
||||
SettingsStore.unwatchSetting(this.sizeWatcher);
|
||||
}
|
||||
|
||||
private videoOnPlay = async (): Promise<void> => {
|
||||
if (this.hasContentUrl() || this.state.fetchingData || this.state.error) {
|
||||
// We have the file, we are fetching the file, or there is an error.
|
||||
return;
|
||||
}
|
||||
this.setState({
|
||||
// To stop subsequent download attempts
|
||||
fetchingData: true,
|
||||
});
|
||||
if (!this.props.mediaEventHelper!.media.isEncrypted) {
|
||||
this.setState({
|
||||
error: "No file given in content",
|
||||
});
|
||||
return;
|
||||
}
|
||||
this.setState(
|
||||
{
|
||||
decryptedUrl: await this.props.mediaEventHelper!.sourceUrl.value,
|
||||
decryptedBlob: await this.props.mediaEventHelper!.sourceBlob.value,
|
||||
fetchingData: false,
|
||||
},
|
||||
() => {
|
||||
if (!this.videoRef.current) return;
|
||||
this.videoRef.current.play();
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
protected get showFileBody(): boolean {
|
||||
return (
|
||||
this.context.timelineRenderingType !== TimelineRenderingType.Room &&
|
||||
this.context.timelineRenderingType !== TimelineRenderingType.Pinned &&
|
||||
this.context.timelineRenderingType !== TimelineRenderingType.Search
|
||||
);
|
||||
}
|
||||
|
||||
private getFileBody = (): ReactNode => {
|
||||
if (this.props.forExport) return null;
|
||||
return this.showFileBody && renderMBody({ ...this.props, showFileInfo: false }, FileBodyFactory);
|
||||
};
|
||||
|
||||
public render(): React.ReactNode {
|
||||
const content = this.props.mxEvent.getContent();
|
||||
const autoplay = !this.props.inhibitInteraction && SettingsStore.getValue("autoplayVideo");
|
||||
|
||||
let aspectRatio;
|
||||
if (content.info?.w && content.info?.h) {
|
||||
aspectRatio = `${content.info.w}/${content.info.h}`;
|
||||
}
|
||||
const { w: maxWidth, h: maxHeight } = suggestedVideoSize(SettingsStore.getValue("Images.size") as ImageSize, {
|
||||
w: content.info?.w,
|
||||
h: content.info?.h,
|
||||
});
|
||||
|
||||
// HACK: This div fills out space while the video loads, to prevent scroll jumps
|
||||
const spaceFiller = <div style={{ width: maxWidth, height: maxHeight }} />;
|
||||
|
||||
if (this.state.error !== null) {
|
||||
return (
|
||||
<MediaProcessingError className="mx_MVideoBody">
|
||||
{_t("timeline|m.video|error_decrypting")}
|
||||
</MediaProcessingError>
|
||||
);
|
||||
}
|
||||
|
||||
// Users may not even want to show a poster, so instead show a preview button.
|
||||
if (!this.props.mediaVisible) {
|
||||
return (
|
||||
<span className="mx_MVideoBody">
|
||||
<div
|
||||
className="mx_MVideoBody_container"
|
||||
style={{ width: maxWidth, height: maxHeight, aspectRatio }}
|
||||
>
|
||||
<HiddenMediaPlaceholder onClick={this.onClick}>
|
||||
{_t("timeline|m.video|show_video")}
|
||||
</HiddenMediaPlaceholder>
|
||||
</div>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
// Important: If we aren't autoplaying and we haven't decrypted it yet, show a video with a poster.
|
||||
if (!this.props.forExport && content.file !== undefined && this.state.decryptedUrl === null && autoplay) {
|
||||
// Need to decrypt the attachment
|
||||
// The attachment is decrypted in componentDidMount.
|
||||
// For now show a spinner.
|
||||
return (
|
||||
<span className="mx_MVideoBody">
|
||||
<div className="mx_MVideoBody_container" style={{ maxWidth, maxHeight, aspectRatio }}>
|
||||
<InlineSpinner />
|
||||
</div>
|
||||
{spaceFiller}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
const contentUrl = this.getContentUrl();
|
||||
const thumbUrl = this.getThumbUrl();
|
||||
let poster: string | undefined;
|
||||
let preload = "metadata";
|
||||
if (content.info && thumbUrl) {
|
||||
poster = thumbUrl;
|
||||
preload = "none";
|
||||
}
|
||||
|
||||
const fileBody = this.getFileBody();
|
||||
return (
|
||||
<span className="mx_MVideoBody">
|
||||
<div className="mx_MVideoBody_container" style={{ maxWidth, maxHeight, aspectRatio }}>
|
||||
<video
|
||||
className="mx_MVideoBody"
|
||||
ref={this.videoRef}
|
||||
src={contentUrl}
|
||||
title={content.body}
|
||||
controls={!this.props.inhibitInteraction}
|
||||
// Disable downloading as it doesn't work with e2ee video,
|
||||
// users should use the dedicated Download button in the Message Action Bar
|
||||
controlsList="nodownload"
|
||||
// The video uses a cross-origin request.
|
||||
// Firefox explicitly bypasses services workers for crossorigin
|
||||
// video elements without crossorigin attribute.
|
||||
crossOrigin="anonymous"
|
||||
preload={preload}
|
||||
muted={autoplay}
|
||||
autoPlay={autoplay}
|
||||
poster={poster}
|
||||
onPlay={this.videoOnPlay}
|
||||
/>
|
||||
{spaceFiller}
|
||||
</div>
|
||||
{fileBody}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Wrap MVideoBody component so we can use a hook here.
|
||||
const MVideoBody: React.FC<IBodyProps> = (props) => {
|
||||
const [mediaVisible, setVisible] = useMediaVisible(props.mxEvent);
|
||||
return <MVideoBodyInner mediaVisible={mediaVisible} setMediaVisible={setVisible} {...props} />;
|
||||
};
|
||||
|
||||
export default MVideoBody;
|
||||
@@ -28,14 +28,19 @@ import { type IBodyProps } from "./IBodyProps";
|
||||
import TextualBody from "./TextualBody";
|
||||
import MImageBody from "./MImageBody";
|
||||
import MVoiceOrAudioBody from "./MVoiceOrAudioBody";
|
||||
import MVideoBody from "./MVideoBody";
|
||||
import MStickerBody from "./MStickerBody";
|
||||
import MPollBody from "./MPollBody";
|
||||
import MLocationBody from "./MLocationBody";
|
||||
import MjolnirBody from "./MjolnirBody";
|
||||
import MBeaconBody from "./MBeaconBody";
|
||||
import { type GetRelationsForEvent, type IEventTileOps } from "../rooms/EventTile";
|
||||
import { DecryptionFailureBodyFactory, FileBodyFactory, RedactedBodyFactory, renderMBody } from "./MBodyFactory";
|
||||
import {
|
||||
DecryptionFailureBodyFactory,
|
||||
FileBodyFactory,
|
||||
RedactedBodyFactory,
|
||||
VideoBodyFactory,
|
||||
renderMBody,
|
||||
} from "./MBodyFactory";
|
||||
|
||||
// onMessageAllowed is handled internally
|
||||
interface IProps extends Omit<IBodyProps, "onMessageAllowed" | "mediaEventHelper"> {
|
||||
@@ -65,7 +70,7 @@ const baseBodyTypes = new Map<string, React.ComponentType<IBodyProps>>([
|
||||
[MsgType.Image, MImageBody],
|
||||
[MsgType.File, (props: IBodyProps) => renderMBody(props, FileBodyFactory)!],
|
||||
[MsgType.Audio, MVoiceOrAudioBody],
|
||||
[MsgType.Video, MVideoBody],
|
||||
[MsgType.Video, VideoBodyFactory],
|
||||
]);
|
||||
const baseEvTypes = new Map<string, React.ComponentType<IBodyProps>>([
|
||||
[EventType.Sticker, MStickerBody],
|
||||
@@ -260,7 +265,8 @@ export default class MessageEvent extends React.Component<IProps> implements IMe
|
||||
}
|
||||
|
||||
if (
|
||||
((BodyType === MImageBody || BodyType == MVideoBody) && !this.validateImageOrVideoMimetype(content)) ||
|
||||
((BodyType === MImageBody || BodyType === VideoBodyFactory) &&
|
||||
!this.validateImageOrVideoMimetype(content)) ||
|
||||
(BodyType === MStickerBody && !this.validateStickerMimetype(content))
|
||||
) {
|
||||
BodyType = this.bodyTypes.get(MsgType.File)!;
|
||||
|
||||
@@ -0,0 +1,543 @@
|
||||
/*
|
||||
* Copyright 2026 Element Creations Ltd.
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
* Please see LICENSE files in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { decode } from "blurhash";
|
||||
import { type RefObject } from "react";
|
||||
import { logger } from "matrix-js-sdk/src/logger";
|
||||
import { type MatrixEvent } from "matrix-js-sdk/src/matrix";
|
||||
import { type MediaEventContent, type VideoInfo } from "matrix-js-sdk/src/types";
|
||||
import {
|
||||
BaseViewModel,
|
||||
VideoBodyViewState,
|
||||
type VideoBodyViewModel as VideoBodyViewModelInterface,
|
||||
type VideoBodyViewSnapshot,
|
||||
} from "@element-hq/web-shared-components";
|
||||
|
||||
import { _t } from "../../languageHandler";
|
||||
import SettingsStore from "../../settings/SettingsStore";
|
||||
import { mediaFromContent } from "../../customisations/Media";
|
||||
import { BLURHASH_FIELD } from "../../utils/image-media";
|
||||
import { type ImageSize, suggestedSize as suggestedVideoSize } from "../../settings/enums/ImageSize";
|
||||
import { type MediaEventHelper } from "../../utils/MediaEventHelper";
|
||||
|
||||
export interface VideoBodyViewModelProps {
|
||||
/**
|
||||
* Video event being rendered.
|
||||
*/
|
||||
mxEvent: MatrixEvent;
|
||||
/**
|
||||
* Helper for resolving encrypted and unencrypted media sources.
|
||||
*/
|
||||
mediaEventHelper?: MediaEventHelper;
|
||||
/**
|
||||
* Whether the video is being rendered for export instead of live playback.
|
||||
*/
|
||||
forExport?: boolean;
|
||||
/**
|
||||
* Whether playback controls and autoplay should be disabled.
|
||||
*/
|
||||
inhibitInteraction?: boolean;
|
||||
/**
|
||||
* Whether the media should currently be shown instead of the preview button.
|
||||
*/
|
||||
mediaVisible: boolean;
|
||||
/**
|
||||
* Callback invoked when the hidden-media preview is revealed.
|
||||
*/
|
||||
onPreviewClick?: () => void;
|
||||
/**
|
||||
* Ref to the underlying video element used for replay after lazy decryption.
|
||||
*/
|
||||
videoRef: RefObject<HTMLVideoElement | null>;
|
||||
}
|
||||
|
||||
interface InternalState {
|
||||
/**
|
||||
* Decrypted playable media URL for encrypted videos.
|
||||
*/
|
||||
decryptedUrl: string | null;
|
||||
/**
|
||||
* Decrypted thumbnail URL for encrypted videos.
|
||||
*/
|
||||
decryptedThumbnailUrl: string | null;
|
||||
/**
|
||||
* Decrypted media blob cached for download or replay.
|
||||
*/
|
||||
decryptedBlob: Blob | null;
|
||||
/**
|
||||
* Last media-processing error, if any.
|
||||
*/
|
||||
error: unknown | null;
|
||||
/**
|
||||
* Whether an on-demand media fetch is in progress.
|
||||
*/
|
||||
fetchingData: boolean;
|
||||
/**
|
||||
* Whether the blurhash poster is being shown while the real poster loads.
|
||||
*/
|
||||
posterLoading: boolean;
|
||||
/**
|
||||
* Data URL generated from the blurhash placeholder.
|
||||
*/
|
||||
blurhashUrl: string | null;
|
||||
/**
|
||||
* Current media sizing preference from settings.
|
||||
*/
|
||||
imageSize: ImageSize;
|
||||
}
|
||||
|
||||
type VideoInfoWithBlurhash = VideoInfo & {
|
||||
[BLURHASH_FIELD]?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* View model for the video message body, encapsulating media-loading and playback state.
|
||||
*/
|
||||
export class VideoBodyViewModel
|
||||
extends BaseViewModel<VideoBodyViewSnapshot, VideoBodyViewModelProps>
|
||||
implements VideoBodyViewModelInterface
|
||||
{
|
||||
private state: InternalState;
|
||||
|
||||
public constructor(props: VideoBodyViewModelProps) {
|
||||
const initialState = VideoBodyViewModel.createInitialState();
|
||||
super(props, VideoBodyViewModel.computeSnapshot(props, initialState));
|
||||
this.state = initialState;
|
||||
|
||||
const imageSizeWatcherRef = SettingsStore.watchSetting("Images.size", null, (_s, _r, _l, _nvl, value) => {
|
||||
this.setImageSize(value as ImageSize);
|
||||
});
|
||||
this.disposables.track(() => SettingsStore.unwatchSetting(imageSizeWatcherRef));
|
||||
}
|
||||
|
||||
public loadInitialMediaIfVisible(): void {
|
||||
if (this.props.mediaVisible) {
|
||||
void this.downloadVideo();
|
||||
}
|
||||
}
|
||||
|
||||
private static createInitialState(): InternalState {
|
||||
return {
|
||||
fetchingData: false,
|
||||
decryptedUrl: null,
|
||||
decryptedThumbnailUrl: null,
|
||||
decryptedBlob: null,
|
||||
error: null,
|
||||
posterLoading: false,
|
||||
blurhashUrl: null,
|
||||
imageSize: SettingsStore.getValue("Images.size") as ImageSize,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the aspect ratio for the video frame from the event metadata, when available.
|
||||
*/
|
||||
private static getAspectRatio(mxEvent: MatrixEvent): string | undefined {
|
||||
const { w, h } = (mxEvent.getContent<MediaEventContent>().info as VideoInfoWithBlurhash | undefined) ?? {};
|
||||
if (!w || !h) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return `${w}/${h}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the rendered video dimensions from the event metadata and current image-size setting.
|
||||
*/
|
||||
private static getDimensions(mxEvent: MatrixEvent, imageSize: ImageSize): Required<{ w?: number; h?: number }> {
|
||||
const { w, h } = (mxEvent.getContent<MediaEventContent>().info as VideoInfoWithBlurhash | undefined) ?? {};
|
||||
return suggestedVideoSize(imageSize, { w, h });
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the current playable video source URL for the event.
|
||||
*/
|
||||
private static getContentUrl(props: VideoBodyViewModelProps, state: InternalState): string | undefined {
|
||||
const content = props.mxEvent.getContent<MediaEventContent>();
|
||||
if (props.forExport) {
|
||||
return content.file?.url ?? content.url;
|
||||
}
|
||||
|
||||
const media = mediaFromContent(content);
|
||||
if (media.isEncrypted) {
|
||||
return state.decryptedUrl ?? undefined;
|
||||
}
|
||||
|
||||
return media.srcHttp ?? undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the best thumbnail or poster URL for the current video state.
|
||||
*/
|
||||
private static getThumbnailUrl(props: VideoBodyViewModelProps, state: InternalState): string | null {
|
||||
if (props.forExport) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const content = props.mxEvent.getContent<MediaEventContent>();
|
||||
const media = mediaFromContent(content);
|
||||
|
||||
if (media.isEncrypted && state.decryptedThumbnailUrl) {
|
||||
return state.decryptedThumbnailUrl;
|
||||
}
|
||||
if (state.posterLoading) {
|
||||
return state.blurhashUrl;
|
||||
}
|
||||
if (media.hasThumbnail) {
|
||||
return media.thumbnailHttp;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static computeSnapshot(props: VideoBodyViewModelProps, state: InternalState): VideoBodyViewSnapshot {
|
||||
const content = props.mxEvent.getContent<MediaEventContent>();
|
||||
const autoplay = !props.inhibitInteraction && (SettingsStore.getValue("autoplayVideo") as boolean);
|
||||
const aspectRatio = VideoBodyViewModel.getAspectRatio(props.mxEvent);
|
||||
const { w: maxWidth, h: maxHeight } = VideoBodyViewModel.getDimensions(props.mxEvent, state.imageSize);
|
||||
|
||||
if (state.error !== null) {
|
||||
return {
|
||||
state: VideoBodyViewState.ERROR,
|
||||
errorLabel: _t("timeline|m.video|error_decrypting"),
|
||||
maxWidth,
|
||||
maxHeight,
|
||||
aspectRatio,
|
||||
};
|
||||
}
|
||||
|
||||
if (!props.mediaVisible) {
|
||||
return {
|
||||
state: VideoBodyViewState.HIDDEN,
|
||||
hiddenButtonLabel: _t("timeline|m.video|show_video"),
|
||||
maxWidth,
|
||||
maxHeight,
|
||||
aspectRatio,
|
||||
};
|
||||
}
|
||||
|
||||
if (!props.forExport && content.file !== undefined && state.decryptedUrl === null && autoplay) {
|
||||
return {
|
||||
state: VideoBodyViewState.LOADING,
|
||||
maxWidth,
|
||||
maxHeight,
|
||||
aspectRatio,
|
||||
};
|
||||
}
|
||||
|
||||
const thumbnailUrl = VideoBodyViewModel.getThumbnailUrl(props, state);
|
||||
let preload: VideoBodyViewSnapshot["preload"] = "metadata";
|
||||
let poster: string | undefined;
|
||||
if (content.info && thumbnailUrl) {
|
||||
preload = "none";
|
||||
poster = thumbnailUrl;
|
||||
}
|
||||
|
||||
return {
|
||||
state: VideoBodyViewState.READY,
|
||||
videoLabel: content.body,
|
||||
videoTitle: content.body,
|
||||
maxWidth,
|
||||
maxHeight,
|
||||
aspectRatio,
|
||||
src: VideoBodyViewModel.getContentUrl(props, state),
|
||||
poster,
|
||||
preload,
|
||||
controls: !props.inhibitInteraction,
|
||||
muted: autoplay,
|
||||
autoPlay: autoplay,
|
||||
};
|
||||
}
|
||||
|
||||
private updateSnapshotFromState(): void {
|
||||
this.snapshot.set(VideoBodyViewModel.computeSnapshot(this.props, this.state));
|
||||
}
|
||||
|
||||
private hasContentUrl(): boolean {
|
||||
const url = VideoBodyViewModel.getContentUrl(this.props, this.state);
|
||||
return !!url && !url.startsWith("data:");
|
||||
}
|
||||
|
||||
private setImageSize(imageSize: ImageSize): void {
|
||||
if (this.state.imageSize === imageSize) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.state = {
|
||||
...this.state,
|
||||
imageSize,
|
||||
};
|
||||
this.updateSnapshotFromState();
|
||||
}
|
||||
|
||||
private resetMediaState(): void {
|
||||
this.state = {
|
||||
...this.state,
|
||||
decryptedUrl: null,
|
||||
decryptedThumbnailUrl: null,
|
||||
decryptedBlob: null,
|
||||
error: null,
|
||||
fetchingData: false,
|
||||
posterLoading: false,
|
||||
blurhashUrl: null,
|
||||
};
|
||||
}
|
||||
|
||||
private loadBlurhash(): void {
|
||||
const info = this.props.mxEvent.getContent<MediaEventContent>().info as VideoInfoWithBlurhash | undefined;
|
||||
const blurhash = info?.[BLURHASH_FIELD];
|
||||
if (!blurhash) {
|
||||
return;
|
||||
}
|
||||
|
||||
const canvas = document.createElement("canvas");
|
||||
const { w: width, h: height } = VideoBodyViewModel.getDimensions(this.props.mxEvent, this.state.imageSize);
|
||||
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
|
||||
const pixels = decode(blurhash, width, height);
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) {
|
||||
return;
|
||||
}
|
||||
|
||||
const imgData = ctx.createImageData(width, height);
|
||||
imgData.data.set(pixels);
|
||||
ctx.putImageData(imgData, 0, 0);
|
||||
|
||||
this.state = {
|
||||
...this.state,
|
||||
blurhashUrl: canvas.toDataURL(),
|
||||
posterLoading: true,
|
||||
};
|
||||
this.updateSnapshotFromState();
|
||||
|
||||
const media = mediaFromContent(this.props.mxEvent.getContent<MediaEventContent>());
|
||||
if (!media.hasThumbnail || !media.thumbnailHttp) {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentEvent = this.props.mxEvent;
|
||||
const image = new Image();
|
||||
image.onload = (): void => {
|
||||
if (this.isDisposed || currentEvent !== this.props.mxEvent || !this.state.posterLoading) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.state = {
|
||||
...this.state,
|
||||
posterLoading: false,
|
||||
};
|
||||
this.updateSnapshotFromState();
|
||||
};
|
||||
image.src = media.thumbnailHttp;
|
||||
}
|
||||
|
||||
private async downloadVideo(): Promise<void> {
|
||||
try {
|
||||
this.loadBlurhash();
|
||||
} catch (error) {
|
||||
logger.error("Failed to load blurhash", error);
|
||||
}
|
||||
|
||||
if (!this.props.mediaEventHelper?.media.isEncrypted || this.state.decryptedUrl !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentEvent = this.props.mxEvent;
|
||||
const currentHelper = this.props.mediaEventHelper;
|
||||
try {
|
||||
const autoplay = !this.props.inhibitInteraction && (SettingsStore.getValue("autoplayVideo") as boolean);
|
||||
const thumbnailUrl = await currentHelper.thumbnailUrl.value;
|
||||
|
||||
if (
|
||||
this.isDisposed ||
|
||||
currentEvent !== this.props.mxEvent ||
|
||||
currentHelper !== this.props.mediaEventHelper
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (autoplay) {
|
||||
logger.log("Preloading video");
|
||||
this.state = {
|
||||
...this.state,
|
||||
decryptedUrl: await currentHelper.sourceUrl.value,
|
||||
decryptedThumbnailUrl: thumbnailUrl,
|
||||
decryptedBlob: await currentHelper.sourceBlob.value,
|
||||
};
|
||||
} else {
|
||||
logger.log("NOT preloading video");
|
||||
const content = currentEvent.getContent<MediaEventContent>();
|
||||
let mimetype = content.info?.mimetype ?? "application/octet-stream";
|
||||
if (mimetype === "video/quicktime") {
|
||||
mimetype = "video/mp4";
|
||||
}
|
||||
|
||||
this.state = {
|
||||
...this.state,
|
||||
decryptedUrl: `data:${mimetype},`,
|
||||
decryptedThumbnailUrl: thumbnailUrl || `data:${mimetype},`,
|
||||
decryptedBlob: null,
|
||||
};
|
||||
}
|
||||
|
||||
this.updateSnapshotFromState();
|
||||
} catch (error) {
|
||||
if (
|
||||
this.isDisposed ||
|
||||
currentEvent !== this.props.mxEvent ||
|
||||
currentHelper !== this.props.mediaEventHelper
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.warn("Unable to decrypt attachment: ", error);
|
||||
this.state = {
|
||||
...this.state,
|
||||
error,
|
||||
};
|
||||
this.updateSnapshotFromState();
|
||||
}
|
||||
}
|
||||
|
||||
public setEvent(mxEvent: MatrixEvent, mediaEventHelper?: MediaEventHelper): void {
|
||||
if (this.props.mxEvent === mxEvent && this.props.mediaEventHelper === mediaEventHelper) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.props = {
|
||||
...this.props,
|
||||
mxEvent,
|
||||
mediaEventHelper,
|
||||
};
|
||||
this.resetMediaState();
|
||||
this.updateSnapshotFromState();
|
||||
|
||||
if (this.props.mediaVisible) {
|
||||
void this.downloadVideo();
|
||||
}
|
||||
}
|
||||
|
||||
public setForExport(forExport?: boolean): void {
|
||||
if (this.props.forExport === forExport) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.props = {
|
||||
...this.props,
|
||||
forExport,
|
||||
};
|
||||
this.updateSnapshotFromState();
|
||||
}
|
||||
|
||||
public setInhibitInteraction(inhibitInteraction?: boolean): void {
|
||||
if (this.props.inhibitInteraction === inhibitInteraction) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.props = {
|
||||
...this.props,
|
||||
inhibitInteraction,
|
||||
};
|
||||
this.updateSnapshotFromState();
|
||||
}
|
||||
|
||||
public setMediaVisible(mediaVisible: boolean): void {
|
||||
if (this.props.mediaVisible === mediaVisible) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.props = {
|
||||
...this.props,
|
||||
mediaVisible,
|
||||
};
|
||||
this.updateSnapshotFromState();
|
||||
|
||||
if (mediaVisible) {
|
||||
void this.downloadVideo();
|
||||
}
|
||||
}
|
||||
|
||||
public setOnPreviewClick(onPreviewClick?: () => void): void {
|
||||
if (this.props.onPreviewClick === onPreviewClick) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.props = {
|
||||
...this.props,
|
||||
onPreviewClick,
|
||||
};
|
||||
}
|
||||
|
||||
public onPreviewClick = (): void => {
|
||||
this.props.onPreviewClick?.();
|
||||
};
|
||||
|
||||
public onPlay = async (): Promise<void> => {
|
||||
if (this.hasContentUrl() || this.state.fetchingData || this.state.error !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.state = {
|
||||
...this.state,
|
||||
fetchingData: true,
|
||||
};
|
||||
|
||||
if (!this.props.mediaEventHelper?.media.isEncrypted) {
|
||||
this.state = {
|
||||
...this.state,
|
||||
error: "No file given in content",
|
||||
fetchingData: false,
|
||||
};
|
||||
this.updateSnapshotFromState();
|
||||
return;
|
||||
}
|
||||
|
||||
const currentEvent = this.props.mxEvent;
|
||||
const currentHelper = this.props.mediaEventHelper;
|
||||
|
||||
try {
|
||||
const decryptedUrl = await currentHelper.sourceUrl.value;
|
||||
const decryptedBlob = await currentHelper.sourceBlob.value;
|
||||
|
||||
if (
|
||||
this.isDisposed ||
|
||||
currentEvent !== this.props.mxEvent ||
|
||||
currentHelper !== this.props.mediaEventHelper
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.state = {
|
||||
...this.state,
|
||||
decryptedUrl,
|
||||
decryptedBlob,
|
||||
fetchingData: false,
|
||||
};
|
||||
this.updateSnapshotFromState();
|
||||
this.props.videoRef.current?.play();
|
||||
} catch (error) {
|
||||
if (
|
||||
this.isDisposed ||
|
||||
currentEvent !== this.props.mxEvent ||
|
||||
currentHelper !== this.props.mediaEventHelper
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.warn("Unable to decrypt attachment: ", error);
|
||||
this.state = {
|
||||
...this.state,
|
||||
error,
|
||||
fetchingData: false,
|
||||
};
|
||||
this.updateSnapshotFromState();
|
||||
}
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user