Add ringing for matrixRTC (#11870)

* Add ringing for matrixRTC - since we are using m.mentions we start with the Notifier - an event in the Notifier will result in a IncomingCall toast - incomingCallToast is responsible for ringing (as long as one can see the toast it rings) This should make sure visual and audio signal are in sync. Signed-off-by: Timo K <toger5@hotmail.de> * use typed CallNotifyContent Signed-off-by: Timo K <toger5@hotmail.de> * update tests Signed-off-by: Timo K <toger5@hotmail.de> * change to callId Signed-off-by: Timo K <toger5@hotmail.de> * fix tests Signed-off-by: Timo K <toger5@hotmail.de> * only ring in 1:1 calls notify in rooms < 15 member Signed-off-by: Timo K <toger5@hotmail.de> * call_id fallback Signed-off-by: Timo K <toger5@hotmail.de> * Update src/Notifier.ts Co-authored-by: Robin <robin@robin.town> * review Signed-off-by: Timo K <toger5@hotmail.de> * add tests Signed-off-by: Timo K <toger5@hotmail.de> * more tests Signed-off-by: Timo K <toger5@hotmail.de> * unused import Signed-off-by: Timo K <toger5@hotmail.de> * String -> string Signed-off-by: Timo K <toger5@hotmail.de> --------- Signed-off-by: Timo K <toger5@hotmail.de> Co-authored-by: Robin <robin@robin.town>
2023-11-21 17:12:08 +00:00
parent 7ca0cd13d0
commit a26c2d3c78
7 changed files with 230 additions and 50 deletions
@@ -28,6 +28,7 @@ import {
    SyncStateData,
    IRoomTimelineData,
    M_LOCATION,
+    EventType,
 } from "matrix-js-sdk/src/matrix";
 import { logger } from "matrix-js-sdk/src/logger";
 import { PermissionChanged as PermissionChangedEvent } from "@matrix-org/analytics-events/types/typescript/PermissionChanged";
@@ -54,7 +55,6 @@ import { SdkContextClass } from "./contexts/SDKContext";
 import { localNotificationsAreSilenced, createLocalNotificationSettingsIfNeeded } from "./utils/notifications";
 import { getIncomingCallToastKey, IncomingCallToast } from "./toasts/IncomingCallToast";
 import ToastStore from "./stores/ToastStore";
-import { ElementCall } from "./models/Call";
 import { VoiceBroadcastChunkEventType, VoiceBroadcastInfoEventType } from "./voice-broadcast";
 import { getSenderName } from "./utils/event/getSenderName";
 import { stripPlainReply } from "./utils/Reply";
@@ -516,13 +516,27 @@ class NotifierClass {
     * Some events require special handling such as showing in-app toasts
     */
    private performCustomEventHandling(ev: MatrixEvent): void {
-        if (ElementCall.CALL_EVENT_TYPE.names.includes(ev.getType()) && SettingsStore.getValue("feature_group_calls")) {
+        if (
+            EventType.CallNotify === ev.getType() &&
+            SettingsStore.getValue("feature_group_calls") &&
+            (ev.getAge() ?? 0) < 10000
+        ) {
+            const content = ev.getContent();
+            const roomId = ev.getRoomId();
+            if (typeof content.call_id !== "string") {
+                logger.warn("Received malformatted CallNotify event. Did not contain 'call_id' of type 'string'");
+                return;
+            }
+            if (!roomId) {
+                logger.warn("Could not get roomId for CallNotify event");
+                return;
+            }
            ToastStore.sharedInstance().addOrReplaceToast({
-                key: getIncomingCallToastKey(ev.getStateKey()!),
+                key: getIncomingCallToastKey(content.call_id, roomId),
                priority: 100,
                component: IncomingCallToast,
                bodyClassName: "mx_IncomingCallToast",
-                props: { callEvent: ev },
+                props: { notifyEvent: ev },
            });
        }
    }
@@ -33,6 +33,8 @@ import { IWidgetApiRequest } from "matrix-widget-api";
 import { MatrixRTCSession, MatrixRTCSessionEvent } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
 // eslint-disable-next-line no-restricted-imports
 import { MatrixRTCSessionManagerEvents } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSessionManager";
+// eslint-disable-next-line no-restricted-imports
+import { ICallNotifyContent } from "matrix-js-sdk/src/matrixrtc/types";

 import type EventEmitter from "events";
 import type { ClientWidgetApi } from "matrix-widget-api";
@@ -51,6 +53,7 @@ import { getCurrentLanguage } from "../languageHandler";
 import { FontWatcher } from "../settings/watchers/FontWatcher";
 import { PosthogAnalytics } from "../PosthogAnalytics";
 import { UPDATE_EVENT } from "../stores/AsyncStore";
+import { getFunctionalMembers } from "../utils/room/getFunctionalMembers";

 const TIMEOUT_MS = 16000;

@@ -758,10 +761,30 @@ export class ElementCall extends Call {
            SettingsStore.getValue("feature_video_rooms") &&
            SettingsStore.getValue("feature_element_call_video_rooms") &&
            room.isCallRoom();
-
-        console.log("Intend is ", isVideoRoom ? "VideoRoom" : "Prompt", " TODO, handle intent appropriately");
        ElementCall.createOrGetCallWidget(room.roomId, room.client);
        WidgetStore.instance.emit(UPDATE_EVENT, null);
+
+        // Send Call notify
+
+        const existingRoomCallMembers = MatrixRTCSession.callMembershipsForRoom(room).filter(
+            // filter all memberships where the application is m.call and the call_id is ""
+            (m) => m.application === "m.call" && m.callId === "",
+        );
+
+        // We only want to ring in rooms that have less or equal to NOTIFY_MEMBER_LIMIT participants. For really large rooms we don't want to ring.
+        const NOTIFY_MEMBER_LIMIT = 15;
+        const memberCount = getFunctionalMembers(room).length;
+        if (!isVideoRoom && existingRoomCallMembers.length == 0 && memberCount <= NOTIFY_MEMBER_LIMIT) {
+            // send ringing event
+            const content: ICallNotifyContent = {
+                "application": "m.call",
+                "m.mentions": { user_ids: [], room: true },
+                "notify_type": memberCount == 2 ? "ring" : "notify",
+                "call_id": "",
+            };
+
+            await room.client.sendEvent(room.roomId, EventType.CallNotify, content);
+        }
    }

    protected async performConnection(
@@ -14,8 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-import React, { useCallback, useEffect } from "react";
-import { MatrixEvent, MatrixEventEvent } from "matrix-js-sdk/src/matrix";
+import React, { useCallback, useEffect, useMemo } from "react";
+import { MatrixEvent } from "matrix-js-sdk/src/matrix";
+// eslint-disable-next-line no-restricted-imports
+import { MatrixRTCSessionManagerEvents } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSessionManager";
+// eslint-disable-next-line no-restricted-imports
+import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";

 import { _t } from "../languageHandler";
 import RoomAvatar from "../components/views/avatars/RoomAvatar";
@@ -31,14 +35,15 @@ import {
    LiveContentType,
 } from "../components/views/rooms/LiveContentSummary";
 import { useCall, useJoinCallButtonDisabledTooltip } from "../hooks/useCall";
-import { useRoomState } from "../hooks/useRoomState";
 import { ButtonEvent } from "../components/views/elements/AccessibleButton";
 import { useDispatcher } from "../hooks/useDispatcher";
 import { ActionPayload } from "../dispatcher/payloads";
 import { Call } from "../models/Call";
+import { AudioID } from "../LegacyCallHandler";
 import { useTypedEventEmitter } from "../hooks/useEventEmitter";

-export const getIncomingCallToastKey = (stateKey: string): string => `call_${stateKey}`;
+export const getIncomingCallToastKey = (callId: string, roomId: string): string => `call_${callId}_${roomId}`;
+const MAX_RING_TIME_MS = 10 * 1000;

 interface JoinCallButtonWithCallProps {
    onClick: (e: ButtonEvent) => void;
@@ -62,36 +67,48 @@ function JoinCallButtonWithCall({ onClick, call }: JoinCallButtonWithCallProps):
 }

 interface Props {
-    callEvent: MatrixEvent;
+    notifyEvent: MatrixEvent;
 }

-export function IncomingCallToast({ callEvent }: Props): JSX.Element {
-    const roomId = callEvent.getRoomId()!;
+export function IncomingCallToast({ notifyEvent }: Props): JSX.Element {
+    const roomId = notifyEvent.getRoomId()!;
    const room = MatrixClientPeg.safeGet().getRoom(roomId) ?? undefined;
    const call = useCall(roomId);
+    const audio = useMemo(() => document.getElementById(AudioID.Ring) as HTMLMediaElement, []);

+    // Start ringing if not already.
+    useEffect(() => {
+        const isRingToast = (notifyEvent.getContent() as unknown as { notify_type: string })["notify_type"] == "ring";
+        if (isRingToast && audio.paused) {
+            audio.play();
+        }
+    }, [audio, notifyEvent]);
+
+    // Stop ringing on dismiss.
    const dismissToast = useCallback((): void => {
-        ToastStore.sharedInstance().dismissToast(getIncomingCallToastKey(callEvent.getStateKey()!));
-    }, [callEvent]);
+        ToastStore.sharedInstance().dismissToast(
+            getIncomingCallToastKey(notifyEvent.getContent().call_id ?? "", roomId),
+        );
+        audio.pause();
+    }, [audio, notifyEvent, roomId]);

-    const latestEvent = useRoomState(
-        room,
-        useCallback(
-            (state) => {
-                return state.getStateEvents(callEvent.getType(), callEvent.getStateKey()!);
-            },
-            [callEvent],
-        ),
+    // Dismiss if session got ended remotely.
+    const onSessionEnded = useCallback(
+        (endedSessionRoomId: string, session: MatrixRTCSession): void => {
+            if (roomId == endedSessionRoomId && session.callId == notifyEvent.getContent().call_id) {
+                dismissToast();
+            }
+        },
+        [dismissToast, notifyEvent, roomId],
    );

+    // Dismiss on timeout.
    useEffect(() => {
-        if ("m.terminated" in latestEvent.getContent()) {
-            dismissToast();
-        }
-    }, [latestEvent, dismissToast]);
-
-    useTypedEventEmitter(latestEvent, MatrixEventEvent.BeforeRedaction, dismissToast);
+        const timeout = setTimeout(dismissToast, MAX_RING_TIME_MS);
+        return () => clearTimeout(timeout);
+    });

+    // Dismiss on viewing call.
    useDispatcher(
        defaultDispatcher,
        useCallback(
@@ -104,21 +121,23 @@ export function IncomingCallToast({ callEvent }: Props): JSX.Element {
        ),
    );

+    // Dismiss on clicking join.
    const onJoinClick = useCallback(
        (e: ButtonEvent): void => {
            e.stopPropagation();

+            // The toast will be automatically dismissed by the dispatcher callback above
            defaultDispatcher.dispatch<ViewRoomPayload>({
                action: Action.ViewRoom,
                room_id: room?.roomId,
                view_call: true,
                metricsTrigger: undefined,
            });
-            dismissToast();
        },
-        [room, dismissToast],
+        [room],
    );

+    // Dismiss on closing toast.
    const onCloseClick = useCallback(
        (e: ButtonEvent): void => {
            e.stopPropagation();
@@ -128,9 +147,17 @@ export function IncomingCallToast({ callEvent }: Props): JSX.Element {
        [dismissToast],
    );

+    useTypedEventEmitter(
+        MatrixClientPeg.safeGet().matrixRTC,
+        MatrixRTCSessionManagerEvents.SessionEnded,
+        onSessionEnded,
+    );
+
    return (
        <React.Fragment>
-            <RoomAvatar room={room ?? undefined} size="24px" />
+            <div>
+                <RoomAvatar room={room ?? undefined} size="24px" />
+            </div>
            <div className="mx_IncomingCallToast_content">
                <div className="mx_IncomingCallToast_info">
                    <span className="mx_IncomingCallToast_room">