/*
Copyright 2024 New Vector Ltd.
Copyright 2022 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE files in the repository root for full details.
*/
import { mocked } from "jest-mock";
import { logger } from "matrix-js-sdk/src/logger";
import { createAudioContext, decodeOgg } from "../../../src/audio/compat";
import { Playback, PlaybackState } from "../../../src/audio/Playback";
jest.mock("../../../src/WorkerManager", () => ({
WorkerManager: jest.fn(() => ({
call: jest.fn().mockResolvedValue({ waveform: [0, 0, 1, 1] }),
})),
}));
jest.mock("../../../src/audio/compat", () => ({
createAudioContext: jest.fn(),
decodeOgg: jest.fn(),
}));
describe("Playback", () => {
const mockAudioBufferSourceNode = {
addEventListener: jest.fn(),
connect: jest.fn(),
start: jest.fn(),
};
const mockMediaElementSourceNode = {
addEventListener: jest.fn(),
removeEventListener: jest.fn(),
connect: jest.fn(),
disconnect: jest.fn(),
};
const mockAudioContext = {
decodeAudioData: jest.fn(),
suspend: jest.fn(),
resume: jest.fn(),
createBufferSource: jest.fn().mockReturnValue(mockAudioBufferSourceNode),
createMediaElementSource: jest.fn().mockReturnValue(mockMediaElementSourceNode),
currentTime: 1337,
};
const mockAudioBuffer = {
duration: 99,
getChannelData: jest.fn(),
};
const mockChannelData = new Float32Array();
beforeEach(() => {
jest.spyOn(logger, "error").mockRestore();
mockAudioBufferSourceNode.addEventListener.mockClear();
mockAudioBuffer.getChannelData.mockClear().mockReturnValue(mockChannelData);
mockAudioContext.decodeAudioData.mockReset().mockResolvedValue(mockAudioBuffer);
mockAudioContext.resume.mockClear().mockResolvedValue(undefined);
mockAudioContext.suspend.mockClear().mockResolvedValue(undefined);
mocked(decodeOgg).mockClear().mockResolvedValue(new ArrayBuffer(1));
mocked(createAudioContext).mockReturnValue(mockAudioContext as unknown as AudioContext);
});
it("initialises correctly", () => {
const buffer = new ArrayBuffer(8);
const playback = new Playback(buffer);
playback.clockInfo.durationSeconds = mockAudioBuffer.duration;
expect(playback.sizeBytes).toEqual(8);
expect(playback.clockInfo).toBeTruthy();
expect(playback.liveData).toBe(playback.clockInfo.liveData);
expect(playback.timeSeconds).toBe(1337 % 99);
expect(playback.currentState).toEqual(PlaybackState.Decoding);
});
it("toggles playback on from stopped state", async () => {
const buffer = new ArrayBuffer(8);
const playback = new Playback(buffer);
await playback.prepare();
// state is Stopped
await playback.toggle();
expect(mockAudioBufferSourceNode.start).toHaveBeenCalled();
expect(mockAudioContext.resume).toHaveBeenCalled();
expect(playback.currentState).toEqual(PlaybackState.Playing);
});
it("toggles playback to paused from playing state", async () => {
const buffer = new ArrayBuffer(8);
const playback = new Playback(buffer);
await playback.prepare();
await playback.toggle();
expect(playback.currentState).toEqual(PlaybackState.Playing);
await playback.toggle();
expect(mockAudioContext.suspend).toHaveBeenCalled();
expect(playback.currentState).toEqual(PlaybackState.Paused);
});
it("stop playbacks", async () => {
const buffer = new ArrayBuffer(8);
const playback = new Playback(buffer);
await playback.prepare();
await playback.toggle();
expect(playback.currentState).toEqual(PlaybackState.Playing);
await playback.stop();
expect(mockAudioContext.suspend).toHaveBeenCalled();
expect(playback.currentState).toEqual(PlaybackState.Stopped);
});
it("stop when audio source ended", async () => {
const buffer = new ArrayBuffer(8);
const playback = new Playback(buffer);
await playback.prepare();
await playback.play();
// Simulate the audio source ending by calling the 'ended' event listener
const endedListener = mockAudioBufferSourceNode.addEventListener.mock.calls.find(
(call) => call[0] === "ended",
)[1];
await endedListener();
// AudioContext should be suspended
expect(mockAudioContext.suspend).toHaveBeenCalled();
// Playback state should be Stopped
expect(playback.currentState).toEqual(PlaybackState.Stopped);
// Clock should be reset to 0
expect(playback.timeSeconds).toEqual(0);
});
describe("prepare()", () => {
it("decodes audio data when not greater than 5mb", async () => {
const buffer = new ArrayBuffer(8);
const playback = new Playback(buffer);
await playback.prepare();
expect(mockAudioContext.decodeAudioData).toHaveBeenCalledTimes(1);
expect(mockAudioBuffer.getChannelData).toHaveBeenCalledWith(0);
// clock was updated
expect(playback.clockInfo.durationSeconds).toEqual(mockAudioBuffer.duration);
expect(playback.durationSeconds).toEqual(mockAudioBuffer.duration);
expect(playback.currentState).toEqual(PlaybackState.Stopped);
});
it("tries to decode ogg when decodeAudioData fails", async () => {
// stub logger to keep console clean from expected error
jest.spyOn(logger, "error").mockReturnValue(undefined);
jest.spyOn(logger, "warn").mockReturnValue(undefined);
const buffer = new ArrayBuffer(8);
const decodingError = new Error("test");
mockAudioContext.decodeAudioData
.mockRejectedValueOnce(decodingError)
.mockResolvedValueOnce(mockAudioBuffer);
const playback = new Playback(buffer);
await playback.prepare();
expect(mockAudioContext.decodeAudioData).toHaveBeenCalledTimes(2);
expect(decodeOgg).toHaveBeenCalled();
// clock was updated
expect(playback.clockInfo.durationSeconds).toEqual(mockAudioBuffer.duration);
expect(playback.durationSeconds).toEqual(mockAudioBuffer.duration);
expect(playback.currentState).toEqual(PlaybackState.Stopped);
});
it("hands the ogg fallback a buffer which decodeAudioData has not detached", async () => {
// stub logger to keep console clean from expected error
jest.spyOn(logger, "error").mockReturnValue(undefined);
jest.spyOn(logger, "warn").mockReturnValue(undefined);
const buffer = new ArrayBuffer(8);
mockAudioContext.decodeAudioData
.mockImplementationOnce((buf: ArrayBuffer) => {
// The real decodeAudioData detaches the buffer it is handed, even when it fails.
structuredClone(buf, { transfer: [buf] });
return Promise.reject(new Error("test"));
})
.mockResolvedValueOnce(mockAudioBuffer);
// Constructing a view over a detached buffer throws, which is what decodeOgg does first.
mocked(decodeOgg).mockImplementationOnce(async (audioBuffer: ArrayBuffer) => {
expect(() => new Uint8Array(audioBuffer)).not.toThrow();
return new ArrayBuffer(1);
});
const playback = new Playback(buffer);
await playback.prepare();
expect(decodeOgg).toHaveBeenCalled();
expect(mockAudioContext.decodeAudioData).toHaveBeenCalledTimes(2);
expect(playback.currentState).toEqual(PlaybackState.Stopped);
});
it("does not try to re-decode audio", async () => {
const buffer = new ArrayBuffer(8);
const playback = new Playback(buffer);
await playback.prepare();
expect(playback.currentState).toEqual(PlaybackState.Stopped);
await playback.prepare();
// only called once in first prepare
expect(mockAudioContext.decodeAudioData).toHaveBeenCalledTimes(1);
});
});
describe("audio larger than 5mb", () => {
// Anything over 5mb is played through an element rather than a decoded buffer
const largeBuffer = (): ArrayBuffer => new ArrayBuffer(5 * 1024 * 1024 + 1);
let element: ReturnType;
/**
* A stand-in for the element Playback creates. Assigning `src` resolves the load
* the same way the browser does, and listeners registered on it can be fired by hand.
*/
const mockAudioElement = () => {
const listeners = new Map void | Promise>>();
const el = {
duration: 42,
currentTime: 0,
onloadeddata: undefined as undefined | (() => void),
onerror: undefined as undefined | (() => void),
play: jest.fn().mockResolvedValue(undefined),
pause: jest.fn(),
remove: jest.fn(),
addEventListener: jest.fn((type: string, cb: () => void) => {
if (!listeners.has(type)) listeners.set(type, new Set());
listeners.get(type)!.add(cb);
}),
removeEventListener: jest.fn((type: string, cb: () => void) => {
listeners.get(type)?.delete(cb);
}),
listenerCount: (type: string): number => listeners.get(type)?.size ?? 0,
fire: async (type: string): Promise => {
for (const cb of listeners.get(type) ?? []) await cb();
},
};
Object.defineProperty(el, "src", {
set() {
el.onloadeddata?.();
},
});
return el;
};
beforeEach(() => {
element = mockAudioElement();
jest.spyOn(document, "createElement").mockReturnValue(element as unknown as HTMLElement);
global.URL.createObjectURL = jest.fn().mockReturnValue("blob:audio");
global.URL.revokeObjectURL = jest.fn();
mockAudioContext.createMediaElementSource.mockClear().mockReturnValue(mockMediaElementSourceNode);
mockMediaElementSourceNode.connect.mockClear();
});
afterEach(() => {
mocked(document.createElement).mockRestore();
});
it("stops when the media element ends", async () => {
const playback = new Playback(largeBuffer());
await playback.prepare();
await playback.play();
expect(playback.currentState).toEqual(PlaybackState.Playing);
await element.fire("ended");
expect(mockAudioContext.suspend).toHaveBeenCalled();
expect(playback.currentState).toEqual(PlaybackState.Stopped);
expect(playback.timeSeconds).toEqual(0);
});
it("stops listening to the media element once destroyed", async () => {
const playback = new Playback(largeBuffer());
await playback.prepare();
await playback.play();
expect(element.listenerCount("ended")).toEqual(1);
playback.destroy();
expect(element.listenerCount("ended")).toEqual(0);
});
});
});