From fee9866e1947db027cc589fbe307df349726b474 Mon Sep 17 00:00:00 2001 From: Thore Cimbal Date: Mon, 17 Aug 2026 12:00:00 +0000 Subject: [PATCH] fix(audio): verify the sender actually carries the processed track Field finding on Safari as the sending client: assets load, setProcessor reports success, yet suppression level 0-100 makes no audible difference - because LiveKit swaps the sender track via 'this.sender?.replaceTrack(...)', and when the sender is not there at that instant the swap is skipped silently, leaving the raw microphone on the wire. Chromium clients hit the timing, Safari does not. applyAiNoiseSuppression now verifies instead of trusting: wait for the sender if needed, enforce the swap explicitly, and state the outcome in the log line ('Sendepfad gefiltert: ja/NEIN'). Three tests pin the sender cases, including the exact Safari symptom of a sender still carrying the raw track. Co-Authored-By: Claude Opus 4.8 --- embedded/web/package.json | 2 +- src/livekit/aiNoiseSuppression.test.ts | 56 +++++++++++++++++++++++++- src/livekit/aiNoiseSuppression.ts | 39 +++++++++++++++++- 3 files changed, 94 insertions(+), 3 deletions(-) diff --git a/embedded/web/package.json b/embedded/web/package.json index c34219b6..2491d48f 100644 --- a/embedded/web/package.json +++ b/embedded/web/package.json @@ -1,6 +1,6 @@ { "name": "@sorb/threadnet-call-embedded", - "version": "0.19.2-threadnet.11", + "version": "0.19.2-threadnet.12", "files": [ "README.md", "LICENSE-AGPL-3.0", diff --git a/src/livekit/aiNoiseSuppression.test.ts b/src/livekit/aiNoiseSuppression.test.ts index e41e5ba4..2825bd9d 100644 --- a/src/livekit/aiNoiseSuppression.test.ts +++ b/src/livekit/aiNoiseSuppression.test.ts @@ -11,7 +11,10 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { type LocalAudioTrack } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; -import { applyAiNoiseSuppression } from "./aiNoiseSuppression"; +import { + applyAiNoiseSuppression, + ensureSenderCarriesProcessed, +} from "./aiNoiseSuppression"; import { aiNoiseSuppressionDevSetting, aiNoiseSuppressionSetting, @@ -99,3 +102,54 @@ describe("applyAiNoiseSuppression (Weg B, #0054)", () => { await expect(applyAiNoiseSuppression(track, logger)).resolves.toBeUndefined(); }); }); + +describe("ensureSenderCarriesProcessed (Safari-Sendepfad, #0054)", () => { + const processed = { id: "gefiltert" } as unknown as MediaStreamTrack; + + const trackMitSender = ( + sender: { track: MediaStreamTrack | null; replaceTrack: ReturnType } | undefined, + ): never => ({ sender }) as never; + + test("sender already carries the processed track: nothing to do", async () => { + const replaceTrack = vi.fn(); + const ok = await ensureSenderCarriesProcessed( + trackMitSender({ track: processed, replaceTrack }), + { processedTrack: processed }, + 2, + 1, + ); + expect(ok).toBe(true); + expect(replaceTrack).not.toHaveBeenCalled(); + }); + + test("sender still carries the RAW track: swap is enforced", async () => { + // Genau der Safari-Befund: setProcessor meldete Erfolg, der Sender sendet + // trotzdem das rohe Mikro - Staerke 0-100 ohne hoerbare Wirkung. + const roh = { id: "roh" } as unknown as MediaStreamTrack; + const sender = { + track: roh as MediaStreamTrack | null, + replaceTrack: vi.fn(async (t: MediaStreamTrack) => { + sender.track = t; + await Promise.resolve(); + }), + }; + const ok = await ensureSenderCarriesProcessed( + trackMitSender(sender), + { processedTrack: processed }, + 2, + 1, + ); + expect(ok).toBe(true); + expect(sender.replaceTrack).toHaveBeenCalledWith(processed); + }); + + test("sender never appears: reports false instead of pretending", async () => { + const ok = await ensureSenderCarriesProcessed( + trackMitSender(undefined), + { processedTrack: processed }, + 3, + 1, + ); + expect(ok).toBe(false); + }); +}); diff --git a/src/livekit/aiNoiseSuppression.ts b/src/livekit/aiNoiseSuppression.ts index 32888417..dfe990bd 100644 --- a/src/livekit/aiNoiseSuppression.ts +++ b/src/livekit/aiNoiseSuppression.ts @@ -142,11 +142,48 @@ export async function applyAiNoiseSuppression( } track.setAudioContext(sharedAudioContext); await track.setProcessor(processor); + // Verifizieren statt vertrauen: LiveKits setProcessor tauscht den + // Sender-Track per `this.sender?.replaceTrack(...)` - ist der Sender in + // dem Moment nicht da (Safari-Timing), wird der Tausch STUMM uebersprungen + // und das rohe Mikro bleibt auf der Leitung. Symptom: Filter "an", Assets + // geladen, aber Staerke 0-100 ohne jede hoerbare Wirkung. + const angekommen = await ensureSenderCarriesProcessed(track, processor); parentLogger.info( - `KI-Geraeuschunterdrueckung aktiv (Staerke ${aiNoiseSuppressionLevelSetting.getValue()} %)`, + `KI-Geraeuschunterdrueckung aktiv (Staerke ${aiNoiseSuppressionLevelSetting.getValue()} %), ` + + `Sendepfad gefiltert: ${angekommen ? "ja" : "NEIN - Sender nicht gefunden"}`, ); } catch (e) { // Track ist publiziert, der Call laeuft - der Filter faellt aus, mehr nicht. parentLogger.error("KI-Geraeuschunterdrueckung konnte nicht angehaengt werden", e); } } + +/** + * Stellt sicher, dass der RTCRtpSender wirklich den GEFILTERTEN Track sendet. + * Wartet notfalls auf den Sender (er kann beim LocalTrackPublished-Event je + * nach Engine noch fehlen) und zieht den Tausch explizit nach. + * + * Gibt `true` zurueck, wenn der Sender nachweislich den gefilterten Track + * traegt - das ist die Aussage, die in der Log-Zeile oben landet. + */ +export async function ensureSenderCarriesProcessed( + track: LocalAudioTrack, + processor: { processedTrack?: MediaStreamTrack }, + versuche = 20, + intervallMs = 250, +): Promise { + const processed = processor.processedTrack; + if (!processed) return false; + for (let i = 0; i < versuche; i++) { + // `sender` ist in LiveKit als @internal markiert, aber oeffentlich lesbar - + // es ist die einzige Stelle, an der sich die Wahrheit pruefen laesst. + const sender = (track as unknown as { sender?: RTCRtpSender }).sender; + if (sender) { + if (sender.track === processed) return true; + await sender.replaceTrack(processed); + return sender.track === processed; + } + await new Promise((r) => setTimeout(r, intervallMs)); + } + return false; +}