feat(audio): opt-in AI noise suppression against keyboard noise
The WebRTC noise suppression estimates a running noise profile, so it removes stationary noise but not keystrokes, which are transient and never look like noise to it. Chiclet keyboards are as affected as mechanical ones. DeepFilterNet3 runs client-side as a LiveKit track processor and removes them. Off by default and the model is only fetched once a user switches it on, so nobody pays the download who does not want the filter. Default strength is 35 percent rather than full: measured, that already gives keystrokes gone with the voice still natural, and more attenuation only adds artefact risk. Assets ship with us instead of the package's default CDN, which would report every participant's IP to a third party at call start and tie call setup to foreign infrastructure. The Dockerfile gzips the model wasm, which the existing top-level glob missed — 4.1 MB instead of 15.7 MB per client. Browser noise suppression is switched off while the filter runs so the two do not work against each other. Regulation goes through the model's own attenuation limit, so there is no dry/wet mixer and no delay compensation to get wrong. Decision and measurements: management ADR-0018 and issue #0054. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Thore Cimbal
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
c63be9ab94
commit
3f17001720
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
Copyright 2026 aXion1337.chat
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
|
||||
ThreadNet-Fork-Anpassung (ADR-0018) - nicht Upstream. Siehe docs/axion1337-fork.md.
|
||||
*/
|
||||
|
||||
import {
|
||||
type AudioProcessorOptions,
|
||||
type Track,
|
||||
type TrackProcessor,
|
||||
} from "livekit-client";
|
||||
import { DeepFilterNoiseFilterProcessor } from "deepfilternet3-noise-filter";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import {
|
||||
aiNoiseSuppressionLevelSetting,
|
||||
aiNoiseSuppressionSetting,
|
||||
} from "../settings/settings";
|
||||
|
||||
/**
|
||||
* KI-Geraeuschunterdrueckung (DeepFilterNet3) als LiveKit-TrackProcessor.
|
||||
*
|
||||
* Warum ueberhaupt: Der WebRTC-Standardfilter schaetzt ein laufendes Rauschprofil
|
||||
* und filtert damit STATIONAERE Stoerungen (Luefter, Brummen). Tastaturanschlaege
|
||||
* sind TRANSIENT - sehr kurzer Anstieg, unvorhersehbares Spektrum - und werden
|
||||
* nicht als Stoerung erkannt. Genau die filtert DeepFilterNet3 weg.
|
||||
*
|
||||
* Warum die Assets von uns kommen: Das Paket laedt Modell und wasm sonst von
|
||||
* cdn.mezon.ai. Fuer eine selbstgehostete Plattform hiesse das, dass jeder
|
||||
* Teilnehmer bei jedem Call-Start seine IP an einen Dritten meldet und die
|
||||
* Verfuegbarkeit an fremder Infrastruktur haengt. `assetConfig.cdnUrl` zeigt
|
||||
* deshalb auf unsere eigene Auslieferung (public/assets/dfn3/).
|
||||
*/
|
||||
const ASSET_PFAD = "assets/dfn3";
|
||||
|
||||
/**
|
||||
* Baut den Prozessor - oder gibt `undefined` zurueck, wenn der Nutzer den Filter
|
||||
* nicht eingeschaltet hat.
|
||||
*
|
||||
* Der Import ist statisch, kostet aber nur den ~23-KB-Wrapper. Die ~23 MB
|
||||
* Modell-Assets holt das Paket erst in seinem `init()`, also erst wenn der
|
||||
* Prozessor wirklich an einen Track gehaengt wird. Damit bleibt das Opt-in aus
|
||||
* ADR-0018 auch wirtschaftlich eines: wer den Filter aus laesst, laedt nichts.
|
||||
*/
|
||||
export function createAiNoiseSuppressionProcessor():
|
||||
| TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>
|
||||
| undefined {
|
||||
if (!aiNoiseSuppressionSetting.getValue()) return undefined;
|
||||
|
||||
try {
|
||||
return new DeepFilterNoiseFilterProcessor({
|
||||
sampleRate: 48000,
|
||||
noiseReductionLevel: aiNoiseSuppressionLevelSetting.getValue(),
|
||||
assetConfig: {
|
||||
cdnUrl: new URL(ASSET_PFAD, window.location.href).href,
|
||||
},
|
||||
}) as unknown as TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>;
|
||||
} catch (e) {
|
||||
// Bewusst kein Abbruch: lieber ein Call ohne Filter als kein Call.
|
||||
logger.error("KI-Geraeuschunterdrueckung nicht verfuegbar", e);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,8 @@ import { useMediaDevices } from "../MediaDevicesContext";
|
||||
import { widget } from "../widget";
|
||||
import {
|
||||
useSetting,
|
||||
aiNoiseSuppressionSetting,
|
||||
aiNoiseSuppressionLevelSetting,
|
||||
soundEffectVolume as soundEffectVolumeSetting,
|
||||
backgroundBlur as backgroundBlurSetting,
|
||||
developerMode,
|
||||
@@ -109,6 +111,68 @@ export const SettingsModal: FC<Props> = ({
|
||||
);
|
||||
};
|
||||
|
||||
// ThreadNet-Fork (ADR-0018): KI-Geraeuschunterdrueckung gegen Tastaturgeraeusche.
|
||||
// Bewusst OPT-IN - die Modell-Assets sind ~23 MB und werden erst geladen, wenn
|
||||
// der Filter eingeschaltet ist. Standard-Daempfung 35 %: gemessen reicht das
|
||||
// fuer "Tastatur weg und Stimme natuerlich", mehr erhoeht nur das
|
||||
// Artefaktrisiko (#0054).
|
||||
const AiNoiseSuppressionSettings: React.FC = (): ReactNode => {
|
||||
const [aiActive, setAiActive] = useSetting(aiNoiseSuppressionSetting);
|
||||
const [level, setLevel] = useSetting(aiNoiseSuppressionLevelSetting);
|
||||
const [levelRaw, setLevelRaw] = useState(level);
|
||||
|
||||
return (
|
||||
<>
|
||||
<h4>
|
||||
{t("settings.ai_noise_suppression_header", "AI noise suppression")}
|
||||
</h4>
|
||||
<FieldRow>
|
||||
<InputField
|
||||
id="activateAiNoiseSuppression"
|
||||
label={t(
|
||||
"settings.ai_noise_suppression_label",
|
||||
"Filter keyboard and background noise",
|
||||
)}
|
||||
description={t(
|
||||
"settings.ai_noise_suppression_description",
|
||||
"Downloads a ~23 MB model the first time it is switched on. Applies on the next call join.",
|
||||
)}
|
||||
type="checkbox"
|
||||
checked={aiActive}
|
||||
onChange={(e): void => setAiActive(e.target.checked)}
|
||||
/>
|
||||
</FieldRow>
|
||||
{aiActive && (
|
||||
<div className={styles.volumeSlider}>
|
||||
<label>
|
||||
{t("settings.ai_noise_suppression_strength_label", "Strength")}
|
||||
{": "}
|
||||
<span className={styles.settingValue}>{levelRaw}%</span>
|
||||
</label>
|
||||
<p>
|
||||
{t(
|
||||
"settings.ai_noise_suppression_strength_description",
|
||||
"Lower keeps the room sound natural, higher isolates the voice more strictly.",
|
||||
)}
|
||||
</p>
|
||||
<Slider
|
||||
label={t(
|
||||
"settings.ai_noise_suppression_strength_label",
|
||||
"Strength",
|
||||
)}
|
||||
value={levelRaw}
|
||||
onValueChange={setLevelRaw}
|
||||
onValueCommit={setLevel}
|
||||
min={0}
|
||||
max={100}
|
||||
step={5}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
const devices = useMediaDevices();
|
||||
useEffect(() => {
|
||||
if (open) devices.requestDeviceNames(); // No-op after the first call
|
||||
@@ -181,6 +245,8 @@ export const SettingsModal: FC<Props> = ({
|
||||
step={0.01}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<AiNoiseSuppressionSettings />
|
||||
</Form>
|
||||
</>
|
||||
),
|
||||
|
||||
@@ -226,6 +226,23 @@ export const autoGainControlSetting = new Setting<boolean>(
|
||||
true,
|
||||
);
|
||||
|
||||
// KI-Geraeuschunterdrueckung (ThreadNet-Fork, ADR-0018).
|
||||
// Bewusst OPT-IN: die Modell-Assets sind ~23 MB und werden erst geladen, wenn
|
||||
// der Prozessor tatsaechlich an den Track gehaengt wird. Wer den Filter nicht
|
||||
// einschaltet, laedt nichts.
|
||||
export const aiNoiseSuppressionSetting = new Setting<boolean>(
|
||||
"ai-noise-suppression",
|
||||
false,
|
||||
);
|
||||
|
||||
// Daempfung in Prozent. 35 statt 100 ist gemessen, nicht geraten: bei ~35 %
|
||||
// waren Tastaturanschlaege weg UND die Stimme klang natuerlich (#0054).
|
||||
// Mehr Daempfung heisst mehr Artefaktrisiko, nicht mehr Nutzen.
|
||||
export const aiNoiseSuppressionLevelSetting = new Setting<number>(
|
||||
"ai-noise-suppression-level",
|
||||
35,
|
||||
);
|
||||
|
||||
/**
|
||||
* Seed setting defaults from config.json's media_quality section.
|
||||
* Call this after Config.init() has resolved.
|
||||
|
||||
@@ -38,7 +38,9 @@ import {
|
||||
echoCancellationSetting,
|
||||
noiseSuppressionSetting,
|
||||
autoGainControlSetting,
|
||||
aiNoiseSuppressionSetting,
|
||||
} from "../../../settings/settings.ts";
|
||||
import { createAiNoiseSuppressionProcessor } from "../../../livekit/aiNoiseSuppression.ts";
|
||||
|
||||
// TODO evaluate if this should be done like the Publisher Factory
|
||||
export interface ConnectionFactory {
|
||||
@@ -175,8 +177,16 @@ function generateRoomOption({
|
||||
...liveKitOptions.audioCaptureDefaults,
|
||||
deviceId: devices.audioInput.selected$.value?.id,
|
||||
echoCancellation: echoCancellationSetting.getValue(),
|
||||
noiseSuppression: noiseSuppressionSetting.getValue(),
|
||||
// Bei aktiver KI-Filterung die Browser-Rauschunterdrueckung ausschalten:
|
||||
// sonst arbeiten zwei Filter gegeneinander und der Browser schneidet dem
|
||||
// Modell bereits Signalanteile weg (ADR-0018).
|
||||
noiseSuppression:
|
||||
noiseSuppressionSetting.getValue() &&
|
||||
!aiNoiseSuppressionSetting.getValue(),
|
||||
autoGainControl: autoGainControlSetting.getValue(),
|
||||
// ThreadNet-Fork: KI-Geraeuschunterdrueckung gegen Tastaturgeraeusche.
|
||||
// `undefined`, wenn nicht eingeschaltet - dann wird auch nichts geladen.
|
||||
processor: createAiNoiseSuppressionProcessor(),
|
||||
},
|
||||
audioOutput: {
|
||||
// When using controlled audio devices, we don't want to set the
|
||||
|
||||
Reference in New Issue
Block a user