From 75507dc723dd4f251d043b4f22a7b6bff0f9f687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20K=C3=A4mmerer?= Date: Fri, 25 Sep 2026 00:08:17 +0200 Subject: [PATCH] Let Whisper listen where the diarization found speakers instead of where Silero heard voice activity, so a voice far quieter than the one beside it is transcribed instead of cut away before Whisper hears it With speaker detection, the diarization's turns replace Silero's speech map for faster-whisper: only the one call that returns the map is swapped, so the chunks are collected and the timestamps mapped back exactly as before. Silero kept 27-41 % of a voice 15-20 dB below its neighbour, and Whisper wrote almost none of what it dropped. On all 23 AMI meetings the diarization's map makes 1.39 points fewer errors per reference word (95 % [+0.93, +1.92]), with more words found in every meeting, and on four hand-checked German passages WER falls from 9.28 % to 8.75 %. Without speaker detection nothing changes. Co-Authored-By: Claude Opus 5.5 --- noScribe/main.py | 12 +- noScribe/whisper_mp_worker.py | 97 +++++++++++--- tests/test_empty_transcript.py | 8 +- tests/test_transcript_media_timeline.py | 2 +- tests/test_whisper_speech_map.py | 171 ++++++++++++++++++++++++ 5 files changed, 267 insertions(+), 23 deletions(-) create mode 100644 tests/test_whisper_speech_map.py diff --git a/noScribe/main.py b/noScribe/main.py index 438c2dd7..d42cbd79 100644 --- a/noScribe/main.py +++ b/noScribe/main.py @@ -2841,6 +2841,7 @@ def find_speaker(diarization, transcript_start, transcript_end) -> str: # Start Diarization: + diarization = None if job.speaker_detection != 'none': try: job.status = JobStatus.SPEAKER_IDENTIFICATION @@ -3209,7 +3210,7 @@ def __init__(self, d): pass try: - info = self._run_whisper_subprocess_stream(tmp_audio_file, job, on_segment) + info = self._run_whisper_subprocess_stream(tmp_audio_file, job, on_segment, diarization) if first_segment: raise ValueError(t('err_empty_transcript')) transcription_success = True @@ -3357,10 +3358,12 @@ def _handle_cuda_fallback(self, component: str, error: Exception) -> bool: return False - def _run_whisper_subprocess_stream(self, tmp_audio_file: str, job, on_segment): + def _run_whisper_subprocess_stream(self, tmp_audio_file: str, job, on_segment, diarization=None): """Spawn a subprocess to run Faster-Whisper and stream segments. Calls on_segment(dict) for each segment streamed by the child. - Returns a simple info object (duration at least). + Returns a simple info object (duration at least). With a diarization, + its turns are the speech map in place of Silero's + (the measurement is above whisper_mp_worker._speech_map_from). """ global force_whisper_cpu # Language code for non-auto/multilingual @@ -3392,6 +3395,9 @@ def _run_whisper_subprocess_stream(self, tmp_audio_file: str, job, on_segment): "vad_filter": True, "vad_threshold": vad_threshold, "locale": config.get("locale", "en"), + # Same converted-WAV timeline the worker reads, in seconds. + "speech_turns": ([[seg["start"] / 1000.0, seg["end"] / 1000.0] for seg in diarization] + if diarization else None), } # Spawn child process using spawn start method diff --git a/noScribe/whisper_mp_worker.py b/noScribe/whisper_mp_worker.py index df91ba92..f05e11bd 100644 --- a/noScribe/whisper_mp_worker.py +++ b/noScribe/whisper_mp_worker.py @@ -1,3 +1,4 @@ +import contextlib import gc import logging import os @@ -15,6 +16,64 @@ logger = logging.getLogger(__name__) +# Gaps closed and edges padded by the same amounts as the Silero options in +# whisper_proc_entrypoint, and kept equal on purpose -- but measured for this map on +# its own: closing gaps up to 2 s instead let one phone call become a single 31-s +# chunk that Whisper decoded into an invented sentence. +SPEECH_MAP_MIN_GAP_S = 0.5 +SPEECH_MAP_PAD_S = 0.05 + + +def speech_map(turns, n_samples, sampling_rate=16000): + """faster-whisper's speech chunks ({'start', 'end'} in samples) from + [start_s, end_s] turns: padded by SPEECH_MAP_PAD_S, merged across gaps under + SPEECH_MAP_MIN_GAP_S, clipped to the audio.""" + spans = sorted((max(0, int((s - SPEECH_MAP_PAD_S) * sampling_rate)), + min(n_samples, int((e + SPEECH_MAP_PAD_S) * sampling_rate))) + for s, e in turns if e > s) # an empty turn is no speech, padded or not + out = [] + for s, e in spans: + if s >= e: # a turn that lies past the end of the audio + continue + if out and s - out[-1]['end'] < SPEECH_MAP_MIN_GAP_S * sampling_rate: + out[-1]['end'] = max(out[-1]['end'], e) + else: + out.append({'start': s, 'end': e}) + return out + + +# With speaker detection, the diarization's turns tell Whisper where speech is, +# in place of Silero's voice activity. Silero drops quiet speech before Whisper +# ever hears it: of a voice 15-20 dB below the one beside it, it kept 27-41 % of +# the time, and Whisper wrote 2-3 % of the words in what it dropped. Lowering its +# threshold does not help (0.35 is within noise on AMI, 0.15 lets room tone in). +# The diarization's turns kept 67 % of that voice, and with them as the speech +# map Whisper wrote 50-65 % of its words instead of 27-46 %, with none in pure +# noise. On all 23 AMI meetings, against their manual words, it made 1.39 points +# fewer errors per reference word (95 % [+0.93, +1.92]), with more words found +# in every meeting; on four hand-checked German passages 9.28 % -> 8.75 % WER, +# all four better. Single files vary between runs either way -- faster-whisper +# re-decodes an unsure window by sampling, unseeded, and quiet speech is where it +# is unsure (one recording gave 219 to 292 words) -- so only the totals say anything. +@contextlib.contextmanager +def _speech_map_from(turns): + """While active, faster-whisper takes its speech chunks from `turns` instead + of Silero. It asks for them through the one name it imports into its + transcribe module, and nothing else there changes: the chunks are collected + and the timestamps mapped back exactly as with Silero's. Yields whether the + swap is in place -- not without turns, nor if a faster-whisper release no + longer has that name (tests/test_whisper_speech_map.py pins it).""" + import faster_whisper.transcribe as ft + silero = getattr(ft, "get_speech_timestamps", None) + if not turns or silero is None: + yield False + return + ft.get_speech_timestamps = lambda audio, *a, **k: speech_map(turns, len(audio)) + try: + yield True + finally: + ft.get_speech_timestamps = silero + def whisper_proc_entrypoint(args: dict, q): """ @@ -119,15 +178,18 @@ def log_cb(level, msg): else: whisper_lang = language_code + speech_turns = args.get("speech_turns") + # Detect language if requested (Auto) if language_name == "Auto": audio = decode_audio( audio_path, sampling_rate=model.feature_extractor.sampling_rate ) try: - whisper_lang, language_probability, _ = model.detect_language( - audio, vad_filter=True, vad_parameters=vad_parameters - ) + with _speech_map_from(speech_turns): + whisper_lang, language_probability, _ = model.detect_language( + audio, vad_filter=True, vad_parameters=vad_parameters + ) finally: del audio gc.collect() @@ -148,18 +210,23 @@ def log_cb(level, msg): # Pass the path so faster-whisper can release its original waveform # after VAD, before allocating the full spectrogram (see 28650f2e). - segments, info = model.transcribe( - audio_path, - language=whisper_lang, - multilingual=multilingual, - beam_size=args.get("beam_size", 5), - # temperature=args.get("temperature"), - word_timestamps=args.get("word_timestamps", True), - # initial_prompt=prompt, - hotwords=prompt, - vad_filter=args.get("vad_filter", True), - vad_parameters=vad_parameters, - ) + # transcribe() takes its speech map before it returns the lazy segments, + # so the swap need only last for the call. + with _speech_map_from(speech_turns) as mapped: + segments, info = model.transcribe( + audio_path, + language=whisper_lang, + multilingual=multilingual, + beam_size=args.get("beam_size", 5), + # temperature=args.get("temperature"), + word_timestamps=args.get("word_timestamps", True), + # initial_prompt=prompt, + hotwords=prompt, + vad_filter=args.get("vad_filter", True), + vad_parameters=vad_parameters, + ) + if speech_turns and not mapped: + log_cb("debug", "speech map from the diarization unavailable; Silero decides where speech is") log_cb('info', t('start_transcription') + '\n') diff --git a/tests/test_empty_transcript.py b/tests/test_empty_transcript.py index e7262e88..cafc5bfb 100644 --- a/tests/test_empty_transcript.py +++ b/tests/test_empty_transcript.py @@ -51,7 +51,7 @@ def _segment(text, start=1.0): @pytest.mark.parametrize("detection", ["none", "auto"]) @pytest.mark.parametrize("texts", [[], ["", " \t\n ", "\u2003"]]) def test_empty_result_fails_without_saving(app, tmp_path, monkeypatch, suffix, detection, texts): - def whisper(path, job, on_segment): + def whisper(path, job, on_segment, *speech_map): for text in texts: on_segment(_segment(text)) return {} @@ -82,7 +82,7 @@ def test_batch_continues_after_empty_job(app, tmp_path, monkeypatch): empty = _job(tmp_path, name="empty") valid = _job(tmp_path, name="valid") - def whisper(path, job, on_segment): + def whisper(path, job, on_segment, *speech_map): if job is valid: on_segment(_segment(" Hello")) return {} @@ -100,7 +100,7 @@ def whisper(path, job, on_segment): @pytest.mark.parametrize("suffix", ["html", "txt", "vtt"]) @pytest.mark.parametrize("canceled", [False, True]) def test_failed_or_canceled_job_keeps_real_partial_text(app, tmp_path, monkeypatch, suffix, canceled): - def whisper(path, job, on_segment): + def whisper(path, job, on_segment, *speech_map): on_segment(_segment(" Hello")) on_segment(_segment(" \t ", start=2.0)) if canceled: @@ -138,7 +138,7 @@ def test_segment_logs_metadata_without_text(app, tmp_path, monkeypatch, capsys, monkeypatch.setattr(app, "logn", m.App.logn.__get__(app)) text = " Confidential interview content." - def whisper(path, job, on_segment): + def whisper(path, job, on_segment, *speech_map): on_segment(_segment(text)) on_segment(_segment(" \t ", start=2.0)) return {} diff --git a/tests/test_transcript_media_timeline.py b/tests/test_transcript_media_timeline.py index 78a15197..2808164c 100644 --- a/tests/test_transcript_media_timeline.py +++ b/tests/test_transcript_media_timeline.py @@ -23,7 +23,7 @@ def test_transcript_uses_actual_converted_media_position( ) logs = [] - def stream(path, job, on_segment): + def stream(path, job, on_segment, *speech_map): # Leave a one-second pause before the first phrase to cover pause anchors. on_segment({"start": 1.0, "end": 1.5, "text": " Hello", "words": []}) return {} diff --git a/tests/test_whisper_speech_map.py b/tests/test_whisper_speech_map.py new file mode 100644 index 00000000..8eca88c5 --- /dev/null +++ b/tests/test_whisper_speech_map.py @@ -0,0 +1,171 @@ +"""With speaker detection, Whisper takes its speech map from the diarization. + +Silero dropped a quiet voice before Whisper could hear it; the diarization's +turns keep it (the measurement is above _speech_map_from in +noScribe/whisper_mp_worker.py). The swap works through the one name +faster-whisper imports into its transcribe module, so the guard that matters +most is that it still looks the map up there: a faster-whisper release that +stops doing so would leave the feature silently dead. +""" +from types import SimpleNamespace + +import numpy as np +import pytest + +from noScribe import whisper_mp_worker as w + +SR = 16000 + + +def test_turns_become_padded_chunks_with_short_gaps_closed(): + turns = [[1.0, 2.0], [2.3, 3.0], [5.0, 6.0]] # 0.3 s gap closed, 2 s gap kept + got = w.speech_map(turns, 10 * SR) + pad = int(w.SPEECH_MAP_PAD_S * SR) + assert got == [{'start': SR - pad, 'end': 3 * SR + pad}, {'start': 5 * SR - pad, 'end': 6 * SR + pad}] + + +def test_chunks_stay_inside_the_audio_and_empty_turns_vanish(): + got = w.speech_map([[-1.0, 0.5], [3.0, 3.0], [9.8, 12.0]], 10 * SR) + assert got == [{'start': 0, 'end': int(0.55 * SR)}, {'start': int(9.75 * SR), 'end': 10 * SR}] + + +@pytest.mark.parametrize("min_gap, chunks", [(None, 2), (1.0, 1)]) +def test_the_gap_to_close_is_the_constant(monkeypatch, min_gap, chunks): + turns = [[1.0, 2.0], [2.7, 3.0]] # 0.6 s apart, 0.5 s after padding + if min_gap is not None: + monkeypatch.setattr(w, "SPEECH_MAP_MIN_GAP_S", min_gap) + assert len(w.speech_map(turns, 10 * SR)) == chunks + + +ft = pytest.importorskip("faster_whisper.transcribe") + + +def test_faster_whisper_still_asks_its_module_for_the_speech_map(): + """transcribe() and detect_language() must look get_speech_timestamps up as a + module global -- the name _speech_map_from replaces.""" + assert hasattr(ft, "get_speech_timestamps") + for fn in (ft.WhisperModel.transcribe, ft.WhisperModel.detect_language): + assert "get_speech_timestamps" in fn.__code__.co_names, fn.__qualname__ + + +def test_the_swap_answers_with_the_turns_and_is_undone(): + silero = ft.get_speech_timestamps + audio = np.zeros(10 * SR, np.float32) + with w._speech_map_from([[1.0, 2.0]]) as mapped: + assert mapped + assert ft.get_speech_timestamps(audio, None) == w.speech_map([[1.0, 2.0]], len(audio)) + assert ft.get_speech_timestamps is silero + + +def test_the_swap_is_undone_when_the_call_inside_fails(): + silero = ft.get_speech_timestamps + with pytest.raises(RuntimeError): + with w._speech_map_from([[1.0, 2.0]]): + raise RuntimeError("transcription failed") + assert ft.get_speech_timestamps is silero + + +@pytest.mark.parametrize("turns", [None, []]) +def test_without_turns_silero_stays(turns): + silero = ft.get_speech_timestamps + with w._speech_map_from(turns) as mapped: + assert not mapped and ft.get_speech_timestamps is silero + + +def test_main_hands_the_diarization_to_the_whisper_worker(tmp_path, monkeypatch): + """The worker's arguments, taken where main.py starts the child process.""" + pytest.importorskip("tkinter") + import queue + import noScribe.main as m + monkeypatch.setattr(m, "config", {}) + seen = [] + + class FakeProcess: + def __init__(self, target, args): + self.args, self.exitcode = args, 0 + + def start(self): + seen.append(self.args[0]) + self.args[1].put({"type": "result", "ok": True, "info": {"duration": 1.0}}) + + def is_alive(self): + return False + + def join(self, timeout=None): + pass + + monkeypatch.setattr(m.mp, "get_context", lambda method: SimpleNamespace(Queue=queue.Queue, Process=FakeProcess)) + app = m.HeadlessApp() + job = m.create_transcription_job(audio_file=str(tmp_path / "a.wav"), transcript_file=str(tmp_path / "a.html"), + speaker_detection="auto", cli_mode=True) + app._run_whisper_subprocess_stream("x.wav", job, lambda seg: None, + [{"start": 1500, "end": 4200, "label": "SPEAKER_00"}]) + app._run_whisper_subprocess_stream("x.wav", job, lambda seg: None) + assert [a["speech_turns"] for a in seen] == [[[1.5, 4.2]], None] + + +def test_the_worker_transcribes_inside_the_swap(tmp_path, monkeypatch): + """The wiring itself: whisper_proc_entrypoint must hand faster-whisper the turns' + map while transcribe() runs, and while detect_language() runs for 'Auto' -- + a stand-in model asks for it in both (the pattern of test_whisper_audio_lifetime).""" + from queue import Queue + asked = [] + ask = lambda kwargs: ft.get_speech_timestamps(np.zeros(10 * SR, np.float32), kwargs.get("vad_parameters")) + + class Model: + model = SimpleNamespace(is_multilingual=True) + feature_extractor = SimpleNamespace(sampling_rate=SR) + + def detect_language(self, audio, **kwargs): + asked.append(("detect", ask(kwargs))) + return "de", 1.0, [] + + def transcribe(self, path, **kwargs): + asked.append(ask(kwargs)) + return iter([]), SimpleNamespace(duration=10.0, language="de") + + monkeypatch.setattr("faster_whisper.WhisperModel", lambda *a, **kw: Model()) + monkeypatch.setattr("faster_whisper.audio.decode_audio", lambda path, sampling_rate: np.zeros(SR, np.float32)) + (tmp_path / "a.wav").touch() + want = w.speech_map([[1.0, 2.0]], 10 * SR) + for language, expected in (("German", [want]), ("Auto", [("detect", want), want])): + asked.clear() + messages = Queue() + w.whisper_proc_entrypoint({ + "whisper_model": SimpleNamespace(path=tmp_path), "device": "cpu", "audio_path": str(tmp_path / "a.wav"), + "language_name": language, "language_code": "de", "speech_turns": [[1.0, 2.0]]}, messages) + results = [m for m in list(messages.queue) if m["type"] == "result"] + assert len(results) == 1 and results[0]["ok"], results + assert asked == expected, language + + +def _whisper_stand_in(seen): + """Records its arguments and says one word: a job without any text is an error.""" + def stream(*args): + seen.append(args) + args[2]({"start": 0.5, "end": 1.0, "text": " Hallo", "words": []}) + return SimpleNamespace(duration=3.0) + return stream + + +@pytest.mark.parametrize("detection, handed", [("auto", True), ("none", False)]) +def test_the_diarization_reaches_whisper(tmp_path, monkeypatch, detection, handed): + """Through the pipeline: with speaker detection the turns go to Whisper, without + it there are none (and diarization is None rather than unbound).""" + pytest.importorskip("tkinter") + import soundfile as sf + import noScribe.main as m + monkeypatch.setattr(m, "config_dir", str(tmp_path)) + monkeypatch.setattr(m, "config", {}) + audio = tmp_path / "a.wav" + sf.write(audio, np.zeros(SR * 3, dtype=np.int16), SR) + app = m.HeadlessApp() + turns = [{"start": 0, "end": 2000, "label": "SPEAKER_00"}] + monkeypatch.setattr(app, "_run_diarize_subprocess", lambda *a, **k: turns) + seen = [] + monkeypatch.setattr(app, "_run_whisper_subprocess_stream", _whisper_stand_in(seen)) + job = m.create_transcription_job(audio_file=str(audio), transcript_file=str(tmp_path / "a.html"), + speaker_detection=detection, cli_mode=True) + app._process_single_job(job) + assert seen, "Whisper was not started" + assert seen[0][3] == (turns if handed else None)