Let Whisper listen where the diarization found speakers, so a quiet second voice is transcribed instead of cut away by the VAD

noScribe/main.py

@@ -2841,6 +2841,7 @@ def find_speaker(diarization, transcript_start, transcript_end) -> str:
 
                 # Start Diarization:
 
+                diarization = None
                 if job.speaker_detection != 'none':
                     try:
                         job.status = JobStatus.SPEAKER_IDENTIFICATION
@@ -3209,7 +3210,7 @@ def __init__(self, d):
                             pass
 
                     try:
-                        info = self._run_whisper_subprocess_stream(tmp_audio_file, job, on_segment)
+                        info = self._run_whisper_subprocess_stream(tmp_audio_file, job, on_segment, diarization)
                         if first_segment:
                             raise ValueError(t('err_empty_transcript'))
                         transcription_success = True
@@ -3357,10 +3358,12 @@ def _handle_cuda_fallback(self, component: str, error: Exception) -> bool:
 
         return False
 
-    def _run_whisper_subprocess_stream(self, tmp_audio_file: str, job, on_segment):
+    def _run_whisper_subprocess_stream(self, tmp_audio_file: str, job, on_segment, diarization=None):
         """Spawn a subprocess to run Faster-Whisper and stream segments.
         Calls on_segment(dict) for each segment streamed by the child.
-        Returns a simple info object (duration at least).
+        Returns a simple info object (duration at least). With a diarization,
+        its turns are the speech map in place of Silero's
+        (the measurement is above whisper_mp_worker._speech_map_from).
         """
         global force_whisper_cpu
         # Language code for non-auto/multilingual
@@ -3392,6 +3395,9 @@ def _run_whisper_subprocess_stream(self, tmp_audio_file: str, job, on_segment):
             "vad_filter": True,
             "vad_threshold": vad_threshold,
             "locale": config.get("locale", "en"),
+            # Same converted-WAV timeline the worker reads, in seconds.
+            "speech_turns": ([[seg["start"] / 1000.0, seg["end"] / 1000.0] for seg in diarization]
+                             if diarization else None),
         }
 
         # Spawn child process using spawn start method

noScribe/whisper_mp_worker.py

@@ -1,3 +1,4 @@
+import contextlib
 import gc
 import logging
 import os
@@ -15,6 +16,64 @@
 
 logger = logging.getLogger(__name__)
 
+# Gaps closed and edges padded by the same amounts as the Silero options in
+# whisper_proc_entrypoint, and kept equal on purpose -- but measured for this map on
+# its own: closing gaps up to 2 s instead let one phone call become a single 31-s
+# chunk that Whisper decoded into an invented sentence.
+SPEECH_MAP_MIN_GAP_S = 0.5
+SPEECH_MAP_PAD_S = 0.05
+
+
+def speech_map(turns, n_samples, sampling_rate=16000):
+    """faster-whisper's speech chunks ({'start', 'end'} in samples) from
+    [start_s, end_s] turns: padded by SPEECH_MAP_PAD_S, merged across gaps under
+    SPEECH_MAP_MIN_GAP_S, clipped to the audio."""
+    spans = sorted((max(0, int((s - SPEECH_MAP_PAD_S) * sampling_rate)),
+                    min(n_samples, int((e + SPEECH_MAP_PAD_S) * sampling_rate)))
+                   for s, e in turns if e > s)  # an empty turn is no speech, padded or not
+    out = []
+    for s, e in spans:
+        if s >= e:  # a turn that lies past the end of the audio
+            continue
+        if out and s - out[-1]['end'] < SPEECH_MAP_MIN_GAP_S * sampling_rate:
+            out[-1]['end'] = max(out[-1]['end'], e)
+        else:
+            out.append({'start': s, 'end': e})
+    return out
+
+
+# With speaker detection, the diarization's turns tell Whisper where speech is,
+# in place of Silero's voice activity. Silero drops quiet speech before Whisper
+# ever hears it: of a voice 15-20 dB below the one beside it, it kept 27-41 % of
+# the time, and Whisper wrote 2-3 % of the words in what it dropped. Lowering its
+# threshold does not help (0.35 is within noise on AMI, 0.15 lets room tone in).
+# The diarization's turns kept 67 % of that voice, and with them as the speech
+# map Whisper wrote 50-65 % of its words instead of 27-46 %, with none in pure
+# noise. On all 23 AMI meetings, against their manual words, it made 1.39 points
+# fewer errors per reference word (95 % [+0.93, +1.92]), with more words found
+# in every meeting; on four hand-checked German passages 9.28 % -> 8.75 % WER,
+# all four better. Single files vary between runs either way -- faster-whisper
+# re-decodes an unsure window by sampling, unseeded, and quiet speech is where it
+# is unsure (one recording gave 219 to 292 words) -- so only the totals say anything.
+@contextlib.contextmanager
+def _speech_map_from(turns):
+    """While active, faster-whisper takes its speech chunks from `turns` instead
+    of Silero. It asks for them through the one name it imports into its
+    transcribe module, and nothing else there changes: the chunks are collected
+    and the timestamps mapped back exactly as with Silero's. Yields whether the
+    swap is in place -- not without turns, nor if a faster-whisper release no
+    longer has that name (tests/test_whisper_speech_map.py pins it)."""
+    import faster_whisper.transcribe as ft
+    silero = getattr(ft, "get_speech_timestamps", None)
+    if not turns or silero is None:
+        yield False
+        return
+    ft.get_speech_timestamps = lambda audio, *a, **k: speech_map(turns, len(audio))
+    try:
+        yield True
+    finally:
+        ft.get_speech_timestamps = silero
+
 
 def whisper_proc_entrypoint(args: dict, q):
     """
@@ -119,15 +178,18 @@ def log_cb(level, msg):
         else:
             whisper_lang = language_code
 
+        speech_turns = args.get("speech_turns")
+
         # Detect language if requested (Auto)
         if language_name == "Auto":
             audio = decode_audio(
                 audio_path, sampling_rate=model.feature_extractor.sampling_rate
             )
             try:
-                whisper_lang, language_probability, _ = model.detect_language(
-                    audio, vad_filter=True, vad_parameters=vad_parameters
-                )
+                with _speech_map_from(speech_turns):
+                    whisper_lang, language_probability, _ = model.detect_language(
+                        audio, vad_filter=True, vad_parameters=vad_parameters
+                    )
             finally:
                 del audio
                 gc.collect()
@@ -148,18 +210,23 @@ def log_cb(level, msg):
 
         # Pass the path so faster-whisper can release its original waveform
         # after VAD, before allocating the full spectrogram (see 28650f2e).
-        segments, info = model.transcribe(
-            audio_path,
-            language=whisper_lang,
-            multilingual=multilingual,
-            beam_size=args.get("beam_size", 5),
-            # temperature=args.get("temperature"),
-            word_timestamps=args.get("word_timestamps", True),
-            # initial_prompt=prompt,
-            hotwords=prompt,
-            vad_filter=args.get("vad_filter", True),
-            vad_parameters=vad_parameters,
-        )
+        # transcribe() takes its speech map before it returns the lazy segments,
+        # so the swap need only last for the call.
+        with _speech_map_from(speech_turns) as mapped:
+            segments, info = model.transcribe(
+                audio_path,
+                language=whisper_lang,
+                multilingual=multilingual,
+                beam_size=args.get("beam_size", 5),
+                # temperature=args.get("temperature"),
+                word_timestamps=args.get("word_timestamps", True),
+                # initial_prompt=prompt,
+                hotwords=prompt,
+                vad_filter=args.get("vad_filter", True),
+                vad_parameters=vad_parameters,
+            )
+        if speech_turns and not mapped:
+            log_cb("debug", "speech map from the diarization unavailable; Silero decides where speech is")
 
         log_cb('info', t('start_transcription') + '\n')
 

tests/test_empty_transcript.py

@@ -51,7 +51,7 @@ def _segment(text, start=1.0):
 @pytest.mark.parametrize("detection", ["none", "auto"])
 @pytest.mark.parametrize("texts", [[], ["", " \t\n ", "\u2003"]])
 def test_empty_result_fails_without_saving(app, tmp_path, monkeypatch, suffix, detection, texts):
-    def whisper(path, job, on_segment):
+    def whisper(path, job, on_segment, *speech_map):
         for text in texts:
             on_segment(_segment(text))
         return {}
@@ -82,7 +82,7 @@ def test_batch_continues_after_empty_job(app, tmp_path, monkeypatch):
     empty = _job(tmp_path, name="empty")
     valid = _job(tmp_path, name="valid")
 
-    def whisper(path, job, on_segment):
+    def whisper(path, job, on_segment, *speech_map):
         if job is valid:
             on_segment(_segment(" Hello"))
         return {}
@@ -100,7 +100,7 @@ def whisper(path, job, on_segment):
 @pytest.mark.parametrize("suffix", ["html", "txt", "vtt"])
 @pytest.mark.parametrize("canceled", [False, True])
 def test_failed_or_canceled_job_keeps_real_partial_text(app, tmp_path, monkeypatch, suffix, canceled):
-    def whisper(path, job, on_segment):
+    def whisper(path, job, on_segment, *speech_map):
         on_segment(_segment(" Hello"))
         on_segment(_segment(" \t ", start=2.0))
         if canceled:
@@ -138,7 +138,7 @@ def test_segment_logs_metadata_without_text(app, tmp_path, monkeypatch, capsys,
     monkeypatch.setattr(app, "logn", m.App.logn.__get__(app))
     text = " Confidential interview content."
 
-    def whisper(path, job, on_segment):
+    def whisper(path, job, on_segment, *speech_map):
         on_segment(_segment(text))
         on_segment(_segment(" \t ", start=2.0))
         return {}

tests/test_transcript_media_timeline.py

@@ -23,7 +23,7 @@ def test_transcript_uses_actual_converted_media_position(
     )
     logs = []
 
-    def stream(path, job, on_segment):
+    def stream(path, job, on_segment, *speech_map):
         # Leave a one-second pause before the first phrase to cover pause anchors.
         on_segment({"start": 1.0, "end": 1.5, "text": " Hello", "words": []})
         return {}

tests/test_whisper_speech_map.py

@@ -0,0 +1,171 @@
+"""With speaker detection, Whisper takes its speech map from the diarization.
+
+Silero dropped a quiet voice before Whisper could hear it; the diarization's
+turns keep it (the measurement is above _speech_map_from in
+noScribe/whisper_mp_worker.py). The swap works through the one name
+faster-whisper imports into its transcribe module, so the guard that matters
+most is that it still looks the map up there: a faster-whisper release that
+stops doing so would leave the feature silently dead.
+"""
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+
+from noScribe import whisper_mp_worker as w
+
+SR = 16000
+
+
+def test_turns_become_padded_chunks_with_short_gaps_closed():
+    turns = [[1.0, 2.0], [2.3, 3.0], [5.0, 6.0]]   # 0.3 s gap closed, 2 s gap kept
+    got = w.speech_map(turns, 10 * SR)
+    pad = int(w.SPEECH_MAP_PAD_S * SR)
+    assert got == [{'start': SR - pad, 'end': 3 * SR + pad}, {'start': 5 * SR - pad, 'end': 6 * SR + pad}]
+
+
+def test_chunks_stay_inside_the_audio_and_empty_turns_vanish():
+    got = w.speech_map([[-1.0, 0.5], [3.0, 3.0], [9.8, 12.0]], 10 * SR)
+    assert got == [{'start': 0, 'end': int(0.55 * SR)}, {'start': int(9.75 * SR), 'end': 10 * SR}]
+
+
+@pytest.mark.parametrize("min_gap, chunks", [(None, 2), (1.0, 1)])
+def test_the_gap_to_close_is_the_constant(monkeypatch, min_gap, chunks):
+    turns = [[1.0, 2.0], [2.7, 3.0]]   # 0.6 s apart, 0.5 s after padding
+    if min_gap is not None:
+        monkeypatch.setattr(w, "SPEECH_MAP_MIN_GAP_S", min_gap)
+    assert len(w.speech_map(turns, 10 * SR)) == chunks
+
+
+ft = pytest.importorskip("faster_whisper.transcribe")
+
+
+def test_faster_whisper_still_asks_its_module_for_the_speech_map():
+    """transcribe() and detect_language() must look get_speech_timestamps up as a
+    module global -- the name _speech_map_from replaces."""
+    assert hasattr(ft, "get_speech_timestamps")
+    for fn in (ft.WhisperModel.transcribe, ft.WhisperModel.detect_language):
+        assert "get_speech_timestamps" in fn.__code__.co_names, fn.__qualname__
+
+
+def test_the_swap_answers_with_the_turns_and_is_undone():
+    silero = ft.get_speech_timestamps
+    audio = np.zeros(10 * SR, np.float32)
+    with w._speech_map_from([[1.0, 2.0]]) as mapped:
+        assert mapped
+        assert ft.get_speech_timestamps(audio, None) == w.speech_map([[1.0, 2.0]], len(audio))
+    assert ft.get_speech_timestamps is silero
+
+
+def test_the_swap_is_undone_when_the_call_inside_fails():
+    silero = ft.get_speech_timestamps
+    with pytest.raises(RuntimeError):
+        with w._speech_map_from([[1.0, 2.0]]):
+            raise RuntimeError("transcription failed")
+    assert ft.get_speech_timestamps is silero
+
+
+@pytest.mark.parametrize("turns", [None, []])
+def test_without_turns_silero_stays(turns):
+    silero = ft.get_speech_timestamps
+    with w._speech_map_from(turns) as mapped:
+        assert not mapped and ft.get_speech_timestamps is silero
+
+
+def test_main_hands_the_diarization_to_the_whisper_worker(tmp_path, monkeypatch):
+    """The worker's arguments, taken where main.py starts the child process."""
+    pytest.importorskip("tkinter")
+    import queue
+    import noScribe.main as m
+    monkeypatch.setattr(m, "config", {})
+    seen = []
+
+    class FakeProcess:
+        def __init__(self, target, args):
+            self.args, self.exitcode = args, 0
+
+        def start(self):
+            seen.append(self.args[0])
+            self.args[1].put({"type": "result", "ok": True, "info": {"duration": 1.0}})
+
+        def is_alive(self):
+            return False
+
+        def join(self, timeout=None):
+            pass
+
+    monkeypatch.setattr(m.mp, "get_context", lambda method: SimpleNamespace(Queue=queue.Queue, Process=FakeProcess))
+    app = m.HeadlessApp()
+    job = m.create_transcription_job(audio_file=str(tmp_path / "a.wav"), transcript_file=str(tmp_path / "a.html"),
+                                     speaker_detection="auto", cli_mode=True)
+    app._run_whisper_subprocess_stream("x.wav", job, lambda seg: None,
+                                       [{"start": 1500, "end": 4200, "label": "SPEAKER_00"}])
+    app._run_whisper_subprocess_stream("x.wav", job, lambda seg: None)
+    assert [a["speech_turns"] for a in seen] == [[[1.5, 4.2]], None]
+
+
+def test_the_worker_transcribes_inside_the_swap(tmp_path, monkeypatch):
+    """The wiring itself: whisper_proc_entrypoint must hand faster-whisper the turns'
+    map while transcribe() runs, and while detect_language() runs for 'Auto' --
+    a stand-in model asks for it in both (the pattern of test_whisper_audio_lifetime)."""
+    from queue import Queue
+    asked = []
+    ask = lambda kwargs: ft.get_speech_timestamps(np.zeros(10 * SR, np.float32), kwargs.get("vad_parameters"))
+
+    class Model:
+        model = SimpleNamespace(is_multilingual=True)
+        feature_extractor = SimpleNamespace(sampling_rate=SR)
+
+        def detect_language(self, audio, **kwargs):
+            asked.append(("detect", ask(kwargs)))
+            return "de", 1.0, []
+
+        def transcribe(self, path, **kwargs):
+            asked.append(ask(kwargs))
+            return iter([]), SimpleNamespace(duration=10.0, language="de")
+
+    monkeypatch.setattr("faster_whisper.WhisperModel", lambda *a, **kw: Model())
+    monkeypatch.setattr("faster_whisper.audio.decode_audio", lambda path, sampling_rate: np.zeros(SR, np.float32))
+    (tmp_path / "a.wav").touch()
+    want = w.speech_map([[1.0, 2.0]], 10 * SR)
+    for language, expected in (("German", [want]), ("Auto", [("detect", want), want])):
+        asked.clear()
+        messages = Queue()
+        w.whisper_proc_entrypoint({
+            "whisper_model": SimpleNamespace(path=tmp_path), "device": "cpu", "audio_path": str(tmp_path / "a.wav"),
+            "language_name": language, "language_code": "de", "speech_turns": [[1.0, 2.0]]}, messages)
+        results = [m for m in list(messages.queue) if m["type"] == "result"]
+        assert len(results) == 1 and results[0]["ok"], results
+        assert asked == expected, language
+
+
+def _whisper_stand_in(seen):
+    """Records its arguments and says one word: a job without any text is an error."""
+    def stream(*args):
+        seen.append(args)
+        args[2]({"start": 0.5, "end": 1.0, "text": " Hallo", "words": []})
+        return SimpleNamespace(duration=3.0)
+    return stream
+
+
+@pytest.mark.parametrize("detection, handed", [("auto", True), ("none", False)])
+def test_the_diarization_reaches_whisper(tmp_path, monkeypatch, detection, handed):
+    """Through the pipeline: with speaker detection the turns go to Whisper, without
+    it there are none (and diarization is None rather than unbound)."""
+    pytest.importorskip("tkinter")
+    import soundfile as sf
+    import noScribe.main as m
+    monkeypatch.setattr(m, "config_dir", str(tmp_path))
+    monkeypatch.setattr(m, "config", {})
+    audio = tmp_path / "a.wav"
+    sf.write(audio, np.zeros(SR * 3, dtype=np.int16), SR)
+    app = m.HeadlessApp()
+    turns = [{"start": 0, "end": 2000, "label": "SPEAKER_00"}]
+    monkeypatch.setattr(app, "_run_diarize_subprocess", lambda *a, **k: turns)
+    seen = []
+    monkeypatch.setattr(app, "_run_whisper_subprocess_stream", _whisper_stand_in(seen))
+    job = m.create_transcription_job(audio_file=str(audio), transcript_file=str(tmp_path / "a.html"),
+                                     speaker_detection=detection, cli_mode=True)
+    app._process_single_job(job)
+    assert seen, "Whisper was not started"
+    assert seen[0][3] == (turns if handed else None)