feat(ai-formatting A.1 #22): expand hallucination blocklist for subtitle-training leakage

Whisper was trained on subtitle corpora, so silence and room tone
trigger caption-style artefacts that the previous three-marker
blocklist ("[blank_audio]", "[music]", "[silence]") didn't catch:
"Thanks for watching!", "Please subscribe.", "ご視聴ありがとうござ
いました", "♪♪♪", etc. Documented in WhisperLive #185 / #246 and
ufal/whisper_streaming #121 as the top streaming-transcript-quality
issue after chunk-boundary repeats.

HALLUCINATION_MARKERS widens from 3 to 16 entries: all common
bracketed non-speech tags (applause / laughter / inaudible /
background noise / sounds), parens variants, and musical notation
(♪ / ♫). Still contains-match so the marker triggers even when
Whisper wraps it in other noise.

HALLUCINATION_TRAIL_PHRASES (renamed from AUTO_THANKS_PHRASES) jumps
from 4 to ~30 entries: YouTube sign-offs, subtitle-credit leakage,
and the two most common non-English variants (Japanese "thanks for
watching" + MBC Korean news sign-off). Stays exact-match so
legitimate dialogue containing "thanks" or "subscribe" mid-sentence
never gets dropped — a new regression test pins that invariant.

The <15-char length gate on trail phrases is removed; some of the
new entries (e.g. "please subscribe to our channel.") are longer.
Exact-match against a known list is safety enough.

No caller changes: post_process_segments already drops segments for
which is_hallucination returns true when anti_hallucination is on.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

crates/ai-formatting/src/rule_based.rs

@@ -274,12 +274,91 @@ pub fn format_text(text: &str) -> String {
     result
 }
 
-/// Known hallucination markers that should be filtered from transcriptions.
+/// Substring markers that, if present anywhere in a segment, mean the
-static HALLUCINATION_MARKERS: &[&str] = &["[blank_audio]", "[music]", "[silence]"];
+/// segment is Whisper hallucinating silence / background noise as
+/// structured audio. Whisper's training data includes bracketed
+/// descriptions for non-speech (subtitle conventions), so long pauses
+/// and room tone routinely surface as "[music]", "♪♪♪", etc.
+static HALLUCINATION_MARKERS: &[&str] = &[
+    // Bracketed annotations (whisper.cpp and OpenAI-Whisper both emit these)
+    "[blank_audio]",
+    "[blank audio]",
+    "[silence]",
+    "[music]",
+    "[applause]",
+    "[laughter]",
+    "[laughs]",
+    "[inaudible]",
+    "[background noise]",
+    "[sounds]",
+    "(music)",
+    "(silence)",
+    "(applause)",
+    "(laughter)",
+    // Musical notation — "♪♪♪" appears when Whisper interprets room
+    // tone as a song.
+    "♪",
+    "♫",
+];
 
-static AUTO_THANKS_PHRASES: &[&str] = &["thank you.", "thanks.", "you.", "thank you for watching."];
+/// Exact-match (trimmed + lowercased) phrases that, as a whole segment,
+/// are indistinguishable from Whisper's subtitle-training artefacts.
+/// Compiled from WhisperLive #185, #246 and ufal/whisper_streaming #121
+/// — the YouTube / caption-dataset leakage that triggers on silence or
+/// room tone.
+///
+/// Exact match rather than contains, so real dialogue that happens to
+/// include "thanks" inside a longer sentence still passes.
+static HALLUCINATION_TRAIL_PHRASES: &[&str] = &[
+    // Minimalist false positives on silence.
+    "thank you.",
+    "thank you",
+    "thanks.",
+    "thanks",
+    "you.",
+    "you",
+    "bye.",
+    "bye",
+    // YouTube / subtitle sign-offs.
+    "thank you for watching.",
+    "thank you for watching!",
+    "thanks for watching.",
+    "thanks for watching!",
+    "thanks for watching, bye.",
+    "thanks for listening.",
+    "thanks for listening!",
+    "please subscribe.",
+    "please subscribe to our channel.",
+    "don't forget to subscribe.",
+    "don't forget to like and subscribe.",
+    "like and subscribe.",
+    "see you in the next video.",
+    "see you next time.",
+    // Subtitle-credit leakage.
+    "subtitles by the amara.org community",
+    "subtitles by the",
+    "subtitled by",
+    "subtitles by",
+    "translated by",
+    // Non-English subtitle sign-offs that leak into English-transcription
+    // output on silence. Kept lowercased for exact-match consistency.
+    "ご視聴ありがとうございました",
+    "字幕作成者",
+    "字幕by",
+    "字幕",
+    "mbc 뉴스 김수영입니다",
+];
 
 /// Returns true if a segment's text looks like a hallucination.
+///
+/// Two passes:
+/// - **Contains-match on HALLUCINATION_MARKERS** — catches bracketed
+///   and musical markers even when Whisper surrounds them with other
+///   noise ("♪♪♪ thanks for watching ♪♪♪").
+/// - **Exact-match on HALLUCINATION_TRAIL_PHRASES** — catches the
+///   well-documented subtitle-training leakage without false-positiving
+///   on legitimate dialogue that happens to mention "thanks" or
+///   "subscribe" mid-sentence.
 pub fn is_hallucination(text: &str) -> bool {
     let trimmed = text.trim().to_lowercase();
     if trimmed.is_empty() {
@@ -290,11 +369,9 @@ pub fn is_hallucination(text: &str) -> bool {
             return true;
         }
     }
-    if trimmed.len() < 15 {
+    for phrase in HALLUCINATION_TRAIL_PHRASES {
-        for phrase in AUTO_THANKS_PHRASES {
+        if trimmed == *phrase {
-            if trimmed == *phrase {
+            return true;
-                return true;
-            }
         }
     }
     false
@@ -382,8 +459,44 @@ mod tests {
         assert!(is_hallucination("thanks."));
     }
 
+    #[test]
+    fn is_hallucination_detects_subtitle_trailers() {
+        // WhisperLive #185 / ufal #121 class: subtitle-training leakage
+        // that fires on silence or room tone.
+        assert!(is_hallucination("Thanks for watching!"));
+        assert!(is_hallucination("Thanks for watching."));
+        assert!(is_hallucination("Please subscribe."));
+        assert!(is_hallucination("Don't forget to like and subscribe."));
+        assert!(is_hallucination("See you next time."));
+        assert!(is_hallucination("Subtitles by the Amara.org community"));
+    }
+
+    #[test]
+    fn is_hallucination_detects_music_and_sound_markers() {
+        assert!(is_hallucination("♪"));
+        assert!(is_hallucination("♪♪♪"));
+        assert!(is_hallucination("[applause]"));
+        assert!(is_hallucination("[Laughter]"));
+        assert!(is_hallucination("[Background noise]"));
+    }
+
+    #[test]
+    fn is_hallucination_detects_non_english_subtitle_leakage() {
+        // Japanese "thank you for watching"; MBC Korean news sign-off.
+        assert!(is_hallucination("ご視聴ありがとうございました"));
+        assert!(is_hallucination("MBC 뉴스 김수영입니다"));
+    }
+
     #[test]
     fn is_hallucination_allows_real_text() {
         assert!(!is_hallucination("The meeting is at three o'clock."));
     }
+
+    #[test]
+    fn is_hallucination_allows_dialogue_containing_thanks_mid_sentence() {
+        // Exact-match on trail phrases means legitimate dialogue that
+        // mentions "thanks" or "subscribe" is never dropped.
+        assert!(!is_hallucination("Thanks for the heads up on the migration"));
+        assert!(!is_hallucination("Please subscribe to the RSS feed and tell me when it updates"));
+    }
 }