574 lines
18 KiB
Rust
574 lines
18 KiB
Rust
use std::sync::LazyLock;
|
||
|
||
/// Compiled filler word regexes (built once, reused across calls).
|
||
/// Uses \b word boundaries instead of lookbehinds (regex-lite does not
|
||
/// support lookaround assertions).
|
||
static FILLER_REGEXES: LazyLock<Vec<regex_lite::Regex>> = LazyLock::new(|| {
|
||
let fillers = [
|
||
"um",
|
||
"uh",
|
||
"er",
|
||
"ah",
|
||
"like",
|
||
"you know",
|
||
"sort of",
|
||
"kind of",
|
||
"I mean",
|
||
"basically",
|
||
"actually",
|
||
"literally",
|
||
];
|
||
fillers
|
||
.iter()
|
||
.filter_map(|filler| {
|
||
let escaped = regex_lite::escape(filler);
|
||
let pattern = format!(r"(?i)\b{escaped}\b[,.]?\s*");
|
||
regex_lite::Regex::new(&pattern).ok()
|
||
})
|
||
.collect()
|
||
});
|
||
|
||
fn normalise_repetition_token(token: &str) -> String {
|
||
token
|
||
.trim_matches(|ch: char| !(ch.is_alphanumeric() || ch == '\'' || ch == '-'))
|
||
.to_lowercase()
|
||
}
|
||
|
||
/// Remove common filler words from transcription text (case-insensitive).
|
||
pub fn remove_fillers(text: &str) -> String {
|
||
let mut result = text.to_string();
|
||
|
||
for re in FILLER_REGEXES.iter() {
|
||
result = re.replace_all(&result, " ").to_string();
|
||
}
|
||
|
||
// Collapse runs of whitespace in a single pass.
|
||
let mut collapsed = String::with_capacity(result.len());
|
||
let mut prev_space = false;
|
||
for ch in result.chars() {
|
||
if ch == ' ' {
|
||
if !prev_space {
|
||
collapsed.push(' ');
|
||
}
|
||
prev_space = true;
|
||
} else {
|
||
prev_space = false;
|
||
collapsed.push(ch);
|
||
}
|
||
}
|
||
|
||
collapsed.trim().to_string()
|
||
}
|
||
|
||
/// Collapse obvious stutters and immediate repeated short phrases.
|
||
///
|
||
/// Examples:
|
||
/// - `I I can` -> `I can`
|
||
/// - `I need I need to go` -> `I need to go`
|
||
/// - `Think think that's that` -> `Think that's that`
|
||
pub fn collapse_repetitions(text: &str) -> String {
|
||
if text.trim().is_empty() {
|
||
return String::new();
|
||
}
|
||
|
||
let tokens: Vec<&str> = text.split_whitespace().collect();
|
||
if tokens.len() < 2 {
|
||
return text.trim().to_string();
|
||
}
|
||
|
||
let normalised: Vec<String> = tokens
|
||
.iter()
|
||
.map(|token| normalise_repetition_token(token))
|
||
.collect();
|
||
let mut kept_indices: Vec<usize> = Vec::with_capacity(tokens.len());
|
||
let mut i = 0;
|
||
|
||
while i < tokens.len() {
|
||
let mut skipped_phrase = false;
|
||
|
||
for phrase_len in (1..=3).rev() {
|
||
if kept_indices.len() < phrase_len || i + phrase_len > tokens.len() {
|
||
continue;
|
||
}
|
||
|
||
let repeated = (0..phrase_len).all(|offset| {
|
||
let prev_index = kept_indices[kept_indices.len() - phrase_len + offset];
|
||
let prev = &normalised[prev_index];
|
||
let upcoming = &normalised[i + offset];
|
||
!prev.is_empty() && prev == upcoming
|
||
});
|
||
|
||
if repeated {
|
||
i += phrase_len;
|
||
skipped_phrase = true;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if skipped_phrase {
|
||
continue;
|
||
}
|
||
|
||
if let Some(&last_index) = kept_indices.last() {
|
||
let current = &normalised[i];
|
||
let previous = &normalised[last_index];
|
||
if !current.is_empty() && current == previous {
|
||
i += 1;
|
||
continue;
|
||
}
|
||
}
|
||
|
||
kept_indices.push(i);
|
||
i += 1;
|
||
}
|
||
|
||
kept_indices
|
||
.into_iter()
|
||
.map(|index| tokens[index])
|
||
.collect::<Vec<_>>()
|
||
.join(" ")
|
||
.trim()
|
||
.to_string()
|
||
}
|
||
|
||
/// Replacement pairs for American to British English conversion.
|
||
///
|
||
/// All entries are plain base words (no regex metacharacters). The
|
||
/// `to_british_english` function wraps every entry with `\b` word
|
||
/// boundaries and optional suffix matching automatically.
|
||
static BRITISH_REPLACEMENTS: &[(&str, &str)] = &[
|
||
// -ize → -ise (and inflected forms)
|
||
("organize", "organise"),
|
||
("recognize", "recognise"),
|
||
("realize", "realise"),
|
||
("analyze", "analyse"),
|
||
("apologize", "apologise"),
|
||
("authorize", "authorise"),
|
||
("categorize", "categorise"),
|
||
("characterize", "characterise"),
|
||
("customize", "customise"),
|
||
("digitize", "digitise"),
|
||
("emphasize", "emphasise"),
|
||
("finalize", "finalise"),
|
||
("generalize", "generalise"),
|
||
("harmonize", "harmonise"),
|
||
("initialize", "initialise"),
|
||
("maximize", "maximise"),
|
||
("minimize", "minimise"),
|
||
("modernize", "modernise"),
|
||
("normalize", "normalise"),
|
||
("optimize", "optimise"),
|
||
("prioritize", "prioritise"),
|
||
("revolutionize", "revolutionise"),
|
||
("specialize", "specialise"),
|
||
("standardize", "standardise"),
|
||
("summarize", "summarise"),
|
||
("utilize", "utilise"),
|
||
// -or → -our
|
||
("color", "colour"),
|
||
("favor", "favour"),
|
||
("honor", "honour"),
|
||
("humor", "humour"),
|
||
("labor", "labour"),
|
||
("neighbor", "neighbour"),
|
||
("behavior", "behaviour"),
|
||
// -er → -re
|
||
("center", "centre"),
|
||
("fiber", "fibre"),
|
||
("liter", "litre"),
|
||
("meter", "metre"),
|
||
("theater", "theatre"),
|
||
// -ense → -ence
|
||
("defense", "defence"),
|
||
("offense", "offence"),
|
||
// Other
|
||
("catalog", "catalogue"),
|
||
("dialog", "dialogue"),
|
||
];
|
||
|
||
/// Convert American English spelling to British English (word-boundary aware).
|
||
pub fn to_british_english(text: &str) -> String {
|
||
let mut result = text.to_string();
|
||
|
||
for (us, uk) in BRITISH_REPLACEMENTS {
|
||
// Every entry in BRITISH_REPLACEMENTS is a plain ASCII base word.
|
||
// We wrap it with \b boundaries and optional suffix matching here.
|
||
let pattern = format!("(?i)\\b{}(?:d|s|r|rs)?\\b", regex_lite::escape(us));
|
||
|
||
if let Ok(re) = regex_lite::Regex::new(&pattern) {
|
||
result = re
|
||
.replace_all(&result, |caps: ®ex_lite::Captures| {
|
||
let Some(m) = caps.get(0) else {
|
||
return String::new();
|
||
};
|
||
let matched = m.as_str();
|
||
let base_len = us.len();
|
||
// SAFETY: byte indexing is correct here because both
|
||
// the US base word and the suffix characters (d, s, r)
|
||
// are guaranteed ASCII by the BRITISH_REPLACEMENTS table.
|
||
debug_assert!(us.is_ascii(), "BRITISH_REPLACEMENTS entries must be ASCII");
|
||
debug_assert!(matched.is_ascii(), "matched text expected to be ASCII");
|
||
let suffix = if matched.len() > base_len {
|
||
&matched[base_len..]
|
||
} else {
|
||
""
|
||
};
|
||
let Some(first_char) = matched.chars().next() else {
|
||
return String::new();
|
||
};
|
||
if first_char.is_uppercase() {
|
||
let mut chars = uk.chars();
|
||
let Some(first_uk) = chars.next() else {
|
||
return String::new();
|
||
};
|
||
let upper_first: String = first_uk.to_uppercase().collect();
|
||
format!("{}{}{}", upper_first, chars.collect::<String>(), suffix)
|
||
} else {
|
||
format!("{uk}{suffix}")
|
||
}
|
||
})
|
||
.to_string();
|
||
}
|
||
}
|
||
|
||
result
|
||
}
|
||
|
||
/// Basic formatting: capitalise sentences, fix spacing, clean punctuation.
|
||
pub fn format_text(text: &str) -> String {
|
||
if text.is_empty() {
|
||
return String::new();
|
||
}
|
||
|
||
let mut result = String::with_capacity(text.len());
|
||
let mut capitalise_next = true;
|
||
|
||
let chars: Vec<char> = text.chars().collect();
|
||
let mut i = 0;
|
||
|
||
while i < chars.len() {
|
||
let c = chars[i];
|
||
|
||
if c == ' ' && i + 1 < chars.len() && chars[i + 1] == ' ' {
|
||
i += 1;
|
||
continue;
|
||
}
|
||
|
||
if capitalise_next && c.is_alphabetic() {
|
||
result.extend(c.to_uppercase());
|
||
capitalise_next = false;
|
||
} else {
|
||
result.push(c);
|
||
}
|
||
|
||
if c == '.' || c == '!' || c == '?' {
|
||
capitalise_next = true;
|
||
}
|
||
if c == '\n' {
|
||
capitalise_next = true;
|
||
}
|
||
|
||
i += 1;
|
||
}
|
||
|
||
result
|
||
}
|
||
|
||
/// Substring markers that, if present anywhere in a segment, mean the
|
||
/// segment is Whisper hallucinating silence / background noise as
|
||
/// structured audio. Whisper's training data includes bracketed
|
||
/// descriptions for non-speech (subtitle conventions), so long pauses
|
||
/// and room tone routinely surface as "[music]", "♪♪♪", etc.
|
||
static HALLUCINATION_MARKERS: &[&str] = &[
|
||
// Bracketed annotations (whisper.cpp and OpenAI-Whisper both emit these)
|
||
"[blank_audio]",
|
||
"[blank audio]",
|
||
"[silence]",
|
||
"[music]",
|
||
"[applause]",
|
||
"[laughter]",
|
||
"[laughs]",
|
||
"[inaudible]",
|
||
"[background noise]",
|
||
"[sounds]",
|
||
"(music)",
|
||
"(silence)",
|
||
"(applause)",
|
||
"(laughter)",
|
||
// Musical notation — "♪♪♪" appears when Whisper interprets room
|
||
// tone as a song.
|
||
"♪",
|
||
"♫",
|
||
];
|
||
|
||
/// Exact-match (trimmed + lowercased) phrases that, as a whole segment,
|
||
/// are indistinguishable from Whisper's subtitle-training artefacts.
|
||
/// Compiled from WhisperLive #185, #246 and ufal/whisper_streaming #121
|
||
/// — the YouTube / caption-dataset leakage that triggers on silence or
|
||
/// room tone.
|
||
///
|
||
/// Exact match rather than contains, so real dialogue that happens to
|
||
/// include "thanks" inside a longer sentence still passes.
|
||
static HALLUCINATION_TRAIL_PHRASES: &[&str] = &[
|
||
// Minimalist false positives on silence.
|
||
"thank you.",
|
||
"thank you",
|
||
"thanks.",
|
||
"thanks",
|
||
"you.",
|
||
"you",
|
||
"bye.",
|
||
"bye",
|
||
// YouTube / subtitle sign-offs.
|
||
"thank you for watching.",
|
||
"thank you for watching!",
|
||
"thanks for watching.",
|
||
"thanks for watching!",
|
||
"thanks for watching, bye.",
|
||
"thanks for listening.",
|
||
"thanks for listening!",
|
||
"please subscribe.",
|
||
"please subscribe to our channel.",
|
||
"don't forget to subscribe.",
|
||
"don't forget to like and subscribe.",
|
||
"like and subscribe.",
|
||
"see you in the next video.",
|
||
"see you next time.",
|
||
// Subtitle-credit leakage.
|
||
"subtitles by the amara.org community",
|
||
"subtitles by the",
|
||
"subtitled by",
|
||
"subtitles by",
|
||
"translated by",
|
||
// Non-English subtitle sign-offs that leak into English-transcription
|
||
// output on silence. Kept lowercased for exact-match consistency.
|
||
"ご視聴ありがとうございました",
|
||
"字幕作成者",
|
||
"字幕by",
|
||
"字幕",
|
||
"mbc 뉴스 김수영입니다",
|
||
];
|
||
|
||
/// Minimum run length for the token-repetition detector (brief item
|
||
/// A.1 #26). Whisper's prompt-loop failure mode (ufal #161) typically
|
||
/// produces 5–10+ consecutive identical tokens; requiring 4 catches
|
||
/// those cleanly while leaving natural dialogue alone — three-in-a-row
|
||
/// is common speech ("no no no, that's wrong"), four-in-a-row almost
|
||
/// never is.
|
||
const REPETITION_RUN_THRESHOLD: usize = 4;
|
||
|
||
/// Returns true if a segment's text looks like a hallucination.
|
||
///
|
||
/// Three passes:
|
||
/// - **Contains-match on HALLUCINATION_MARKERS** — catches bracketed
|
||
/// and musical markers even when Whisper surrounds them with other
|
||
/// noise ("♪♪♪ thanks for watching ♪♪♪").
|
||
/// - **Exact-match on HALLUCINATION_TRAIL_PHRASES** — catches the
|
||
/// well-documented subtitle-training leakage without false-positiving
|
||
/// on legitimate dialogue that happens to mention "thanks" or
|
||
/// "subscribe" mid-sentence.
|
||
/// - **Consecutive-repetition detector** — Whisper occasionally enters
|
||
/// a prompt-loop where a single token cascades for dozens of words.
|
||
/// Flagging it here lets the existing anti_hallucination pipeline
|
||
/// drop the chunk rather than emitting "I I I I I I I I I …".
|
||
pub fn is_hallucination(text: &str) -> bool {
|
||
let trimmed = text.trim().to_lowercase();
|
||
if trimmed.is_empty() {
|
||
return true;
|
||
}
|
||
for marker in HALLUCINATION_MARKERS {
|
||
if trimmed.contains(marker) {
|
||
return true;
|
||
}
|
||
}
|
||
for phrase in HALLUCINATION_TRAIL_PHRASES {
|
||
if trimmed == *phrase {
|
||
return true;
|
||
}
|
||
}
|
||
if has_consecutive_repetition(&trimmed, REPETITION_RUN_THRESHOLD) {
|
||
return true;
|
||
}
|
||
false
|
||
}
|
||
|
||
/// Returns true when `text` contains at least `min_run` consecutive
|
||
/// identical whitespace-separated tokens (case-insensitive).
|
||
///
|
||
/// Detects the prompt-loop failure mode that Whisper falls into on
|
||
/// ambiguous audio (ufal #161) without flagging normal triple-repeats
|
||
/// that appear in everyday speech ("no no no, that's wrong"). The
|
||
/// threshold is deliberately conservative — four-in-a-row is almost
|
||
/// never organic.
|
||
fn has_consecutive_repetition(text: &str, min_run: usize) -> bool {
|
||
if min_run < 2 {
|
||
return false;
|
||
}
|
||
let mut run: usize = 1;
|
||
let mut last: Option<String> = None;
|
||
for token in text.split_whitespace() {
|
||
let token_lower = token.to_lowercase();
|
||
if last.as_deref() == Some(token_lower.as_str()) {
|
||
run += 1;
|
||
if run >= min_run {
|
||
return true;
|
||
}
|
||
} else {
|
||
run = 1;
|
||
last = Some(token_lower);
|
||
}
|
||
}
|
||
false
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn remove_fillers_strips_um_and_uh() {
|
||
let input = "So um I was thinking uh about this";
|
||
let result = remove_fillers(input);
|
||
assert!(!result.contains("um"));
|
||
assert!(!result.contains("uh"));
|
||
}
|
||
|
||
#[test]
|
||
fn remove_fillers_preserves_legitimate_words() {
|
||
let input = "The umbrella was actually useful";
|
||
let result = remove_fillers(input);
|
||
assert!(result.contains("umbrella"));
|
||
assert!(result.contains("useful"));
|
||
}
|
||
|
||
#[test]
|
||
fn to_british_english_converts_ize_to_ise() {
|
||
assert!(to_british_english("organize").contains("organise"));
|
||
assert!(to_british_english("realize").contains("realise"));
|
||
}
|
||
|
||
#[test]
|
||
fn to_british_english_preserves_case() {
|
||
let result = to_british_english("Organize the files");
|
||
assert!(result.starts_with("Organise"));
|
||
}
|
||
|
||
#[test]
|
||
fn to_british_english_handles_colour() {
|
||
assert!(to_british_english("the color is red").contains("colour"));
|
||
}
|
||
|
||
#[test]
|
||
fn collapse_repetitions_removes_consecutive_duplicate_words() {
|
||
assert_eq!(collapse_repetitions("I I can do that"), "I can do that");
|
||
assert_eq!(
|
||
collapse_repetitions("Think think that's that"),
|
||
"Think that's that"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn collapse_repetitions_removes_repeated_short_phrases() {
|
||
assert_eq!(
|
||
collapse_repetitions("I need I need to go to the shops"),
|
||
"I need to go to the shops"
|
||
);
|
||
assert_eq!(
|
||
collapse_repetitions("We should review we should review the draft"),
|
||
"We should review the draft"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn format_text_capitalises_after_full_stops() {
|
||
let result = format_text("hello world. this is a test");
|
||
assert!(result.starts_with('H'));
|
||
assert!(result.contains(". T"));
|
||
}
|
||
|
||
#[test]
|
||
fn format_text_handles_empty_string() {
|
||
assert_eq!(format_text(""), "");
|
||
}
|
||
|
||
#[test]
|
||
fn is_hallucination_detects_blank_audio() {
|
||
assert!(is_hallucination("[blank_audio]"));
|
||
assert!(is_hallucination(" [music] "));
|
||
}
|
||
|
||
#[test]
|
||
fn is_hallucination_detects_auto_thanks() {
|
||
assert!(is_hallucination("Thank you."));
|
||
assert!(is_hallucination("thanks."));
|
||
}
|
||
|
||
#[test]
|
||
fn is_hallucination_detects_subtitle_trailers() {
|
||
// WhisperLive #185 / ufal #121 class: subtitle-training leakage
|
||
// that fires on silence or room tone.
|
||
assert!(is_hallucination("Thanks for watching!"));
|
||
assert!(is_hallucination("Thanks for watching."));
|
||
assert!(is_hallucination("Please subscribe."));
|
||
assert!(is_hallucination("Don't forget to like and subscribe."));
|
||
assert!(is_hallucination("See you next time."));
|
||
assert!(is_hallucination("Subtitles by the Amara.org community"));
|
||
}
|
||
|
||
#[test]
|
||
fn is_hallucination_detects_music_and_sound_markers() {
|
||
assert!(is_hallucination("♪"));
|
||
assert!(is_hallucination("♪♪♪"));
|
||
assert!(is_hallucination("[applause]"));
|
||
assert!(is_hallucination("[Laughter]"));
|
||
assert!(is_hallucination("[Background noise]"));
|
||
}
|
||
|
||
#[test]
|
||
fn is_hallucination_detects_non_english_subtitle_leakage() {
|
||
// Japanese "thank you for watching"; MBC Korean news sign-off.
|
||
assert!(is_hallucination("ご視聴ありがとうございました"));
|
||
assert!(is_hallucination("MBC 뉴스 김수영입니다"));
|
||
}
|
||
|
||
#[test]
|
||
fn is_hallucination_allows_real_text() {
|
||
assert!(!is_hallucination("The meeting is at three o'clock."));
|
||
}
|
||
|
||
#[test]
|
||
fn is_hallucination_allows_dialogue_containing_thanks_mid_sentence() {
|
||
// Exact-match on trail phrases means legitimate dialogue that
|
||
// mentions "thanks" or "subscribe" is never dropped.
|
||
assert!(!is_hallucination(
|
||
"Thanks for the heads up on the migration"
|
||
));
|
||
assert!(!is_hallucination(
|
||
"Please subscribe to the RSS feed and tell me when it updates"
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn is_hallucination_detects_prompt_loop_repetition() {
|
||
// ufal #161: Whisper prompt-loop cascade, the classic
|
||
// streaming failure mode. Single-token runs only for now —
|
||
// multi-token phrase repetition ("thank you thank you thank
|
||
// you...") is a documented companion failure mode but needs
|
||
// sliding n-gram matching, which is a future enhancement.
|
||
assert!(is_hallucination("I I I I I I I I I"));
|
||
assert!(is_hallucination("hello hello hello hello world"));
|
||
assert!(is_hallucination("the the the the quick brown fox"));
|
||
// Case-insensitive.
|
||
assert!(is_hallucination("Hello HELLO hello hello"));
|
||
}
|
||
|
||
#[test]
|
||
fn is_hallucination_allows_natural_triple_repeats() {
|
||
// Threshold is 4, so natural speech patterns pass.
|
||
assert!(!is_hallucination("no no no, that's wrong"));
|
||
assert!(!is_hallucination("do do do the thing"));
|
||
// Alternating patterns never trigger regardless of length.
|
||
assert!(!is_hallucination("I am I am I am I am"));
|
||
}
|
||
}
|