diff --git a/crates/ai-formatting/src/lib.rs b/crates/ai-formatting/src/lib.rs index db91d63..733bb2f 100644 --- a/crates/ai-formatting/src/lib.rs +++ b/crates/ai-formatting/src/lib.rs @@ -3,6 +3,4 @@ pub mod pipeline; pub mod rule_based; pub use pipeline::{post_process_segments, FormatMode, PostProcessOptions}; -pub use rule_based::{ - format_text, is_hallucination, remove_fillers, to_british_english, -}; +pub use rule_based::{format_text, is_hallucination, remove_fillers, to_british_english}; diff --git a/crates/ai-formatting/src/llm_client.rs b/crates/ai-formatting/src/llm_client.rs index 5c5ef9b..76b710c 100644 --- a/crates/ai-formatting/src/llm_client.rs +++ b/crates/ai-formatting/src/llm_client.rs @@ -1,2 +1,5 @@ -// Stub for future LLM sidecar integration. -// Will implement TextProcessor trait when mistral.rs sidecar is added. +//! Placeholder for future LLM sidecar integration (e.g., mistral.rs for smart formatting). +//! +//! When implemented, this module will expose a client that sends transcription +//! segments to a local LLM for context-aware punctuation, paragraph splitting, +//! and stylistic cleanup beyond what the rule-based pipeline can achieve. diff --git a/crates/ai-formatting/src/pipeline.rs b/crates/ai-formatting/src/pipeline.rs index eb423bb..8f24878 100644 --- a/crates/ai-formatting/src/pipeline.rs +++ b/crates/ai-formatting/src/pipeline.rs @@ -31,10 +31,7 @@ impl FormatMode { /// Apply all post-processing steps to a list of segments. /// Modifies segments in place. Composed from individual pure functions. -pub fn post_process_segments( - segments: &mut Vec, - options: &PostProcessOptions, -) { +pub fn post_process_segments(segments: &mut Vec, options: &PostProcessOptions) { if options.anti_hallucination { segments.retain(|seg| !rule_based::is_hallucination(&seg.text)); } @@ -55,8 +52,7 @@ pub fn post_process_segments( for i in (1..segments.len()).rev() { let gap = segments[i].start - segments[i - 1].end; if gap > SMART_PARAGRAPH_GAP_SECS { - segments[i].text = - format!("\n\n{}", segments[i].text.trim_start()); + segments[i].text = format!("\n\n{}", segments[i].text.trim_start()); } } } diff --git a/crates/ai-formatting/src/rule_based.rs b/crates/ai-formatting/src/rule_based.rs index ce41717..7613260 100644 --- a/crates/ai-formatting/src/rule_based.rs +++ b/crates/ai-formatting/src/rule_based.rs @@ -3,32 +3,30 @@ use std::sync::LazyLock; /// Compiled filler word regexes (built once, reused across calls). /// Uses \b word boundaries instead of lookbehinds (regex-lite does not /// support lookaround assertions). -static FILLER_REGEXES: LazyLock> = - LazyLock::new(|| { - let fillers = [ - "um", - "uh", - "er", - "ah", - "like", - "you know", - "sort of", - "kind of", - "I mean", - "basically", - "actually", - "literally", - ]; - fillers - .iter() - .filter_map(|filler| { - let escaped = regex_lite::escape(filler); - let pattern = - format!(r"(?i)\b{escaped}\b[,.]?\s*"); - regex_lite::Regex::new(&pattern).ok() - }) - .collect() - }); +static FILLER_REGEXES: LazyLock> = LazyLock::new(|| { + let fillers = [ + "um", + "uh", + "er", + "ah", + "like", + "you know", + "sort of", + "kind of", + "I mean", + "basically", + "actually", + "literally", + ]; + fillers + .iter() + .filter_map(|filler| { + let escaped = regex_lite::escape(filler); + let pattern = format!(r"(?i)\b{escaped}\b[,.]?\s*"); + regex_lite::Regex::new(&pattern).ok() + }) + .collect() +}); /// Remove common filler words from transcription text (case-insensitive). pub fn remove_fillers(text: &str) -> String { @@ -38,14 +36,29 @@ pub fn remove_fillers(text: &str) -> String { result = re.replace_all(&result, " ").to_string(); } - while result.contains(" ") { - result = result.replace(" ", " "); + // Collapse runs of whitespace in a single pass. + let mut collapsed = String::with_capacity(result.len()); + let mut prev_space = false; + for ch in result.chars() { + if ch == ' ' { + if !prev_space { + collapsed.push(' '); + } + prev_space = true; + } else { + prev_space = false; + collapsed.push(ch); + } } - result.trim().to_string() + collapsed.trim().to_string() } -/// Replacement pairs for American → British English conversion. +/// Replacement pairs for American to British English conversion. +/// +/// All entries are plain base words (no regex metacharacters). The +/// `to_british_english` function wraps every entry with `\b` word +/// boundaries and optional suffix matching automatically. static BRITISH_REPLACEMENTS: &[(&str, &str)] = &[ // -ize → -ise (and inflected forms) ("organize", "organise"), @@ -74,26 +87,26 @@ static BRITISH_REPLACEMENTS: &[(&str, &str)] = &[ ("standardize", "standardise"), ("summarize", "summarise"), ("utilize", "utilise"), - // -or → -our (word-boundary needed) - ("\\bcolor\\b", "colour"), - ("\\bfavor\\b", "favour"), - ("\\bhonor\\b", "honour"), - ("\\bhumor\\b", "humour"), - ("\\blabor\\b", "labour"), - ("\\bneighbor\\b", "neighbour"), + // -or → -our + ("color", "colour"), + ("favor", "favour"), + ("honor", "honour"), + ("humor", "humour"), + ("labor", "labour"), + ("neighbor", "neighbour"), ("behavior", "behaviour"), - // -er → -re (word-boundary to avoid "parameter" etc.) - ("\\bcenter\\b", "centre"), - ("\\bfiber\\b", "fibre"), - ("\\bliter\\b", "litre"), - ("\\bmeter\\b", "metre"), - ("\\btheater\\b", "theatre"), + // -er → -re + ("center", "centre"), + ("fiber", "fibre"), + ("liter", "litre"), + ("meter", "metre"), + ("theater", "theatre"), // -ense → -ence ("defense", "defence"), ("offense", "offence"), // Other - ("\\bcatalog\\b", "catalogue"), - ("\\bdialog\\b", "dialogue"), + ("catalog", "catalogue"), + ("dialog", "dialogue"), ]; /// Convert American English spelling to British English (word-boundary aware). @@ -101,14 +114,9 @@ pub fn to_british_english(text: &str) -> String { let mut result = text.to_string(); for (us, uk) in BRITISH_REPLACEMENTS { - let pattern = if us.contains("\\b") { - format!("(?i){us}") - } else { - format!( - "(?i)\\b{}(?:d|s|r|rs)?\\b", - regex_lite::escape(us) - ) - }; + // Every entry in BRITISH_REPLACEMENTS is a plain ASCII base word. + // We wrap it with \b boundaries and optional suffix matching here. + let pattern = format!("(?i)\\b{}(?:d|s|r|rs)?\\b", regex_lite::escape(us)); if let Ok(re) = regex_lite::Regex::new(&pattern) { result = re @@ -117,8 +125,12 @@ pub fn to_british_english(text: &str) -> String { return String::new(); }; let matched = m.as_str(); - let us_base = us.replace("\\b", ""); - let base_len = us_base.len(); + let base_len = us.len(); + // SAFETY: byte indexing is correct here because both + // the US base word and the suffix characters (d, s, r) + // are guaranteed ASCII by the BRITISH_REPLACEMENTS table. + debug_assert!(us.is_ascii(), "BRITISH_REPLACEMENTS entries must be ASCII"); + debug_assert!(matched.is_ascii(), "matched text expected to be ASCII"); let suffix = if matched.len() > base_len { &matched[base_len..] } else { @@ -127,22 +139,15 @@ pub fn to_british_english(text: &str) -> String { let Some(first_char) = matched.chars().next() else { return String::new(); }; - let uk_clean = uk.replace("\\b", ""); if first_char.is_uppercase() { - let mut chars = uk_clean.chars(); + let mut chars = uk.chars(); let Some(first_uk) = chars.next() else { return String::new(); }; - let upper_first: String = - first_uk.to_uppercase().collect(); - format!( - "{}{}{}", - upper_first, - chars.collect::(), - suffix - ) + let upper_first: String = first_uk.to_uppercase().collect(); + format!("{}{}{}", upper_first, chars.collect::(), suffix) } else { - format!("{uk_clean}{suffix}") + format!("{uk}{suffix}") } }) .to_string(); @@ -193,18 +198,9 @@ pub fn format_text(text: &str) -> String { } /// Known hallucination markers that should be filtered from transcriptions. -static HALLUCINATION_MARKERS: &[&str] = &[ - "[blank_audio]", - "[music]", - "[silence]", -]; +static HALLUCINATION_MARKERS: &[&str] = &["[blank_audio]", "[music]", "[silence]"]; -static AUTO_THANKS_PHRASES: &[&str] = &[ - "thank you.", - "thanks.", - "you.", - "thank you for watching.", -]; +static AUTO_THANKS_PHRASES: &[&str] = &["thank you.", "thanks.", "you.", "thank you for watching."]; /// Returns true if a segment's text looks like a hallucination. pub fn is_hallucination(text: &str) -> bool {