jake/Lumotia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(kon): normalise British English table, single-pass whitespace collapse, byte-index safety

Browse Source 
- Normalise BRITISH_REPLACEMENTS: remove baked-in \b from entries so all
  entries are plain base words; the function adds boundaries uniformly
- Replace O(n*m) while-loop double-space removal with single-pass collapse
- Add debug_assert! documenting ASCII assumption for byte-indexed suffix slicing
- Expand llm_client.rs module-level doc comment
- Run cargo fmt on ai-formatting crate

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jake
2026-03-17 00:19:59 +00:00
parent 95c8d490c3
commit 2ac98e6d40
4 changed files with 81 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/ai-formatting/src/lib.rs
View File

@@ -3,6 +3,4 @@ pub mod pipeline;
pub mod rule_based;
pub use pipeline::{post_process_segments, FormatMode, PostProcessOptions};
pub use rule_based::{
format_text, is_hallucination, remove_fillers, to_british_english,
};
pub use rule_based::{format_text, is_hallucination, remove_fillers, to_british_english};

7
crates/ai-formatting/src/llm_client.rs
View File

@@ -1,2 +1,5 @@
// Stub for future LLM sidecar integration.
// Will implement TextProcessor trait when mistral.rs sidecar is added.
//! Placeholder for future LLM sidecar integration (e.g., mistral.rs for smart formatting).
//!
//! When implemented, this module will expose a client that sends transcription
//! segments to a local LLM for context-aware punctuation, paragraph splitting,
//! and stylistic cleanup beyond what the rule-based pipeline can achieve.

8
crates/ai-formatting/src/pipeline.rs
View File

@@ -31,10 +31,7 @@ impl FormatMode {
/// Apply all post-processing steps to a list of segments.
/// Modifies segments in place. Composed from individual pure functions.
pub fn post_process_segments(
segments: &mut Vec<Segment>,
options: &PostProcessOptions,
) {
pub fn post_process_segments(segments: &mut Vec<Segment>, options: &PostProcessOptions) {
if options.anti_hallucination {
segments.retain(|seg| !rule_based::is_hallucination(&seg.text));
}
@@ -55,8 +52,7 @@ pub fn post_process_segments(
for i in (1..segments.len()).rev() {
let gap = segments[i].start - segments[i - 1].end;
if gap > SMART_PARAGRAPH_GAP_SECS {
segments[i].text =
format!("\n\n{}", segments[i].text.trim_start());
segments[i].text = format!("\n\n{}", segments[i].text.trim_start());
}
}
}

108
crates/ai-formatting/src/rule_based.rs
View File

@@ -3,8 +3,7 @@ use std::sync::LazyLock;
/// Compiled filler word regexes (built once, reused across calls).
/// Uses \b word boundaries instead of lookbehinds (regex-lite does not
/// support lookaround assertions).
static FILLER_REGEXES: LazyLock<Vec<regex_lite::Regex>> =
LazyLock::new(|| {
static FILLER_REGEXES: LazyLock<Vec<regex_lite::Regex>> = LazyLock::new(|| {
let fillers = [
"um",
"uh",
@@ -23,12 +22,11 @@ static FILLER_REGEXES: LazyLock<Vec<regex_lite::Regex>> =
.iter()
.filter_map(|filler| {
let escaped = regex_lite::escape(filler);
let pattern =
format!(r"(?i)\b{escaped}\b[,.]?\s*");
let pattern = format!(r"(?i)\b{escaped}\b[,.]?\s*");
regex_lite::Regex::new(&pattern).ok()
})
.collect()
});
});
/// Remove common filler words from transcription text (case-insensitive).
pub fn remove_fillers(text: &str) -> String {
@@ -38,14 +36,29 @@ pub fn remove_fillers(text: &str) -> String {
result = re.replace_all(&result, " ").to_string();
}
while result.contains(" ") {
result = result.replace(" ", " ");
// Collapse runs of whitespace in a single pass.
let mut collapsed = String::with_capacity(result.len());
let mut prev_space = false;
for ch in result.chars() {
if ch == ' ' {
if !prev_space {
collapsed.push(' ');
}
prev_space = true;
} else {
prev_space = false;
collapsed.push(ch);
}
}
result.trim().to_string()
collapsed.trim().to_string()
}
/// Replacement pairs for American → British English conversion.
/// Replacement pairs for American to British English conversion.
///
/// All entries are plain base words (no regex metacharacters). The
/// `to_british_english` function wraps every entry with `\b` word
/// boundaries and optional suffix matching automatically.
static BRITISH_REPLACEMENTS: &[(&str, &str)] = &[
// -ize → -ise (and inflected forms)
("organize", "organise"),
@@ -74,26 +87,26 @@ static BRITISH_REPLACEMENTS: &[(&str, &str)] = &[
("standardize", "standardise"),
("summarize", "summarise"),
("utilize", "utilise"),
// -or → -our (word-boundary needed)
("\\bcolor\\b", "colour"),
("\\bfavor\\b", "favour"),
("\\bhonor\\b", "honour"),
("\\bhumor\\b", "humour"),
("\\blabor\\b", "labour"),
("\\bneighbor\\b", "neighbour"),
// -or → -our
("color", "colour"),
("favor", "favour"),
("honor", "honour"),
("humor", "humour"),
("labor", "labour"),
("neighbor", "neighbour"),
("behavior", "behaviour"),
// -er → -re (word-boundary to avoid "parameter" etc.)
("\\bcenter\\b", "centre"),
("\\bfiber\\b", "fibre"),
("\\bliter\\b", "litre"),
("\\bmeter\\b", "metre"),
("\\btheater\\b", "theatre"),
// -er → -re
("center", "centre"),
("fiber", "fibre"),
("liter", "litre"),
("meter", "metre"),
("theater", "theatre"),
// -ense → -ence
("defense", "defence"),
("offense", "offence"),
// Other
("\\bcatalog\\b", "catalogue"),
("\\bdialog\\b", "dialogue"),
("catalog", "catalogue"),
("dialog", "dialogue"),
];
/// Convert American English spelling to British English (word-boundary aware).
@@ -101,14 +114,9 @@ pub fn to_british_english(text: &str) -> String {
let mut result = text.to_string();
for (us, uk) in BRITISH_REPLACEMENTS {
let pattern = if us.contains("\\b") {
format!("(?i){us}")
} else {
format!(
"(?i)\\b{}(?:d|s|r|rs)?\\b",
regex_lite::escape(us)
)
};
// Every entry in BRITISH_REPLACEMENTS is a plain ASCII base word.
// We wrap it with \b boundaries and optional suffix matching here.
let pattern = format!("(?i)\\b{}(?:d|s|r|rs)?\\b", regex_lite::escape(us));
if let Ok(re) = regex_lite::Regex::new(&pattern) {
result = re
@@ -117,8 +125,12 @@ pub fn to_british_english(text: &str) -> String {
return String::new();
};
let matched = m.as_str();
let us_base = us.replace("\\b", "");
let base_len = us_base.len();
let base_len = us.len();
// SAFETY: byte indexing is correct here because both
// the US base word and the suffix characters (d, s, r)
// are guaranteed ASCII by the BRITISH_REPLACEMENTS table.
debug_assert!(us.is_ascii(), "BRITISH_REPLACEMENTS entries must be ASCII");
debug_assert!(matched.is_ascii(), "matched text expected to be ASCII");
let suffix = if matched.len() > base_len {
&matched[base_len..]
} else {
@@ -127,22 +139,15 @@ pub fn to_british_english(text: &str) -> String {
let Some(first_char) = matched.chars().next() else {
return String::new();
};
let uk_clean = uk.replace("\\b", "");
if first_char.is_uppercase() {
let mut chars = uk_clean.chars();
let mut chars = uk.chars();
let Some(first_uk) = chars.next() else {
return String::new();
};
let upper_first: String =
first_uk.to_uppercase().collect();
format!(
"{}{}{}",
upper_first,
chars.collect::<String>(),
suffix
)
let upper_first: String = first_uk.to_uppercase().collect();
format!("{}{}{}", upper_first, chars.collect::<String>(), suffix)
} else {
format!("{uk_clean}{suffix}")
format!("{uk}{suffix}")
}
})
.to_string();
@@ -193,18 +198,9 @@ pub fn format_text(text: &str) -> String {
}
/// Known hallucination markers that should be filtered from transcriptions.
static HALLUCINATION_MARKERS: &[&str] = &[
"[blank_audio]",
"[music]",
"[silence]",
];
static HALLUCINATION_MARKERS: &[&str] = &["[blank_audio]", "[music]", "[silence]"];
static AUTO_THANKS_PHRASES: &[&str] = &[
"thank you.",
"thanks.",
"you.",
"thank you for watching.",
];
static AUTO_THANKS_PHRASES: &[&str] = &["thank you.", "thanks.", "you.", "thank you for watching."];
/// Returns true if a segment's text looks like a hallucination.
pub fn is_hallucination(text: &str) -> bool {