feat(kon): add ai-formatting crate — filler removal, British English, text pipeline

- Filler word removal using word-boundary regex (fixed regex-lite lookbehind limitation)
- British English conversion: 26 -ize/-ise patterns, -or/-our, -er/-re, -ense/-ence
- Text formatting: sentence capitalisation, spacing cleanup
- Hallucination filter: blank_audio, music, silence, auto-thanks detection
- Post-processing pipeline: composed from pure functions, supports Raw/Clean/Smart modes
- Smart mode inserts paragraph breaks on >2s pauses between segments
- 12 tests passing, clippy clean

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jake
2026-03-16 20:29:46 +00:00
parent 6588130e36
commit 0738fca22c
5 changed files with 421 additions and 2 deletions

View File

@@ -0,0 +1,123 @@
use kon_core::constants::SMART_PARAGRAPH_GAP_SECS;
use kon_core::types::Segment;
use crate::rule_based;
/// Post-processing options for a transcription pipeline run.
pub struct PostProcessOptions {
pub remove_fillers: bool,
pub british_english: bool,
pub anti_hallucination: bool,
pub format_mode: FormatMode,
}
/// How aggressively to format the transcript text.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FormatMode {
Raw,
Clean,
Smart,
}
impl FormatMode {
pub fn parse(s: &str) -> Self {
match s {
"Clean" => Self::Clean,
"Smart" => Self::Smart,
_ => Self::Raw,
}
}
}
/// Apply all post-processing steps to a list of segments.
/// Modifies segments in place. Composed from individual pure functions.
pub fn post_process_segments(
segments: &mut Vec<Segment>,
options: &PostProcessOptions,
) {
if options.anti_hallucination {
segments.retain(|seg| !rule_based::is_hallucination(&seg.text));
}
for seg in segments.iter_mut() {
if options.remove_fillers {
seg.text = rule_based::remove_fillers(&seg.text);
}
if options.british_english {
seg.text = rule_based::to_british_english(&seg.text);
}
if options.format_mode != FormatMode::Raw {
seg.text = rule_based::format_text(&seg.text);
}
}
if options.format_mode == FormatMode::Smart && segments.len() > 1 {
for i in (1..segments.len()).rev() {
let gap = segments[i].start - segments[i - 1].end;
if gap > SMART_PARAGRAPH_GAP_SECS {
segments[i].text =
format!("\n\n{}", segments[i].text.trim_start());
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_segments() -> Vec<Segment> {
vec![
Segment {
start: 0.0,
end: 1.0,
text: "um hello world".into(),
},
Segment {
start: 1.0,
end: 2.0,
text: "[blank_audio]".into(),
},
Segment {
start: 5.0,
end: 6.0,
text: "organize the color scheme".into(),
},
]
}
#[test]
fn post_process_applies_all_filters() {
let mut segments = make_segments();
let options = PostProcessOptions {
remove_fillers: true,
british_english: true,
anti_hallucination: true,
format_mode: FormatMode::Clean,
};
post_process_segments(&mut segments, &options);
assert_eq!(segments.len(), 2);
let lower0 = segments[0].text.to_lowercase();
let lower1 = segments[1].text.to_lowercase();
assert!(!lower0.contains("um"));
assert!(lower1.contains("organise"));
assert!(lower1.contains("colour"));
}
#[test]
fn post_process_adds_paragraph_breaks_on_long_pauses() {
let mut segments = make_segments();
let options = PostProcessOptions {
remove_fillers: false,
british_english: false,
anti_hallucination: false,
format_mode: FormatMode::Smart,
};
post_process_segments(&mut segments, &options);
assert!(segments[2].text.starts_with("\n\n"));
}
}