feat(kon): add ai-formatting crate — filler removal, British English, text pipeline
- Filler word removal using word-boundary regex (fixed regex-lite lookbehind limitation) - British English conversion: 26 -ize/-ise patterns, -or/-our, -er/-re, -ense/-ence - Text formatting: sentence capitalisation, spacing cleanup - Hallucination filter: blank_audio, music, silence, auto-thanks detection - Post-processing pipeline: composed from pure functions, supports Raw/Clean/Smart modes - Smart mode inserts paragraph breaks on >2s pauses between segments - 12 tests passing, clippy clean Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
123
crates/ai-formatting/src/pipeline.rs
Normal file
123
crates/ai-formatting/src/pipeline.rs
Normal file
@@ -0,0 +1,123 @@
|
||||
use kon_core::constants::SMART_PARAGRAPH_GAP_SECS;
|
||||
use kon_core::types::Segment;
|
||||
|
||||
use crate::rule_based;
|
||||
|
||||
/// Post-processing options for a transcription pipeline run.
|
||||
pub struct PostProcessOptions {
|
||||
pub remove_fillers: bool,
|
||||
pub british_english: bool,
|
||||
pub anti_hallucination: bool,
|
||||
pub format_mode: FormatMode,
|
||||
}
|
||||
|
||||
/// How aggressively to format the transcript text.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum FormatMode {
|
||||
Raw,
|
||||
Clean,
|
||||
Smart,
|
||||
}
|
||||
|
||||
impl FormatMode {
|
||||
pub fn parse(s: &str) -> Self {
|
||||
match s {
|
||||
"Clean" => Self::Clean,
|
||||
"Smart" => Self::Smart,
|
||||
_ => Self::Raw,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply all post-processing steps to a list of segments.
|
||||
/// Modifies segments in place. Composed from individual pure functions.
|
||||
pub fn post_process_segments(
|
||||
segments: &mut Vec<Segment>,
|
||||
options: &PostProcessOptions,
|
||||
) {
|
||||
if options.anti_hallucination {
|
||||
segments.retain(|seg| !rule_based::is_hallucination(&seg.text));
|
||||
}
|
||||
|
||||
for seg in segments.iter_mut() {
|
||||
if options.remove_fillers {
|
||||
seg.text = rule_based::remove_fillers(&seg.text);
|
||||
}
|
||||
if options.british_english {
|
||||
seg.text = rule_based::to_british_english(&seg.text);
|
||||
}
|
||||
if options.format_mode != FormatMode::Raw {
|
||||
seg.text = rule_based::format_text(&seg.text);
|
||||
}
|
||||
}
|
||||
|
||||
if options.format_mode == FormatMode::Smart && segments.len() > 1 {
|
||||
for i in (1..segments.len()).rev() {
|
||||
let gap = segments[i].start - segments[i - 1].end;
|
||||
if gap > SMART_PARAGRAPH_GAP_SECS {
|
||||
segments[i].text =
|
||||
format!("\n\n{}", segments[i].text.trim_start());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_segments() -> Vec<Segment> {
|
||||
vec![
|
||||
Segment {
|
||||
start: 0.0,
|
||||
end: 1.0,
|
||||
text: "um hello world".into(),
|
||||
},
|
||||
Segment {
|
||||
start: 1.0,
|
||||
end: 2.0,
|
||||
text: "[blank_audio]".into(),
|
||||
},
|
||||
Segment {
|
||||
start: 5.0,
|
||||
end: 6.0,
|
||||
text: "organize the color scheme".into(),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn post_process_applies_all_filters() {
|
||||
let mut segments = make_segments();
|
||||
let options = PostProcessOptions {
|
||||
remove_fillers: true,
|
||||
british_english: true,
|
||||
anti_hallucination: true,
|
||||
format_mode: FormatMode::Clean,
|
||||
};
|
||||
|
||||
post_process_segments(&mut segments, &options);
|
||||
|
||||
assert_eq!(segments.len(), 2);
|
||||
let lower0 = segments[0].text.to_lowercase();
|
||||
let lower1 = segments[1].text.to_lowercase();
|
||||
assert!(!lower0.contains("um"));
|
||||
assert!(lower1.contains("organise"));
|
||||
assert!(lower1.contains("colour"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn post_process_adds_paragraph_breaks_on_long_pauses() {
|
||||
let mut segments = make_segments();
|
||||
let options = PostProcessOptions {
|
||||
remove_fillers: false,
|
||||
british_english: false,
|
||||
anti_hallucination: false,
|
||||
format_mode: FormatMode::Smart,
|
||||
};
|
||||
|
||||
post_process_segments(&mut segments, &options);
|
||||
|
||||
assert!(segments[2].text.starts_with("\n\n"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user