agent: foundation — import legacy codebase from Obsidian vault

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/ai-formatting/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "kon-ai-formatting"
+version = "0.1.0"
+edition = "2021"
+description = "Text post-processing pipeline: filler removal, British English conversion, formatting for Kon"
+
+[dependencies]
+kon-core = { path = "../core" }
+regex-lite = "0.1"

crates/ai-formatting/src/lib.rs

@@ -0,0 +1,6 @@
+mod llm_client;
+pub mod pipeline;
+pub mod rule_based;
+
+pub use pipeline::{post_process_segments, FormatMode, PostProcessOptions};
+pub use rule_based::{format_text, is_hallucination, remove_fillers, to_british_english};

crates/ai-formatting/src/llm_client.rs

@@ -0,0 +1,5 @@
+//! Placeholder for future LLM sidecar integration (e.g., mistral.rs for smart formatting).
+//!
+//! When implemented, this module will expose a client that sends transcription
+//! segments to a local LLM for context-aware punctuation, paragraph splitting,
+//! and stylistic cleanup beyond what the rule-based pipeline can achieve.

crates/ai-formatting/src/pipeline.rs

@@ -0,0 +1,119 @@
+use kon_core::constants::SMART_PARAGRAPH_GAP_SECS;
+use kon_core::types::Segment;
+
+use crate::rule_based;
+
+/// Post-processing options for a transcription pipeline run.
+pub struct PostProcessOptions {
+    pub remove_fillers: bool,
+    pub british_english: bool,
+    pub anti_hallucination: bool,
+    pub format_mode: FormatMode,
+}
+
+/// How aggressively to format the transcript text.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FormatMode {
+    Raw,
+    Clean,
+    Smart,
+}
+
+impl FormatMode {
+    pub fn parse(s: &str) -> Self {
+        match s {
+            "Clean" => Self::Clean,
+            "Smart" => Self::Smart,
+            _ => Self::Raw,
+        }
+    }
+}
+
+/// Apply all post-processing steps to a list of segments.
+/// Modifies segments in place. Composed from individual pure functions.
+pub fn post_process_segments(segments: &mut Vec<Segment>, options: &PostProcessOptions) {
+    if options.anti_hallucination {
+        segments.retain(|seg| !rule_based::is_hallucination(&seg.text));
+    }
+
+    for seg in segments.iter_mut() {
+        if options.remove_fillers {
+            seg.text = rule_based::remove_fillers(&seg.text);
+        }
+        if options.british_english {
+            seg.text = rule_based::to_british_english(&seg.text);
+        }
+        if options.format_mode != FormatMode::Raw {
+            seg.text = rule_based::format_text(&seg.text);
+        }
+    }
+
+    if options.format_mode == FormatMode::Smart && segments.len() > 1 {
+        for i in (1..segments.len()).rev() {
+            let gap = segments[i].start - segments[i - 1].end;
+            if gap > SMART_PARAGRAPH_GAP_SECS {
+                segments[i].text = format!("\n\n{}", segments[i].text.trim_start());
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn make_segments() -> Vec<Segment> {
+        vec![
+            Segment {
+                start: 0.0,
+                end: 1.0,
+                text: "um hello world".into(),
+            },
+            Segment {
+                start: 1.0,
+                end: 2.0,
+                text: "[blank_audio]".into(),
+            },
+            Segment {
+                start: 5.0,
+                end: 6.0,
+                text: "organize the color scheme".into(),
+            },
+        ]
+    }
+
+    #[test]
+    fn post_process_applies_all_filters() {
+        let mut segments = make_segments();
+        let options = PostProcessOptions {
+            remove_fillers: true,
+            british_english: true,
+            anti_hallucination: true,
+            format_mode: FormatMode::Clean,
+        };
+
+        post_process_segments(&mut segments, &options);
+
+        assert_eq!(segments.len(), 2);
+        let lower0 = segments[0].text.to_lowercase();
+        let lower1 = segments[1].text.to_lowercase();
+        assert!(!lower0.contains("um"));
+        assert!(lower1.contains("organise"));
+        assert!(lower1.contains("colour"));
+    }
+
+    #[test]
+    fn post_process_adds_paragraph_breaks_on_long_pauses() {
+        let mut segments = make_segments();
+        let options = PostProcessOptions {
+            remove_fillers: false,
+            british_english: false,
+            anti_hallucination: false,
+            format_mode: FormatMode::Smart,
+        };
+
+        post_process_segments(&mut segments, &options);
+
+        assert!(segments[2].text.starts_with("\n\n"));
+    }
+}

crates/ai-formatting/src/rule_based.rs

@@ -0,0 +1,291 @@
+use std::sync::LazyLock;
+
+/// Compiled filler word regexes (built once, reused across calls).
+/// Uses \b word boundaries instead of lookbehinds (regex-lite does not
+/// support lookaround assertions).
+static FILLER_REGEXES: LazyLock<Vec<regex_lite::Regex>> = LazyLock::new(|| {
+    let fillers = [
+        "um",
+        "uh",
+        "er",
+        "ah",
+        "like",
+        "you know",
+        "sort of",
+        "kind of",
+        "I mean",
+        "basically",
+        "actually",
+        "literally",
+    ];
+    fillers
+        .iter()
+        .filter_map(|filler| {
+            let escaped = regex_lite::escape(filler);
+            let pattern = format!(r"(?i)\b{escaped}\b[,.]?\s*");
+            regex_lite::Regex::new(&pattern).ok()
+        })
+        .collect()
+});
+
+/// Remove common filler words from transcription text (case-insensitive).
+pub fn remove_fillers(text: &str) -> String {
+    let mut result = text.to_string();
+
+    for re in FILLER_REGEXES.iter() {
+        result = re.replace_all(&result, " ").to_string();
+    }
+
+    // Collapse runs of whitespace in a single pass.
+    let mut collapsed = String::with_capacity(result.len());
+    let mut prev_space = false;
+    for ch in result.chars() {
+        if ch == ' ' {
+            if !prev_space {
+                collapsed.push(' ');
+            }
+            prev_space = true;
+        } else {
+            prev_space = false;
+            collapsed.push(ch);
+        }
+    }
+
+    collapsed.trim().to_string()
+}
+
+/// Replacement pairs for American to British English conversion.
+///
+/// All entries are plain base words (no regex metacharacters). The
+/// `to_british_english` function wraps every entry with `\b` word
+/// boundaries and optional suffix matching automatically.
+static BRITISH_REPLACEMENTS: &[(&str, &str)] = &[
+    // -ize → -ise (and inflected forms)
+    ("organize", "organise"),
+    ("recognize", "recognise"),
+    ("realize", "realise"),
+    ("analyze", "analyse"),
+    ("apologize", "apologise"),
+    ("authorize", "authorise"),
+    ("categorize", "categorise"),
+    ("characterize", "characterise"),
+    ("customize", "customise"),
+    ("digitize", "digitise"),
+    ("emphasize", "emphasise"),
+    ("finalize", "finalise"),
+    ("generalize", "generalise"),
+    ("harmonize", "harmonise"),
+    ("initialize", "initialise"),
+    ("maximize", "maximise"),
+    ("minimize", "minimise"),
+    ("modernize", "modernise"),
+    ("normalize", "normalise"),
+    ("optimize", "optimise"),
+    ("prioritize", "prioritise"),
+    ("revolutionize", "revolutionise"),
+    ("specialize", "specialise"),
+    ("standardize", "standardise"),
+    ("summarize", "summarise"),
+    ("utilize", "utilise"),
+    // -or → -our
+    ("color", "colour"),
+    ("favor", "favour"),
+    ("honor", "honour"),
+    ("humor", "humour"),
+    ("labor", "labour"),
+    ("neighbor", "neighbour"),
+    ("behavior", "behaviour"),
+    // -er → -re
+    ("center", "centre"),
+    ("fiber", "fibre"),
+    ("liter", "litre"),
+    ("meter", "metre"),
+    ("theater", "theatre"),
+    // -ense → -ence
+    ("defense", "defence"),
+    ("offense", "offence"),
+    // Other
+    ("catalog", "catalogue"),
+    ("dialog", "dialogue"),
+];
+
+/// Convert American English spelling to British English (word-boundary aware).
+pub fn to_british_english(text: &str) -> String {
+    let mut result = text.to_string();
+
+    for (us, uk) in BRITISH_REPLACEMENTS {
+        // Every entry in BRITISH_REPLACEMENTS is a plain ASCII base word.
+        // We wrap it with \b boundaries and optional suffix matching here.
+        let pattern = format!("(?i)\\b{}(?:d|s|r|rs)?\\b", regex_lite::escape(us));
+
+        if let Ok(re) = regex_lite::Regex::new(&pattern) {
+            result = re
+                .replace_all(&result, |caps: &regex_lite::Captures| {
+                    let Some(m) = caps.get(0) else {
+                        return String::new();
+                    };
+                    let matched = m.as_str();
+                    let base_len = us.len();
+                    // SAFETY: byte indexing is correct here because both
+                    // the US base word and the suffix characters (d, s, r)
+                    // are guaranteed ASCII by the BRITISH_REPLACEMENTS table.
+                    debug_assert!(us.is_ascii(), "BRITISH_REPLACEMENTS entries must be ASCII");
+                    debug_assert!(matched.is_ascii(), "matched text expected to be ASCII");
+                    let suffix = if matched.len() > base_len {
+                        &matched[base_len..]
+                    } else {
+                        ""
+                    };
+                    let Some(first_char) = matched.chars().next() else {
+                        return String::new();
+                    };
+                    if first_char.is_uppercase() {
+                        let mut chars = uk.chars();
+                        let Some(first_uk) = chars.next() else {
+                            return String::new();
+                        };
+                        let upper_first: String = first_uk.to_uppercase().collect();
+                        format!("{}{}{}", upper_first, chars.collect::<String>(), suffix)
+                    } else {
+                        format!("{uk}{suffix}")
+                    }
+                })
+                .to_string();
+        }
+    }
+
+    result
+}
+
+/// Basic formatting: capitalise sentences, fix spacing, clean punctuation.
+pub fn format_text(text: &str) -> String {
+    if text.is_empty() {
+        return String::new();
+    }
+
+    let mut result = String::with_capacity(text.len());
+    let mut capitalise_next = true;
+
+    let chars: Vec<char> = text.chars().collect();
+    let mut i = 0;
+
+    while i < chars.len() {
+        let c = chars[i];
+
+        if c == ' ' && i + 1 < chars.len() && chars[i + 1] == ' ' {
+            i += 1;
+            continue;
+        }
+
+        if capitalise_next && c.is_alphabetic() {
+            result.extend(c.to_uppercase());
+            capitalise_next = false;
+        } else {
+            result.push(c);
+        }
+
+        if c == '.' || c == '!' || c == '?' {
+            capitalise_next = true;
+        }
+        if c == '\n' {
+            capitalise_next = true;
+        }
+
+        i += 1;
+    }
+
+    result
+}
+
+/// Known hallucination markers that should be filtered from transcriptions.
+static HALLUCINATION_MARKERS: &[&str] = &["[blank_audio]", "[music]", "[silence]"];
+
+static AUTO_THANKS_PHRASES: &[&str] = &["thank you.", "thanks.", "you.", "thank you for watching."];
+
+/// Returns true if a segment's text looks like a hallucination.
+pub fn is_hallucination(text: &str) -> bool {
+    let trimmed = text.trim().to_lowercase();
+    if trimmed.is_empty() {
+        return true;
+    }
+    for marker in HALLUCINATION_MARKERS {
+        if trimmed.contains(marker) {
+            return true;
+        }
+    }
+    if trimmed.len() < 15 {
+        for phrase in AUTO_THANKS_PHRASES {
+            if trimmed == *phrase {
+                return true;
+            }
+        }
+    }
+    false
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn remove_fillers_strips_um_and_uh() {
+        let input = "So um I was thinking uh about this";
+        let result = remove_fillers(input);
+        assert!(!result.contains("um"));
+        assert!(!result.contains("uh"));
+    }
+
+    #[test]
+    fn remove_fillers_preserves_legitimate_words() {
+        let input = "The umbrella was actually useful";
+        let result = remove_fillers(input);
+        assert!(result.contains("umbrella"));
+        assert!(result.contains("useful"));
+    }
+
+    #[test]
+    fn to_british_english_converts_ize_to_ise() {
+        assert!(to_british_english("organize").contains("organise"));
+        assert!(to_british_english("realize").contains("realise"));
+    }
+
+    #[test]
+    fn to_british_english_preserves_case() {
+        let result = to_british_english("Organize the files");
+        assert!(result.starts_with("Organise"));
+    }
+
+    #[test]
+    fn to_british_english_handles_colour() {
+        assert!(to_british_english("the color is red").contains("colour"));
+    }
+
+    #[test]
+    fn format_text_capitalises_after_full_stops() {
+        let result = format_text("hello world. this is a test");
+        assert!(result.starts_with('H'));
+        assert!(result.contains(". T"));
+    }
+
+    #[test]
+    fn format_text_handles_empty_string() {
+        assert_eq!(format_text(""), "");
+    }
+
+    #[test]
+    fn is_hallucination_detects_blank_audio() {
+        assert!(is_hallucination("[blank_audio]"));
+        assert!(is_hallucination("  [music]  "));
+    }
+
+    #[test]
+    fn is_hallucination_detects_auto_thanks() {
+        assert!(is_hallucination("Thank you."));
+        assert!(is_hallucination("thanks."));
+    }
+
+    #[test]
+    fn is_hallucination_allows_real_text() {
+        assert!(!is_hallucination("The meeting is at three o'clock."));
+    }
+}