diff --git a/crates/ai-formatting/src/to_plain_text.rs b/crates/ai-formatting/src/to_plain_text.rs index ce31d30..53e9f7d 100644 --- a/crates/ai-formatting/src/to_plain_text.rs +++ b/crates/ai-formatting/src/to_plain_text.rs @@ -41,12 +41,28 @@ pub fn to_plain_text(segments: &[Segment]) -> String { normalise_whitespace(&joined).trim().to_string() } -/// Collapse any run of unicode whitespace into a single ASCII space. +/// Collapse any run of unicode whitespace into a single ASCII space, +/// and strip zero-width format characters entirely. +/// +/// Zero-width chars (U+200B/C/D, U+2060, U+FEFF) are handled as a +/// separate class from whitespace: `char::is_whitespace()` returns +/// false for them, so the standard whitespace pass would let them +/// through to the LLM where they waste tokens without contributing +/// any natural-language content. Treating them as "strip entirely" +/// rather than "collapse to a space" avoids silently inserting word +/// breaks where the source had none. +/// /// Kept private; the module's contract is `to_plain_text`. fn normalise_whitespace(s: &str) -> String { let mut out = String::with_capacity(s.len()); let mut prev_was_space = false; for ch in s.chars() { + if is_zero_width_format(ch) { + // Strip without emitting anything. prev_was_space unchanged + // so a space on either side of a zero-width char still + // collapses correctly. + continue; + } if ch.is_whitespace() { if !prev_was_space { out.push(' '); @@ -60,6 +76,20 @@ fn normalise_whitespace(s: &str) -> String { out } +/// Zero-width format characters the transcription pipeline should +/// never feed to an LLM. Sourced from common "invisible" codepoints: +/// - U+200B ZERO WIDTH SPACE +/// - U+200C ZERO WIDTH NON-JOINER +/// - U+200D ZERO WIDTH JOINER +/// - U+2060 WORD JOINER +/// - U+FEFF ZERO WIDTH NO-BREAK SPACE (also BOM) +fn is_zero_width_format(ch: char) -> bool { + matches!( + ch, + '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{2060}' | '\u{FEFF}' + ) +} + #[cfg(test)] mod tests { use super::*; @@ -123,6 +153,41 @@ mod tests { assert_eq!(out, "hello world"); } + #[test] + fn zero_width_format_chars_strip_entirely() { + // char::is_whitespace returns false for all of these, so the + // default whitespace pass would let them through. They carry + // no natural-language content — stripping them saves LLM + // tokens without changing meaning. + let cases = [ + ("hello\u{200B}world", "helloworld"), // ZERO WIDTH SPACE + ("hello\u{200C}world", "helloworld"), // ZWNJ + ("hello\u{200D}world", "helloworld"), // ZWJ + ("hello\u{2060}world", "helloworld"), // WORD JOINER + ("hello\u{FEFF}world", "helloworld"), // ZWNBSP / BOM + ]; + for (input, expected) in cases { + let out = to_plain_text(&[seg(input)]); + assert_eq!(out, expected, "input {input:?} should strip to {expected:?}"); + } + } + + #[test] + fn zero_width_chars_do_not_break_adjacent_whitespace_collapsing() { + // "hello \u{FEFF} world" — the zero-width char between two + // spaces should strip, leaving a single collapsed space. + let out = to_plain_text(&[seg("hello \u{FEFF} world")]); + assert_eq!(out, "hello world"); + } + + #[test] + fn leading_bom_is_stripped() { + // BOM at start of segment — common artifact when Whisper + // consumes a file whose encoding pass inserted one. + let out = to_plain_text(&[seg("\u{FEFF}hello world")]); + assert_eq!(out, "hello world"); + } + #[test] fn newlines_inside_segments_collapse() { let out = to_plain_text(&[seg("line one\nline two\n\nline three")]);