fix(A.4 #29): strip zero-width format chars in to_plain_text

Review feedback (MINOR): char::is_whitespace returns false for
zero-width format codepoints (U+200B ZWSP, U+200C ZWNJ, U+200D ZWJ,
U+2060 WORD JOINER, U+FEFF ZWNBSP / BOM). The original normalise
pass let them through to the LLM where they waste tokens without
contributing any natural-language content.

Makes the decision explicit: these chars STRIP entirely rather than
collapse to a space. Collapsing would silently insert a word break
where the source had none ("hello<FEFF>world" → "hello world"
would merge two words into a space-separated pair that the original
author did not intend). Stripping preserves the original token
boundaries and drops the invisible noise.

Three new tests:
- zero_width_format_chars_strip_entirely — exhaustive coverage of
  all five handled codepoints.
- zero_width_chars_do_not_break_adjacent_whitespace_collapsing —
  "hello <FEFF> world" still collapses to "hello world" (the
  strip does not leave behind an artefact that breaks the whitespace
  collapse pass).
- leading_bom_is_stripped — a BOM at segment start, the common
  artefact pattern when Whisper consumes an encoded file.

crates/ai-formatting/src/to_plain_text.rs

@@ -41,12 +41,28 @@ pub fn to_plain_text(segments: &[Segment]) -> String {
     normalise_whitespace(&joined).trim().to_string()
 }
 
-/// Collapse any run of unicode whitespace into a single ASCII space.
+/// Collapse any run of unicode whitespace into a single ASCII space,
+/// and strip zero-width format characters entirely.
+///
+/// Zero-width chars (U+200B/C/D, U+2060, U+FEFF) are handled as a
+/// separate class from whitespace: `char::is_whitespace()` returns
+/// false for them, so the standard whitespace pass would let them
+/// through to the LLM where they waste tokens without contributing
+/// any natural-language content. Treating them as "strip entirely"
+/// rather than "collapse to a space" avoids silently inserting word
+/// breaks where the source had none.
+///
 /// Kept private; the module's contract is `to_plain_text`.
 fn normalise_whitespace(s: &str) -> String {
     let mut out = String::with_capacity(s.len());
     let mut prev_was_space = false;
     for ch in s.chars() {
+        if is_zero_width_format(ch) {
+            // Strip without emitting anything. prev_was_space unchanged
+            // so a space on either side of a zero-width char still
+            // collapses correctly.
+            continue;
+        }
         if ch.is_whitespace() {
             if !prev_was_space {
                 out.push(' ');
@@ -60,6 +76,20 @@ fn normalise_whitespace(s: &str) -> String {
     out
 }
 
+/// Zero-width format characters the transcription pipeline should
+/// never feed to an LLM. Sourced from common "invisible" codepoints:
+/// - U+200B ZERO WIDTH SPACE
+/// - U+200C ZERO WIDTH NON-JOINER
+/// - U+200D ZERO WIDTH JOINER
+/// - U+2060 WORD JOINER
+/// - U+FEFF ZERO WIDTH NO-BREAK SPACE (also BOM)
+fn is_zero_width_format(ch: char) -> bool {
+    matches!(
+        ch,
+        '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{2060}' | '\u{FEFF}'
+    )
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -123,6 +153,41 @@ mod tests {
         assert_eq!(out, "hello world");
     }
 
+    #[test]
+    fn zero_width_format_chars_strip_entirely() {
+        // char::is_whitespace returns false for all of these, so the
+        // default whitespace pass would let them through. They carry
+        // no natural-language content — stripping them saves LLM
+        // tokens without changing meaning.
+        let cases = [
+            ("hello\u{200B}world", "helloworld"), // ZERO WIDTH SPACE
+            ("hello\u{200C}world", "helloworld"), // ZWNJ
+            ("hello\u{200D}world", "helloworld"), // ZWJ
+            ("hello\u{2060}world", "helloworld"), // WORD JOINER
+            ("hello\u{FEFF}world", "helloworld"), // ZWNBSP / BOM
+        ];
+        for (input, expected) in cases {
+            let out = to_plain_text(&[seg(input)]);
+            assert_eq!(out, expected, "input {input:?} should strip to {expected:?}");
+        }
+    }
+
+    #[test]
+    fn zero_width_chars_do_not_break_adjacent_whitespace_collapsing() {
+        // "hello \u{FEFF} world" — the zero-width char between two
+        // spaces should strip, leaving a single collapsed space.
+        let out = to_plain_text(&[seg("hello \u{FEFF} world")]);
+        assert_eq!(out, "hello world");
+    }
+
+    #[test]
+    fn leading_bom_is_stripped() {
+        // BOM at start of segment — common artifact when Whisper
+        // consumes a file whose encoding pass inserted one.
+        let out = to_plain_text(&[seg("\u{FEFF}hello world")]);
+        assert_eq!(out, "hello world");
+    }
+
     #[test]
     fn newlines_inside_segments_collapse() {
         let out = to_plain_text(&[seg("line one\nline two\n\nline three")]);