From 53fe8489790354787ab20ae10e379ddcb043529a Mon Sep 17 00:00:00 2001 From: Jake Date: Wed, 22 Apr 2026 08:34:04 +0100 Subject: [PATCH] feat(A.4 #29): plain-text pre-formatter before LLM cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New crates/ai-formatting/src/to_plain_text.rs module with one public function: to_plain_text(&[Segment]) -> String. Rules the function enforces: - each segment's text is whitespace-normalised (any run of unicode whitespace collapses to a single ASCII space, so tabs, newlines, and NBSPs never reach the LLM), - empty and whitespace-only segments are dropped, - remaining segments are joined with a single ASCII space, - the joined string is normalised again (so a segment ending in a space followed by one starting in a space does not produce a double space) and trimmed end-to-end. pipeline.rs's inline join is replaced with this call. Whisper's timestamp fields (Segment.start / .end) are carried separately and never reach the LLM by construction — the "timestamps stripped" half of brief item #29's acceptance falls out of using Segment.text alone. The work the module actually adds is whitespace discipline and the tested boundary (empty input, empty-only input, NBSPs, pathological whitespace runs, idempotence, double-space at join boundaries). Source: Scriberr PR #288 — feeding raw Whisper JSON (with timestamps and per-segment structure) degraded cleanup quality; plain-text input raised it back. --- crates/ai-formatting/src/lib.rs | 2 + crates/ai-formatting/src/pipeline.rs | 14 +- crates/ai-formatting/src/to_plain_text.rs | 155 ++++++++++++++++++++++ 3 files changed, 164 insertions(+), 7 deletions(-) create mode 100644 crates/ai-formatting/src/to_plain_text.rs diff --git a/crates/ai-formatting/src/lib.rs b/crates/ai-formatting/src/lib.rs index 7d1a758..c3c8200 100644 --- a/crates/ai-formatting/src/lib.rs +++ b/crates/ai-formatting/src/lib.rs @@ -2,8 +2,10 @@ pub mod correction_learning; mod llm_client; pub mod pipeline; pub mod rule_based; +pub mod to_plain_text; pub use correction_learning::extract_corrections; pub use llm_client::{cleanup_text as llm_cleanup_text, LlmPromptPreset}; pub use pipeline::{post_process_segments, FormatMode, PostProcessOptions}; pub use rule_based::{format_text, is_hallucination, remove_fillers, to_british_english}; +pub use to_plain_text::to_plain_text; diff --git a/crates/ai-formatting/src/pipeline.rs b/crates/ai-formatting/src/pipeline.rs index 2d5c935..631310f 100644 --- a/crates/ai-formatting/src/pipeline.rs +++ b/crates/ai-formatting/src/pipeline.rs @@ -2,7 +2,7 @@ use kon_core::constants::SMART_PARAGRAPH_GAP_SECS; use kon_core::types::Segment; use kon_llm::LlmEngine; -use crate::{llm_client, rule_based}; +use crate::{llm_client, rule_based, to_plain_text::to_plain_text}; /// Post-processing options for a transcription pipeline run. pub struct PostProcessOptions { @@ -68,12 +68,12 @@ pub fn post_process_segments( if let Some(engine) = llm { if engine.is_loaded() && options.format_mode != FormatMode::Raw { - let joined = segments - .iter() - .map(|segment| segment.text.trim()) - .filter(|segment| !segment.is_empty()) - .collect::>() - .join(" "); + // Plain-text pre-formatter (brief item #29): collapse + // segments into a single natural-language string before + // the LLM call. Whitespace normalisation + empty-filter + // live in `to_plain_text`; the pipeline's job here is + // deciding whether to invoke the LLM at all. + let joined = to_plain_text(segments); if !joined.is_empty() { // Pipeline-internal cleanup (used by file-based + live diff --git a/crates/ai-formatting/src/to_plain_text.rs b/crates/ai-formatting/src/to_plain_text.rs new file mode 100644 index 0000000..ce31d30 --- /dev/null +++ b/crates/ai-formatting/src/to_plain_text.rs @@ -0,0 +1,155 @@ +//! Plain-text pre-formatter for LLM cleanup. +//! +//! Brief item #29: before sending transcription segments to the LLM, +//! join them into a single natural-language string with timestamps +//! stripped and whitespace normalised. Source: Scriberr PR #288 — +//! feeding raw Whisper JSON (with its timestamps and per-segment +//! structure) degraded cleanup quality materially; plain-text input +//! raised it back. +//! +//! `Segment.text` in Kon already holds just the spoken text (the +//! `start`/`end` f64 fields carry the timing), so "timestamp +//! stripping" falls out of using the text field alone. The work here +//! is the whitespace pass and empty-segment filter, plus a single +//! public function the pipeline can depend on. + +use kon_core::types::Segment; + +/// Join transcription segments into a single plain-text string +/// suitable for feeding to an LLM cleanup prompt. +/// +/// Rules: +/// - each segment's text is whitespace-normalised (any run of +/// whitespace — spaces, tabs, newlines, non-breaking spaces — +/// collapses to a single ASCII space), +/// - segments that are empty or whitespace-only are dropped, +/// - the remaining segments are joined with a single ASCII space, +/// - the final string is whitespace-normalised again (so a segment +/// ending in a space and the next beginning with one do not produce +/// a double space) and trimmed of leading/trailing whitespace. +/// +/// Pure function. No panics. Returns an empty string if every segment +/// filters out. +pub fn to_plain_text(segments: &[Segment]) -> String { + let joined = segments + .iter() + .map(|s| normalise_whitespace(&s.text)) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect::>() + .join(" "); + normalise_whitespace(&joined).trim().to_string() +} + +/// Collapse any run of unicode whitespace into a single ASCII space. +/// Kept private; the module's contract is `to_plain_text`. +fn normalise_whitespace(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut prev_was_space = false; + for ch in s.chars() { + if ch.is_whitespace() { + if !prev_was_space { + out.push(' '); + prev_was_space = true; + } + } else { + out.push(ch); + prev_was_space = false; + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn seg(text: &str) -> Segment { + Segment { + start: 0.0, + end: 1.0, + text: text.into(), + } + } + + #[test] + fn empty_input_is_empty_output() { + assert_eq!(to_plain_text(&[]), ""); + } + + #[test] + fn single_segment_returns_its_text_trimmed() { + let out = to_plain_text(&[seg(" hello world ")]); + assert_eq!(out, "hello world"); + } + + #[test] + fn multiple_segments_are_joined_with_single_space() { + let out = to_plain_text(&[seg("the cat"), seg("sat on the mat")]); + assert_eq!(out, "the cat sat on the mat"); + } + + #[test] + fn empty_and_whitespace_segments_are_filtered() { + let out = to_plain_text(&[ + seg("hello"), + seg(""), + seg(" "), + seg("\n\t "), + seg("world"), + ]); + assert_eq!(out, "hello world"); + } + + #[test] + fn internal_whitespace_runs_collapse_to_single_space() { + let out = to_plain_text(&[seg("hello\t\t \nworld")]); + assert_eq!(out, "hello world"); + } + + #[test] + fn join_boundary_does_not_produce_double_spaces() { + // First segment ends with whitespace, next starts with it — + // naive join would produce "foo bar". + let out = to_plain_text(&[seg("foo "), seg(" bar")]); + assert_eq!(out, "foo bar"); + } + + #[test] + fn non_breaking_space_is_treated_as_whitespace() { + // \u{00A0} is NBSP — char::is_whitespace returns true for it. + // LLM cleanup should not see NBSP leaked in. + let out = to_plain_text(&[seg("hello\u{00A0}world")]); + assert_eq!(out, "hello world"); + } + + #[test] + fn newlines_inside_segments_collapse() { + let out = to_plain_text(&[seg("line one\nline two\n\nline three")]); + assert_eq!(out, "line one line two line three"); + } + + #[test] + fn idempotent_on_already_normalised_text() { + // If the pipeline ever calls us twice, the second call must + // not mangle the result. + let once = to_plain_text(&[seg("hello world"), seg("foo bar")]); + let twice = to_plain_text(&[seg(&once)]); + assert_eq!(once, twice); + } + + #[test] + fn only_empty_segments_yields_empty_string() { + let out = to_plain_text(&[seg(""), seg(" "), seg("\t")]); + assert_eq!(out, ""); + } + + #[test] + fn no_panic_on_pathological_whitespace_runs() { + // A segment that is 10k spaces long normalises in linear time + // without panicking on capacity guesses. + let big_spaces = " ".repeat(10_000); + let out = to_plain_text(&[seg(&format!("a{big_spaces}b"))]); + assert_eq!(out, "a b"); + } +}