feat(A.4 #29): plain-text pre-formatter before LLM cleanup
New crates/ai-formatting/src/to_plain_text.rs module with one public function: to_plain_text(&[Segment]) -> String. Rules the function enforces: - each segment's text is whitespace-normalised (any run of unicode whitespace collapses to a single ASCII space, so tabs, newlines, and NBSPs never reach the LLM), - empty and whitespace-only segments are dropped, - remaining segments are joined with a single ASCII space, - the joined string is normalised again (so a segment ending in a space followed by one starting in a space does not produce a double space) and trimmed end-to-end. pipeline.rs's inline join is replaced with this call. Whisper's timestamp fields (Segment.start / .end) are carried separately and never reach the LLM by construction — the "timestamps stripped" half of brief item #29's acceptance falls out of using Segment.text alone. The work the module actually adds is whitespace discipline and the tested boundary (empty input, empty-only input, NBSPs, pathological whitespace runs, idempotence, double-space at join boundaries). Source: Scriberr PR #288 — feeding raw Whisper JSON (with timestamps and per-segment structure) degraded cleanup quality; plain-text input raised it back.
This commit is contained in:
@@ -2,7 +2,7 @@ use kon_core::constants::SMART_PARAGRAPH_GAP_SECS;
|
||||
use kon_core::types::Segment;
|
||||
use kon_llm::LlmEngine;
|
||||
|
||||
use crate::{llm_client, rule_based};
|
||||
use crate::{llm_client, rule_based, to_plain_text::to_plain_text};
|
||||
|
||||
/// Post-processing options for a transcription pipeline run.
|
||||
pub struct PostProcessOptions {
|
||||
@@ -68,12 +68,12 @@ pub fn post_process_segments(
|
||||
|
||||
if let Some(engine) = llm {
|
||||
if engine.is_loaded() && options.format_mode != FormatMode::Raw {
|
||||
let joined = segments
|
||||
.iter()
|
||||
.map(|segment| segment.text.trim())
|
||||
.filter(|segment| !segment.is_empty())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
// Plain-text pre-formatter (brief item #29): collapse
|
||||
// segments into a single natural-language string before
|
||||
// the LLM call. Whitespace normalisation + empty-filter
|
||||
// live in `to_plain_text`; the pipeline's job here is
|
||||
// deciding whether to invoke the LLM at all.
|
||||
let joined = to_plain_text(segments);
|
||||
|
||||
if !joined.is_empty() {
|
||||
// Pipeline-internal cleanup (used by file-based + live
|
||||
|
||||
Reference in New Issue
Block a user