feat(ai-formatting): collapse adjacent repetitions in Clean/Smart modes + chore(audio): swap deprecated cpal .name() for description-based helper

ai-formatting:
  - rule_based.rs: collapse_repetitions() merges adjacent duplicate
    tokens like 'I I can' -> 'I can' and 'think think that' -> 'think
    that'. Normalises case and punctuation before comparison.
  - pipeline.rs: post_process now calls collapse_repetitions when
    format_mode is Clean or Smart. Added unit coverage.

audio:
  - capture.rs: replace the seven deprecated cpal DeviceTrait::name()
    call sites with a device_display_name() helper that uses the
    non-deprecated description() path. Keeps identical behaviour,
    silences compile warnings, ready for cpal upgrade.

Addresses the 'Christ. Christ.' live-transcription boundary duplicate
Jake saw during Group 1 dogfooding. Does not fix all cross-chunk
overlap cases (see live.rs OVERLAP_SAMPLES for the root cause) but
catches the common stutter pattern at post-processing.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-19 20:05:36 +01:00
parent 27d85a0b28
commit 6605266587
3 changed files with 133 additions and 7 deletions

View File

@@ -47,6 +47,7 @@ pub fn post_process_segments(segments: &mut Vec<Segment>, options: &PostProcessO
seg.text = rule_based::to_british_english(&seg.text);
}
if options.format_mode != FormatMode::Raw {
seg.text = rule_based::collapse_repetitions(&seg.text);
seg.text = rule_based::format_text(&seg.text);
}
}
@@ -134,4 +135,24 @@ mod tests {
assert!(segments[2].text.starts_with("\n\n"));
}
#[test]
fn post_process_collapses_repeated_phrases_in_clean_modes() {
let mut segments = vec![Segment {
start: 0.0,
end: 1.0,
text: "I need I need to go to the shops".into(),
}];
let options = PostProcessOptions {
remove_fillers: false,
british_english: false,
anti_hallucination: false,
format_mode: FormatMode::Clean,
dictionary_terms: vec![],
};
post_process_segments(&mut segments, &options);
assert_eq!(segments[0].text, "I need to go to the shops");
}
}

View File

@@ -28,6 +28,12 @@ static FILLER_REGEXES: LazyLock<Vec<regex_lite::Regex>> = LazyLock::new(|| {
.collect()
});
fn normalise_repetition_token(token: &str) -> String {
token
.trim_matches(|ch: char| !(ch.is_alphanumeric() || ch == '\'' || ch == '-'))
.to_lowercase()
}
/// Remove common filler words from transcription text (case-insensitive).
pub fn remove_fillers(text: &str) -> String {
let mut result = text.to_string();
@@ -54,6 +60,77 @@ pub fn remove_fillers(text: &str) -> String {
collapsed.trim().to_string()
}
/// Collapse obvious stutters and immediate repeated short phrases.
///
/// Examples:
/// - `I I can` -> `I can`
/// - `I need I need to go` -> `I need to go`
/// - `Think think that's that` -> `Think that's that`
pub fn collapse_repetitions(text: &str) -> String {
if text.trim().is_empty() {
return String::new();
}
let tokens: Vec<&str> = text.split_whitespace().collect();
if tokens.len() < 2 {
return text.trim().to_string();
}
let normalised: Vec<String> = tokens
.iter()
.map(|token| normalise_repetition_token(token))
.collect();
let mut kept_indices: Vec<usize> = Vec::with_capacity(tokens.len());
let mut i = 0;
while i < tokens.len() {
let mut skipped_phrase = false;
for phrase_len in (1..=3).rev() {
if kept_indices.len() < phrase_len || i + phrase_len > tokens.len() {
continue;
}
let repeated = (0..phrase_len).all(|offset| {
let prev_index = kept_indices[kept_indices.len() - phrase_len + offset];
let prev = &normalised[prev_index];
let upcoming = &normalised[i + offset];
!prev.is_empty() && prev == upcoming
});
if repeated {
i += phrase_len;
skipped_phrase = true;
break;
}
}
if skipped_phrase {
continue;
}
if let Some(&last_index) = kept_indices.last() {
let current = &normalised[i];
let previous = &normalised[last_index];
if !current.is_empty() && current == previous {
i += 1;
continue;
}
}
kept_indices.push(i);
i += 1;
}
kept_indices
.into_iter()
.map(|index| tokens[index])
.collect::<Vec<_>>()
.join(" ")
.trim()
.to_string()
}
/// Replacement pairs for American to British English conversion.
///
/// All entries are plain base words (no regex metacharacters). The
@@ -260,6 +337,27 @@ mod tests {
assert!(to_british_english("the color is red").contains("colour"));
}
#[test]
fn collapse_repetitions_removes_consecutive_duplicate_words() {
assert_eq!(collapse_repetitions("I I can do that"), "I can do that");
assert_eq!(
collapse_repetitions("Think think that's that"),
"Think that's that"
);
}
#[test]
fn collapse_repetitions_removes_repeated_short_phrases() {
assert_eq!(
collapse_repetitions("I need I need to go to the shops"),
"I need to go to the shops"
);
assert_eq!(
collapse_repetitions("We should review we should review the draft"),
"We should review the draft"
);
}
#[test]
fn format_text_capitalises_after_full_stops() {
let result = format_text("hello world. this is a test");