feat(ai-formatting): collapse adjacent repetitions in Clean/Smart modes + chore(audio): swap deprecated cpal .name() for description-based helper
ai-formatting:
- rule_based.rs: collapse_repetitions() merges adjacent duplicate
tokens like 'I I can' -> 'I can' and 'think think that' -> 'think
that'. Normalises case and punctuation before comparison.
- pipeline.rs: post_process now calls collapse_repetitions when
format_mode is Clean or Smart. Added unit coverage.
audio:
- capture.rs: replace the seven deprecated cpal DeviceTrait::name()
call sites with a device_display_name() helper that uses the
non-deprecated description() path. Keeps identical behaviour,
silences compile warnings, ready for cpal upgrade.
Addresses the 'Christ. Christ.' live-transcription boundary duplicate
Jake saw during Group 1 dogfooding. Does not fix all cross-chunk
overlap cases (see live.rs OVERLAP_SAMPLES for the root cause) but
catches the common stutter pattern at post-processing.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -28,6 +28,12 @@ static FILLER_REGEXES: LazyLock<Vec<regex_lite::Regex>> = LazyLock::new(|| {
|
||||
.collect()
|
||||
});
|
||||
|
||||
fn normalise_repetition_token(token: &str) -> String {
|
||||
token
|
||||
.trim_matches(|ch: char| !(ch.is_alphanumeric() || ch == '\'' || ch == '-'))
|
||||
.to_lowercase()
|
||||
}
|
||||
|
||||
/// Remove common filler words from transcription text (case-insensitive).
|
||||
pub fn remove_fillers(text: &str) -> String {
|
||||
let mut result = text.to_string();
|
||||
@@ -54,6 +60,77 @@ pub fn remove_fillers(text: &str) -> String {
|
||||
collapsed.trim().to_string()
|
||||
}
|
||||
|
||||
/// Collapse obvious stutters and immediate repeated short phrases.
|
||||
///
|
||||
/// Examples:
|
||||
/// - `I I can` -> `I can`
|
||||
/// - `I need I need to go` -> `I need to go`
|
||||
/// - `Think think that's that` -> `Think that's that`
|
||||
pub fn collapse_repetitions(text: &str) -> String {
|
||||
if text.trim().is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let tokens: Vec<&str> = text.split_whitespace().collect();
|
||||
if tokens.len() < 2 {
|
||||
return text.trim().to_string();
|
||||
}
|
||||
|
||||
let normalised: Vec<String> = tokens
|
||||
.iter()
|
||||
.map(|token| normalise_repetition_token(token))
|
||||
.collect();
|
||||
let mut kept_indices: Vec<usize> = Vec::with_capacity(tokens.len());
|
||||
let mut i = 0;
|
||||
|
||||
while i < tokens.len() {
|
||||
let mut skipped_phrase = false;
|
||||
|
||||
for phrase_len in (1..=3).rev() {
|
||||
if kept_indices.len() < phrase_len || i + phrase_len > tokens.len() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let repeated = (0..phrase_len).all(|offset| {
|
||||
let prev_index = kept_indices[kept_indices.len() - phrase_len + offset];
|
||||
let prev = &normalised[prev_index];
|
||||
let upcoming = &normalised[i + offset];
|
||||
!prev.is_empty() && prev == upcoming
|
||||
});
|
||||
|
||||
if repeated {
|
||||
i += phrase_len;
|
||||
skipped_phrase = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if skipped_phrase {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(&last_index) = kept_indices.last() {
|
||||
let current = &normalised[i];
|
||||
let previous = &normalised[last_index];
|
||||
if !current.is_empty() && current == previous {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
kept_indices.push(i);
|
||||
i += 1;
|
||||
}
|
||||
|
||||
kept_indices
|
||||
.into_iter()
|
||||
.map(|index| tokens[index])
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Replacement pairs for American to British English conversion.
|
||||
///
|
||||
/// All entries are plain base words (no regex metacharacters). The
|
||||
@@ -260,6 +337,27 @@ mod tests {
|
||||
assert!(to_british_english("the color is red").contains("colour"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collapse_repetitions_removes_consecutive_duplicate_words() {
|
||||
assert_eq!(collapse_repetitions("I I can do that"), "I can do that");
|
||||
assert_eq!(
|
||||
collapse_repetitions("Think think that's that"),
|
||||
"Think that's that"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collapse_repetitions_removes_repeated_short_phrases() {
|
||||
assert_eq!(
|
||||
collapse_repetitions("I need I need to go to the shops"),
|
||||
"I need to go to the shops"
|
||||
);
|
||||
assert_eq!(
|
||||
collapse_repetitions("We should review we should review the draft"),
|
||||
"We should review the draft"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_text_capitalises_after_full_stops() {
|
||||
let result = format_text("hello world. this is a test");
|
||||
|
||||
Reference in New Issue
Block a user