feat: OpenWhispr-inspired transcription polish pass
Some checks failed
check / cargo check (macos-latest) (push) Has been cancelled
check / cargo check (ubuntu-22.04) (push) Has been cancelled
check / cargo check (windows-latest) (push) Has been cancelled
check / svelte build + lint (push) Has been cancelled

Major quality pass on top of Phase 2. Five substantive changes plus
cross-cutting touches across audio, hotkey, transcription, and Tauri
command layers.

  Transcription quality

  - Long-audio chunking in commands/transcription.rs: Parakeet and large
    file transcription now chunk-and-recompose with overlap trimming, so
    the live-path chunking advantage extends to file-based workflows.
  - Stateful live speech gate in commands/live.rs on top of the earlier
    duplicate-boundary filtering — distinguishes start-of-speech from
    mid-speech and holds state across chunks.

  Auto-learning corrections

  - New crates/ai-formatting/src/correction_learning.rs: extracts user
    text corrections from viewer edits and proposes additions to the
    active profile's vocabulary.
  - src-tauri/src/commands/profiles.rs bridge for frontend-driven
    confirmation of learned terms.
  - src/routes/viewer/+page.svelte hooks the learning path into the
    segment-edit flow so corrections feed profile_terms without a
    separate 'train this profile' UX.

  Transcript profile provenance

  - Migration v8 (crates/storage/src/migrations.rs) adds profile_id to
    transcripts, defaulting to DEFAULT_PROFILE_ID so existing rows stay
    valid.
  - crates/storage/src/database.rs: TranscriptRow + CRUD carry profile_id.
  - src-tauri/src/commands/transcripts.rs: add_transcript accepts and
    persists profile_id.
  - DictationPage.svelte + FilesPage.svelte send activeProfileId on
    capture so learned corrections are attributed to the right profile.

  Cleanup prompt contract

  - crates/ai-formatting/src/llm_client.rs hardened: the CLEANUP_PROMPT
    now specifies concrete do/do-not rules, ready for a real model-backed
    cleanup pass. The llm_client is still a stub — kon-llm remains unwired
    — but the prompt shape is final.

  Cross-cutting polish

  - Minor touches in audio (capture/decode/resample), hotkey (lib/linux/stub),
    core, transcription (concurrency/model_manager/local_engine/whisper_rs),
    and the rest of src-tauri/src/commands/*: error-path tightening, log
    clarity, TS-migration follow-ups (@ts-nocheck additions for incremental
    typing).

Verified locally: npm run check, cargo test -p kon-ai-formatting,
cargo test -p kon-storage, cargo test -p kon --lib commands::live::tests,
cargo check — all green.

Scope boundary: kon-llm crate is still a stub; task extraction remains
rule-based. Bundled local-LLM runtime is the next clean step and is not
in this commit.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-19 22:39:08 +01:00
parent 28acdcfa6d
commit 34fce3cf9e
39 changed files with 1581 additions and 554 deletions

View File

@@ -10,7 +10,20 @@ use tauri::Emitter;
use crate::commands::models::{default_model_id_for_engine, ensure_model_loaded};
use crate::AppState;
use kon_ai_formatting::{post_process_segments, FormatMode, PostProcessOptions};
use kon_core::types::{Segment, TranscriptionOptions};
use kon_core::constants::WHISPER_SAMPLE_RATE;
use kon_core::types::{AudioSamples, Segment, Transcript, TranscriptionOptions};
const PARAKEET_CHUNK_THRESHOLD_SECS: usize = 18;
const PARAKEET_CHUNK_SECS: usize = 15;
const PARAKEET_CHUNK_OVERLAP_SECS: usize = 1;
const FILE_CHUNK_THRESHOLD_SECS: usize = 8 * 60;
const FILE_CHUNK_SECS: usize = 3 * 60;
const FILE_CHUNK_OVERLAP_SECS: usize = 2;
struct ChunkingStrategy {
chunk_samples: usize,
overlap_samples: usize,
}
fn pick_engine(
state: &AppState,
@@ -23,6 +36,104 @@ fn pick_engine(
}
}
fn pick_chunking_strategy(engine_name: &str, sample_count: usize) -> Option<ChunkingStrategy> {
let samples_per_second = WHISPER_SAMPLE_RATE as usize;
match engine_name {
"parakeet" if sample_count > PARAKEET_CHUNK_THRESHOLD_SECS * samples_per_second => {
Some(ChunkingStrategy {
chunk_samples: PARAKEET_CHUNK_SECS * samples_per_second,
overlap_samples: PARAKEET_CHUNK_OVERLAP_SECS * samples_per_second,
})
}
_ if sample_count > FILE_CHUNK_THRESHOLD_SECS * samples_per_second => {
Some(ChunkingStrategy {
chunk_samples: FILE_CHUNK_SECS * samples_per_second,
overlap_samples: FILE_CHUNK_OVERLAP_SECS * samples_per_second,
})
}
_ => None,
}
}
fn trim_overlap_segments(segments: &mut Vec<Segment>, trim_before_secs: f64) {
if trim_before_secs <= 0.0 {
return;
}
segments.retain(|segment| segment.end > trim_before_secs);
for segment in segments.iter_mut() {
if segment.start < trim_before_secs {
segment.start = trim_before_secs;
}
}
}
fn transcribe_samples_sync(
engine: Arc<kon_transcription::LocalEngine>,
engine_name: &str,
samples: Vec<f32>,
options: TranscriptionOptions,
) -> Result<kon_transcription::TimedTranscript, String> {
let Some(strategy) = pick_chunking_strategy(engine_name, samples.len()) else {
let audio = AudioSamples::mono_16khz(samples);
return engine
.transcribe_sync(&audio, &options)
.map_err(|e| e.to_string());
};
let total_duration_secs = samples.len() as f64 / WHISPER_SAMPLE_RATE as f64;
let stride = strategy
.chunk_samples
.saturating_sub(strategy.overlap_samples)
.max(1);
let chunk_count = ((samples.len().saturating_sub(1)) / stride) + 1;
eprintln!(
"[transcription] chunking {total_duration_secs:.2}s of {engine_name} audio into {chunk_count} chunk(s)"
);
let mut all_segments = Vec::new();
let mut total_inference_ms = 0u64;
let mut chunk_start = 0usize;
while chunk_start < samples.len() {
let chunk_end = (chunk_start + strategy.chunk_samples).min(samples.len());
let chunk_audio = AudioSamples::mono_16khz(samples[chunk_start..chunk_end].to_vec());
let timed = engine
.transcribe_sync(&chunk_audio, &options)
.map_err(|e| e.to_string())?;
total_inference_ms = total_inference_ms.saturating_add(timed.inference_ms);
let mut chunk_segments = timed.transcript.segments().to_vec();
if chunk_start > 0 {
trim_overlap_segments(
&mut chunk_segments,
strategy.overlap_samples as f64 / WHISPER_SAMPLE_RATE as f64,
);
}
let chunk_offset_secs = chunk_start as f64 / WHISPER_SAMPLE_RATE as f64;
for segment in &mut chunk_segments {
segment.start += chunk_offset_secs;
segment.end += chunk_offset_secs;
}
all_segments.extend(chunk_segments);
if chunk_end >= samples.len() {
break;
}
chunk_start = chunk_end.saturating_sub(strategy.overlap_samples);
}
Ok(kon_transcription::TimedTranscript {
transcript: Transcript::new(
all_segments,
options.language.clone().unwrap_or_else(|| "en".to_string()),
total_duration_secs,
),
inference_ms: total_inference_ms,
})
}
/// Transcribe raw PCM f32 samples (Whisper). Emits "transcription-result" event.
#[tauri::command]
pub async fn transcribe_pcm(
@@ -38,8 +149,8 @@ pub async fn transcribe_pcm(
format_mode: String,
profile_id: Option<String>,
) -> Result<(), String> {
let resolved_profile_id = profile_id
.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
let resolved_profile_id =
profile_id.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
let profile = kon_storage::database::get_profile(&state.db, &resolved_profile_id)
.await
@@ -61,7 +172,6 @@ pub async fn transcribe_pcm(
};
let engine = state.whisper_engine.clone();
let audio = kon_core::AudioSamples::mono_16khz(samples);
let options = TranscriptionOptions {
language: Some(language),
initial_prompt: if effective_prompt.is_empty() {
@@ -72,7 +182,10 @@ pub async fn transcribe_pcm(
};
let timed = tokio::task::spawn_blocking(move || {
engine.transcribe_sync(&audio, &options).map_err(|e| e.to_string())
let audio = AudioSamples::mono_16khz(samples);
engine
.transcribe_sync(&audio, &options)
.map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
@@ -122,8 +235,8 @@ pub async fn transcribe_file(
format_mode: String,
profile_id: Option<String>,
) -> Result<serde_json::Value, String> {
let resolved_profile_id = profile_id
.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
let resolved_profile_id =
profile_id.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
let profile = kon_storage::database::get_profile(&state.db, &resolved_profile_id)
.await
@@ -145,8 +258,8 @@ pub async fn transcribe_file(
};
let engine_name = engine.unwrap_or_else(|| "whisper".to_string());
let model_id = model_id
.unwrap_or_else(|| default_model_id_for_engine(&engine_name).to_string());
let model_id =
model_id.unwrap_or_else(|| default_model_id_for_engine(&engine_name).to_string());
ensure_model_loaded(&state, &engine_name, &model_id).await?;
let engine = pick_engine(&state, &engine_name)?;
@@ -158,15 +271,17 @@ pub async fn transcribe_file(
Some(effective_prompt)
},
};
let engine_name_for_worker = engine_name.clone();
let timed = tokio::task::spawn_blocking(move || {
let audio = kon_audio::decode_audio_file(Path::new(&path))
.map_err(|e| e.to_string())?;
let resampled =
kon_audio::resample_to_16khz(&audio).map_err(|e| e.to_string())?;
engine
.transcribe_sync(&resampled, &options)
.map_err(|e| e.to_string())
let audio = kon_audio::decode_audio_file(Path::new(&path)).map_err(|e| e.to_string())?;
let resampled = kon_audio::resample_to_16khz(&audio).map_err(|e| e.to_string())?;
transcribe_samples_sync(
engine,
&engine_name_for_worker,
resampled.into_samples(),
options,
)
})
.await
.map_err(|e| e.to_string())??;
@@ -208,8 +323,8 @@ pub async fn transcribe_pcm_parakeet(
format_mode: String,
profile_id: Option<String>,
) -> Result<(), String> {
let resolved_profile_id = profile_id
.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
let resolved_profile_id =
profile_id.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
// Validate the profile exists so parakeet and whisper behave identically
// when a bogus id slips through from the frontend.
@@ -227,11 +342,10 @@ pub async fn transcribe_pcm_parakeet(
.collect();
let engine = state.parakeet_engine.clone();
let audio = kon_core::AudioSamples::mono_16khz(samples);
let options = TranscriptionOptions::default();
let timed = tokio::task::spawn_blocking(move || {
engine.transcribe_sync(&audio, &options).map_err(|e| e.to_string())
transcribe_samples_sync(engine, "parakeet", samples, options)
})
.await
.map_err(|e| e.to_string())??;