feat: OpenWhispr-inspired transcription polish pass
Major quality pass on top of Phase 2. Five substantive changes plus
cross-cutting touches across audio, hotkey, transcription, and Tauri
command layers.
Transcription quality
- Long-audio chunking in commands/transcription.rs: Parakeet and large
file transcription now chunk-and-recompose with overlap trimming, so
the live-path chunking advantage extends to file-based workflows.
- Stateful live speech gate in commands/live.rs on top of the earlier
duplicate-boundary filtering — distinguishes start-of-speech from
mid-speech and holds state across chunks.
Auto-learning corrections
- New crates/ai-formatting/src/correction_learning.rs: extracts user
text corrections from viewer edits and proposes additions to the
active profile's vocabulary.
- src-tauri/src/commands/profiles.rs bridge for frontend-driven
confirmation of learned terms.
- src/routes/viewer/+page.svelte hooks the learning path into the
segment-edit flow so corrections feed profile_terms without a
separate 'train this profile' UX.
Transcript profile provenance
- Migration v8 (crates/storage/src/migrations.rs) adds profile_id to
transcripts, defaulting to DEFAULT_PROFILE_ID so existing rows stay
valid.
- crates/storage/src/database.rs: TranscriptRow + CRUD carry profile_id.
- src-tauri/src/commands/transcripts.rs: add_transcript accepts and
persists profile_id.
- DictationPage.svelte + FilesPage.svelte send activeProfileId on
capture so learned corrections are attributed to the right profile.
Cleanup prompt contract
- crates/ai-formatting/src/llm_client.rs hardened: the CLEANUP_PROMPT
now specifies concrete do/do-not rules, ready for a real model-backed
cleanup pass. The llm_client is still a stub — kon-llm remains unwired
— but the prompt shape is final.
Cross-cutting polish
- Minor touches in audio (capture/decode/resample), hotkey (lib/linux/stub),
core, transcription (concurrency/model_manager/local_engine/whisper_rs),
and the rest of src-tauri/src/commands/*: error-path tightening, log
clarity, TS-migration follow-ups (@ts-nocheck additions for incremental
typing).
Verified locally: npm run check, cargo test -p kon-ai-formatting,
cargo test -p kon-storage, cargo test -p kon --lib commands::live::tests,
cargo check — all green.
Scope boundary: kon-llm crate is still a stub; task extraction remains
rule-based. Bundled local-LLM runtime is the next clean step and is not
in this commit.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -10,7 +10,20 @@ use tauri::Emitter;
|
||||
use crate::commands::models::{default_model_id_for_engine, ensure_model_loaded};
|
||||
use crate::AppState;
|
||||
use kon_ai_formatting::{post_process_segments, FormatMode, PostProcessOptions};
|
||||
use kon_core::types::{Segment, TranscriptionOptions};
|
||||
use kon_core::constants::WHISPER_SAMPLE_RATE;
|
||||
use kon_core::types::{AudioSamples, Segment, Transcript, TranscriptionOptions};
|
||||
|
||||
const PARAKEET_CHUNK_THRESHOLD_SECS: usize = 18;
|
||||
const PARAKEET_CHUNK_SECS: usize = 15;
|
||||
const PARAKEET_CHUNK_OVERLAP_SECS: usize = 1;
|
||||
const FILE_CHUNK_THRESHOLD_SECS: usize = 8 * 60;
|
||||
const FILE_CHUNK_SECS: usize = 3 * 60;
|
||||
const FILE_CHUNK_OVERLAP_SECS: usize = 2;
|
||||
|
||||
struct ChunkingStrategy {
|
||||
chunk_samples: usize,
|
||||
overlap_samples: usize,
|
||||
}
|
||||
|
||||
fn pick_engine(
|
||||
state: &AppState,
|
||||
@@ -23,6 +36,104 @@ fn pick_engine(
|
||||
}
|
||||
}
|
||||
|
||||
fn pick_chunking_strategy(engine_name: &str, sample_count: usize) -> Option<ChunkingStrategy> {
|
||||
let samples_per_second = WHISPER_SAMPLE_RATE as usize;
|
||||
match engine_name {
|
||||
"parakeet" if sample_count > PARAKEET_CHUNK_THRESHOLD_SECS * samples_per_second => {
|
||||
Some(ChunkingStrategy {
|
||||
chunk_samples: PARAKEET_CHUNK_SECS * samples_per_second,
|
||||
overlap_samples: PARAKEET_CHUNK_OVERLAP_SECS * samples_per_second,
|
||||
})
|
||||
}
|
||||
_ if sample_count > FILE_CHUNK_THRESHOLD_SECS * samples_per_second => {
|
||||
Some(ChunkingStrategy {
|
||||
chunk_samples: FILE_CHUNK_SECS * samples_per_second,
|
||||
overlap_samples: FILE_CHUNK_OVERLAP_SECS * samples_per_second,
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn trim_overlap_segments(segments: &mut Vec<Segment>, trim_before_secs: f64) {
|
||||
if trim_before_secs <= 0.0 {
|
||||
return;
|
||||
}
|
||||
|
||||
segments.retain(|segment| segment.end > trim_before_secs);
|
||||
for segment in segments.iter_mut() {
|
||||
if segment.start < trim_before_secs {
|
||||
segment.start = trim_before_secs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn transcribe_samples_sync(
|
||||
engine: Arc<kon_transcription::LocalEngine>,
|
||||
engine_name: &str,
|
||||
samples: Vec<f32>,
|
||||
options: TranscriptionOptions,
|
||||
) -> Result<kon_transcription::TimedTranscript, String> {
|
||||
let Some(strategy) = pick_chunking_strategy(engine_name, samples.len()) else {
|
||||
let audio = AudioSamples::mono_16khz(samples);
|
||||
return engine
|
||||
.transcribe_sync(&audio, &options)
|
||||
.map_err(|e| e.to_string());
|
||||
};
|
||||
|
||||
let total_duration_secs = samples.len() as f64 / WHISPER_SAMPLE_RATE as f64;
|
||||
let stride = strategy
|
||||
.chunk_samples
|
||||
.saturating_sub(strategy.overlap_samples)
|
||||
.max(1);
|
||||
let chunk_count = ((samples.len().saturating_sub(1)) / stride) + 1;
|
||||
eprintln!(
|
||||
"[transcription] chunking {total_duration_secs:.2}s of {engine_name} audio into {chunk_count} chunk(s)"
|
||||
);
|
||||
|
||||
let mut all_segments = Vec::new();
|
||||
let mut total_inference_ms = 0u64;
|
||||
let mut chunk_start = 0usize;
|
||||
|
||||
while chunk_start < samples.len() {
|
||||
let chunk_end = (chunk_start + strategy.chunk_samples).min(samples.len());
|
||||
let chunk_audio = AudioSamples::mono_16khz(samples[chunk_start..chunk_end].to_vec());
|
||||
let timed = engine
|
||||
.transcribe_sync(&chunk_audio, &options)
|
||||
.map_err(|e| e.to_string())?;
|
||||
total_inference_ms = total_inference_ms.saturating_add(timed.inference_ms);
|
||||
|
||||
let mut chunk_segments = timed.transcript.segments().to_vec();
|
||||
if chunk_start > 0 {
|
||||
trim_overlap_segments(
|
||||
&mut chunk_segments,
|
||||
strategy.overlap_samples as f64 / WHISPER_SAMPLE_RATE as f64,
|
||||
);
|
||||
}
|
||||
|
||||
let chunk_offset_secs = chunk_start as f64 / WHISPER_SAMPLE_RATE as f64;
|
||||
for segment in &mut chunk_segments {
|
||||
segment.start += chunk_offset_secs;
|
||||
segment.end += chunk_offset_secs;
|
||||
}
|
||||
all_segments.extend(chunk_segments);
|
||||
|
||||
if chunk_end >= samples.len() {
|
||||
break;
|
||||
}
|
||||
chunk_start = chunk_end.saturating_sub(strategy.overlap_samples);
|
||||
}
|
||||
|
||||
Ok(kon_transcription::TimedTranscript {
|
||||
transcript: Transcript::new(
|
||||
all_segments,
|
||||
options.language.clone().unwrap_or_else(|| "en".to_string()),
|
||||
total_duration_secs,
|
||||
),
|
||||
inference_ms: total_inference_ms,
|
||||
})
|
||||
}
|
||||
|
||||
/// Transcribe raw PCM f32 samples (Whisper). Emits "transcription-result" event.
|
||||
#[tauri::command]
|
||||
pub async fn transcribe_pcm(
|
||||
@@ -38,8 +149,8 @@ pub async fn transcribe_pcm(
|
||||
format_mode: String,
|
||||
profile_id: Option<String>,
|
||||
) -> Result<(), String> {
|
||||
let resolved_profile_id = profile_id
|
||||
.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
|
||||
let resolved_profile_id =
|
||||
profile_id.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
|
||||
|
||||
let profile = kon_storage::database::get_profile(&state.db, &resolved_profile_id)
|
||||
.await
|
||||
@@ -61,7 +172,6 @@ pub async fn transcribe_pcm(
|
||||
};
|
||||
|
||||
let engine = state.whisper_engine.clone();
|
||||
let audio = kon_core::AudioSamples::mono_16khz(samples);
|
||||
let options = TranscriptionOptions {
|
||||
language: Some(language),
|
||||
initial_prompt: if effective_prompt.is_empty() {
|
||||
@@ -72,7 +182,10 @@ pub async fn transcribe_pcm(
|
||||
};
|
||||
|
||||
let timed = tokio::task::spawn_blocking(move || {
|
||||
engine.transcribe_sync(&audio, &options).map_err(|e| e.to_string())
|
||||
let audio = AudioSamples::mono_16khz(samples);
|
||||
engine
|
||||
.transcribe_sync(&audio, &options)
|
||||
.map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
@@ -122,8 +235,8 @@ pub async fn transcribe_file(
|
||||
format_mode: String,
|
||||
profile_id: Option<String>,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let resolved_profile_id = profile_id
|
||||
.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
|
||||
let resolved_profile_id =
|
||||
profile_id.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
|
||||
|
||||
let profile = kon_storage::database::get_profile(&state.db, &resolved_profile_id)
|
||||
.await
|
||||
@@ -145,8 +258,8 @@ pub async fn transcribe_file(
|
||||
};
|
||||
|
||||
let engine_name = engine.unwrap_or_else(|| "whisper".to_string());
|
||||
let model_id = model_id
|
||||
.unwrap_or_else(|| default_model_id_for_engine(&engine_name).to_string());
|
||||
let model_id =
|
||||
model_id.unwrap_or_else(|| default_model_id_for_engine(&engine_name).to_string());
|
||||
ensure_model_loaded(&state, &engine_name, &model_id).await?;
|
||||
|
||||
let engine = pick_engine(&state, &engine_name)?;
|
||||
@@ -158,15 +271,17 @@ pub async fn transcribe_file(
|
||||
Some(effective_prompt)
|
||||
},
|
||||
};
|
||||
let engine_name_for_worker = engine_name.clone();
|
||||
|
||||
let timed = tokio::task::spawn_blocking(move || {
|
||||
let audio = kon_audio::decode_audio_file(Path::new(&path))
|
||||
.map_err(|e| e.to_string())?;
|
||||
let resampled =
|
||||
kon_audio::resample_to_16khz(&audio).map_err(|e| e.to_string())?;
|
||||
engine
|
||||
.transcribe_sync(&resampled, &options)
|
||||
.map_err(|e| e.to_string())
|
||||
let audio = kon_audio::decode_audio_file(Path::new(&path)).map_err(|e| e.to_string())?;
|
||||
let resampled = kon_audio::resample_to_16khz(&audio).map_err(|e| e.to_string())?;
|
||||
transcribe_samples_sync(
|
||||
engine,
|
||||
&engine_name_for_worker,
|
||||
resampled.into_samples(),
|
||||
options,
|
||||
)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
@@ -208,8 +323,8 @@ pub async fn transcribe_pcm_parakeet(
|
||||
format_mode: String,
|
||||
profile_id: Option<String>,
|
||||
) -> Result<(), String> {
|
||||
let resolved_profile_id = profile_id
|
||||
.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
|
||||
let resolved_profile_id =
|
||||
profile_id.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
|
||||
|
||||
// Validate the profile exists so parakeet and whisper behave identically
|
||||
// when a bogus id slips through from the frontend.
|
||||
@@ -227,11 +342,10 @@ pub async fn transcribe_pcm_parakeet(
|
||||
.collect();
|
||||
|
||||
let engine = state.parakeet_engine.clone();
|
||||
let audio = kon_core::AudioSamples::mono_16khz(samples);
|
||||
let options = TranscriptionOptions::default();
|
||||
|
||||
let timed = tokio::task::spawn_blocking(move || {
|
||||
engine.transcribe_sync(&audio, &options).map_err(|e| e.to_string())
|
||||
transcribe_samples_sync(engine, "parakeet", samples, options)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
|
||||
Reference in New Issue
Block a user