Major quality pass on top of Phase 2. Five substantive changes plus
cross-cutting touches across audio, hotkey, transcription, and Tauri
command layers.
Transcription quality
- Long-audio chunking in commands/transcription.rs: Parakeet and large
file transcription now chunk-and-recompose with overlap trimming, so
the live-path chunking advantage extends to file-based workflows.
- Stateful live speech gate in commands/live.rs on top of the earlier
duplicate-boundary filtering — distinguishes start-of-speech from
mid-speech and holds state across chunks.
Auto-learning corrections
- New crates/ai-formatting/src/correction_learning.rs: extracts user
text corrections from viewer edits and proposes additions to the
active profile's vocabulary.
- src-tauri/src/commands/profiles.rs bridge for frontend-driven
confirmation of learned terms.
- src/routes/viewer/+page.svelte hooks the learning path into the
segment-edit flow so corrections feed profile_terms without a
separate 'train this profile' UX.
Transcript profile provenance
- Migration v8 (crates/storage/src/migrations.rs) adds profile_id to
transcripts, defaulting to DEFAULT_PROFILE_ID so existing rows stay
valid.
- crates/storage/src/database.rs: TranscriptRow + CRUD carry profile_id.
- src-tauri/src/commands/transcripts.rs: add_transcript accepts and
persists profile_id.
- DictationPage.svelte + FilesPage.svelte send activeProfileId on
capture so learned corrections are attributed to the right profile.
Cleanup prompt contract
- crates/ai-formatting/src/llm_client.rs hardened: the CLEANUP_PROMPT
now specifies concrete do/do-not rules, ready for a real model-backed
cleanup pass. The llm_client is still a stub — kon-llm remains unwired
— but the prompt shape is final.
Cross-cutting polish
- Minor touches in audio (capture/decode/resample), hotkey (lib/linux/stub),
core, transcription (concurrency/model_manager/local_engine/whisper_rs),
and the rest of src-tauri/src/commands/*: error-path tightening, log
clarity, TS-migration follow-ups (@ts-nocheck additions for incremental
typing).
Verified locally: npm run check, cargo test -p kon-ai-formatting,
cargo test -p kon-storage, cargo test -p kon --lib commands::live::tests,
cargo check — all green.
Scope boundary: kon-llm crate is still a stub; task extraction remains
rule-based. Bundled local-LLM runtime is the next clean step and is not
in this commit.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
282 lines
9.8 KiB
Rust
282 lines
9.8 KiB
Rust
use std::path::PathBuf;
|
||
use std::sync::{Arc, Mutex};
|
||
|
||
use tauri::{Emitter, Manager};
|
||
use tokio::sync::mpsc as tokio_mpsc;
|
||
|
||
use kon_audio::{DeviceInfo, MicrophoneCapture};
|
||
use kon_core::constants::WHISPER_SAMPLE_RATE;
|
||
use kon_core::types::AudioSamples;
|
||
|
||
/// Enumerate every input device available to cpal, with metadata for the
|
||
/// Settings device-picker UI. Includes a flag for likely PulseAudio /
|
||
/// PipeWire monitor sources so the UI can warn the user.
|
||
#[tauri::command]
|
||
pub async fn list_audio_devices() -> Result<Vec<DeviceInfo>, String> {
|
||
tokio::task::spawn_blocking(MicrophoneCapture::list_devices)
|
||
.await
|
||
.map_err(|e| format!("join error: {e}"))?
|
||
.map_err(|e| e.to_string())
|
||
}
|
||
|
||
/// Shared state for native microphone capture.
|
||
pub struct NativeCaptureState {
|
||
/// Stop signal sender — dropping this stops the accumulator task.
|
||
stop_tx: Mutex<Option<tokio_mpsc::Sender<()>>>,
|
||
/// All captured samples (16kHz mono) for save_audio.
|
||
all_samples: Arc<Mutex<Vec<f32>>>,
|
||
}
|
||
|
||
impl NativeCaptureState {
|
||
pub fn new() -> Self {
|
||
Self {
|
||
stop_tx: Mutex::new(None),
|
||
all_samples: Arc::new(Mutex::new(Vec::new())),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Start native microphone capture via cpal.
|
||
/// Streams 16kHz mono PCM chunks to the frontend via `native-pcm` events.
|
||
///
|
||
/// `device_name`: explicit device name (from `list_audio_devices`) or None / ""
|
||
/// to auto-select. The frontend passes `settings.microphoneDevice` here so the
|
||
/// user's pick from Settings → Audio → Microphone takes effect.
|
||
#[tauri::command]
|
||
pub async fn start_native_capture(
|
||
app: tauri::AppHandle,
|
||
state: tauri::State<'_, NativeCaptureState>,
|
||
device_name: Option<String>,
|
||
) -> Result<(), String> {
|
||
eprintln!(
|
||
"[native-capture] start_native_capture called (device='{}')",
|
||
device_name.as_deref().unwrap_or("<auto>")
|
||
);
|
||
|
||
// Stop any existing capture: send an explicit stop signal first, then
|
||
// drop the sender. The accumulator task watches for `Disconnected` too,
|
||
// but signalling explicitly avoids the brief race window.
|
||
// (Codex review 2026/04/17 D1)
|
||
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
|
||
let _ = tx.try_send(());
|
||
drop(tx);
|
||
}
|
||
|
||
// `MicrophoneCapture::start()` is synchronous and may spend up to
|
||
// DEVICE_VALIDATION_MS per device per pass (350ms × N devices × 2 passes
|
||
// worst case). Run on a blocking thread so the async runtime stays
|
||
// responsive to other Tauri commands. (Codex review 2026/04/17 D2)
|
||
let device_name_for_blocking = device_name.clone();
|
||
let (capture, rx) =
|
||
tokio::task::spawn_blocking(move || match device_name_for_blocking.as_deref() {
|
||
Some(name) if !name.is_empty() => MicrophoneCapture::start_with_device(name),
|
||
_ => MicrophoneCapture::start(),
|
||
})
|
||
.await
|
||
.map_err(|e| format!("audio task join error: {e}"))?
|
||
.map_err(|e| {
|
||
eprintln!("[native-capture] MicrophoneCapture::start failed: {e}");
|
||
e.to_string()
|
||
})?;
|
||
eprintln!(
|
||
"[native-capture] cpal capture started successfully on '{}'",
|
||
capture.device_name
|
||
);
|
||
|
||
// Wrap capture in Arc<Mutex> so it can be moved into the blocking task
|
||
let capture = Arc::new(Mutex::new(Some(capture)));
|
||
let capture_clone = capture.clone();
|
||
|
||
let all_samples = state.all_samples.clone();
|
||
all_samples.lock().unwrap().clear();
|
||
|
||
let (stop_tx, mut stop_rx) = tokio_mpsc::channel::<()>(1);
|
||
*state.stop_tx.lock().unwrap() = Some(stop_tx);
|
||
|
||
let all_samples_clone = all_samples.clone();
|
||
|
||
// Spawn a task that reads cpal chunks, downsamples to 16kHz mono,
|
||
// and emits events to the frontend
|
||
tokio::spawn(async move {
|
||
let mut pcm_buffer: Vec<f32> = Vec::new();
|
||
let chunk_size = 8000_usize; // ~0.5s at 16kHz
|
||
|
||
loop {
|
||
// Check for stop signal (non-blocking)
|
||
if stop_rx.try_recv().is_ok() {
|
||
break;
|
||
}
|
||
|
||
// Drain available audio chunks from cpal (non-blocking).
|
||
// Distinguish Empty (try again) from Disconnected (capture stream
|
||
// is dead — exit the loop, don't spin forever).
|
||
// (Codex review 2026/04/17 M3)
|
||
let mut got_data = false;
|
||
let mut capture_dead = false;
|
||
loop {
|
||
match rx.try_recv() {
|
||
Ok(chunk) => {
|
||
got_data = true;
|
||
let sample_rate = chunk.sample_rate;
|
||
let channels = chunk.channels as usize;
|
||
|
||
// Downmix to mono if stereo
|
||
let mono: Vec<f32> = if channels > 1 {
|
||
chunk
|
||
.samples
|
||
.chunks(channels)
|
||
.map(|frame| frame.iter().sum::<f32>() / channels as f32)
|
||
.collect()
|
||
} else {
|
||
chunk.samples
|
||
};
|
||
|
||
// Downsample to 16kHz using simple decimation
|
||
// (acceptable quality for speech — same approach as pcm-processor.js)
|
||
let ratio = sample_rate as f64 / WHISPER_SAMPLE_RATE as f64;
|
||
if (ratio - 1.0).abs() < 0.01 {
|
||
pcm_buffer.extend_from_slice(&mono);
|
||
} else {
|
||
let mut pos: f64 = 0.0;
|
||
for &s in &mono {
|
||
pos += 1.0;
|
||
if pos >= ratio {
|
||
pcm_buffer.push(s);
|
||
pos -= ratio;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Err(std::sync::mpsc::TryRecvError::Empty) => break,
|
||
Err(std::sync::mpsc::TryRecvError::Disconnected) => {
|
||
eprintln!(
|
||
"[native-capture] capture stream disconnected; accumulator exiting"
|
||
);
|
||
capture_dead = true;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
// Emit chunks to frontend when we have enough
|
||
while pcm_buffer.len() >= chunk_size {
|
||
let chunk: Vec<f32> = pcm_buffer.drain(..chunk_size).collect();
|
||
|
||
// Store for save_audio
|
||
if let Ok(mut all) = all_samples_clone.lock() {
|
||
all.extend_from_slice(&chunk);
|
||
}
|
||
|
||
let _ = app.emit(
|
||
"native-pcm",
|
||
serde_json::json!({
|
||
"samples": chunk,
|
||
}),
|
||
);
|
||
}
|
||
|
||
if capture_dead {
|
||
break;
|
||
}
|
||
if !got_data {
|
||
// Avoid busy-spinning when no audio data is available
|
||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||
}
|
||
}
|
||
|
||
// Emit any remaining samples
|
||
if !pcm_buffer.is_empty() {
|
||
if let Ok(mut all) = all_samples_clone.lock() {
|
||
all.extend_from_slice(&pcm_buffer);
|
||
}
|
||
let _ = app.emit(
|
||
"native-pcm",
|
||
serde_json::json!({
|
||
"samples": pcm_buffer,
|
||
}),
|
||
);
|
||
}
|
||
|
||
// Drop the capture to stop the cpal stream
|
||
if let Ok(mut cap) = capture_clone.lock() {
|
||
cap.take();
|
||
}
|
||
});
|
||
|
||
Ok(())
|
||
}
|
||
|
||
/// Stop native microphone capture. Returns all captured samples (16kHz mono).
|
||
#[tauri::command]
|
||
pub async fn stop_native_capture(
|
||
state: tauri::State<'_, NativeCaptureState>,
|
||
) -> Result<Vec<f32>, String> {
|
||
// Extract the stop sender without holding the guard across an await
|
||
let stop_tx = state.stop_tx.lock().unwrap().take();
|
||
if let Some(tx) = stop_tx {
|
||
let _ = tx.send(()).await;
|
||
}
|
||
|
||
// Brief delay to let the accumulator flush
|
||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||
|
||
let samples = {
|
||
let mut all = state.all_samples.lock().unwrap();
|
||
std::mem::take(&mut *all)
|
||
};
|
||
|
||
Ok(samples)
|
||
}
|
||
|
||
pub async fn persist_audio_samples(
|
||
app: &tauri::AppHandle,
|
||
samples: Vec<f32>,
|
||
output_folder: Option<String>,
|
||
) -> Result<String, String> {
|
||
let recordings_dir = if let Some(ref folder) = output_folder {
|
||
if !folder.is_empty() {
|
||
PathBuf::from(folder)
|
||
} else {
|
||
app.path()
|
||
.app_local_data_dir()
|
||
.map_err(|e: tauri::Error| e.to_string())?
|
||
.join("recordings")
|
||
}
|
||
} else {
|
||
app.path()
|
||
.app_local_data_dir()
|
||
.map_err(|e: tauri::Error| e.to_string())?
|
||
.join("recordings")
|
||
};
|
||
|
||
std::fs::create_dir_all(&recordings_dir)
|
||
.map_err(|e| format!("Failed to create recordings dir: {e}"))?;
|
||
|
||
let timestamp = std::time::SystemTime::now()
|
||
.duration_since(std::time::UNIX_EPOCH)
|
||
.unwrap_or_default()
|
||
.as_secs();
|
||
let filename = format!("kon-{timestamp}.wav");
|
||
let path = recordings_dir.join(&filename);
|
||
let path_clone = path.clone();
|
||
|
||
tokio::task::spawn_blocking(move || {
|
||
let audio = AudioSamples::mono_16khz(samples);
|
||
kon_audio::write_wav(&path_clone, &audio).map_err(|e| e.to_string())
|
||
})
|
||
.await
|
||
.map_err(|e| e.to_string())??;
|
||
|
||
Ok(path.to_string_lossy().to_string())
|
||
}
|
||
|
||
/// Save PCM f32 samples as a WAV file. Returns the file path.
|
||
#[tauri::command]
|
||
pub async fn save_audio(
|
||
app: tauri::AppHandle,
|
||
samples: Vec<f32>,
|
||
output_folder: Option<String>,
|
||
) -> Result<String, String> {
|
||
persist_audio_samples(&app, samples, output_folder).await
|
||
}
|