audio: fix mic capture — skip monitor sources, validate by RMS, add device picker
Day 1 of the upgrade plan (output/reports/kon-upgrade-plan-2026-04-17.md
in the CORBEL workspace). Fixes the HANDOVER.md blocker: native live
transcription was capturing silence because PulseAudio/PipeWire monitor
sources (speaker loopback) were winning the device-selection race —
they deliver zero-valued bytes that satisfied the original
"any device that produces data within 350ms" check.
WHAT CHANGED:
crates/audio/src/capture.rs (rewrite):
- New `DeviceInfo` struct (serde-derived) for the Settings device picker
- New `MicrophoneCapture::list_devices()` enumerates inputs with metadata
- New `MicrophoneCapture::start_with_device(name)` for explicit selection
- Refactored `start()` with monitor-source filtering by name pattern
(.monitor suffix, "Monitor of " prefix, "loopback" substring) and
RMS-energy validation in a 350ms window
- Two-pass selection: real inputs first, monitor sources only as
last-resort fallback with explicit warning log
- Drop counter (Arc<AtomicU64>) tracks chunks dropped by `try_send`
failure under load — Codex review caught this as a silent-failure risk
- `dropped_chunks()` accessor for the live session
- Verbose tracing at every step for diagnosis
- Unit test for monitor-name detection
- `cargo check -p kon-audio` passes clean
crates/audio/src/lib.rs:
- Re-export `DeviceInfo`
crates/audio/Cargo.toml:
- Add serde dependency (for DeviceInfo derives)
src-tauri/src/commands/audio.rs:
- New `list_audio_devices` Tauri command (returns Vec<DeviceInfo>)
src-tauri/src/lib.rs:
- Register `list_audio_devices` in invoke_handler
src/lib/pages/SettingsPage.svelte:
- New "Audio" section with microphone picker dropdown
- Auto-populates on mount via `list_audio_devices`
- Refresh button + clear messaging about monitor sources
- Likely-monitor entries marked disabled in the dropdown
- Auto mode is the default (empty string in settings.microphoneDevice)
src/lib/stores/page.svelte.js:
- New `microphoneDevice` field in defaults (empty = auto-select)
NEXT STEPS (per the upgrade plan):
- Wire `microphoneDevice` from settings into `MicrophoneCapture::start_with_device`
in the live and standalone capture paths (currently both still call
the auto-selecting `start()`)
- Test on real hardware (Wayland + multiple input devices)
- Codex sanity-check of this diff is running in parallel; addendum to
follow if anything substantive comes back
Refs: /home/jake/Documents/CORBEL-Projects/kon/HANDOVER.md
output/reports/kon-upgrade-plan-2026-04-17.md (CORBEL workspace)
This commit is contained in:
@@ -1,13 +1,183 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use tauri::Manager;
|
||||
use tauri::{Emitter, Manager};
|
||||
use tokio::sync::mpsc as tokio_mpsc;
|
||||
|
||||
use kon_audio::{DeviceInfo, MicrophoneCapture};
|
||||
use kon_core::constants::WHISPER_SAMPLE_RATE;
|
||||
use kon_core::types::AudioSamples;
|
||||
|
||||
/// Save PCM f32 samples as a WAV file. Returns the file path.
|
||||
/// Enumerate every input device available to cpal, with metadata for the
|
||||
/// Settings device-picker UI. Includes a flag for likely PulseAudio /
|
||||
/// PipeWire monitor sources so the UI can warn the user.
|
||||
#[tauri::command]
|
||||
pub async fn save_audio(
|
||||
pub async fn list_audio_devices() -> Result<Vec<DeviceInfo>, String> {
|
||||
tokio::task::spawn_blocking(MicrophoneCapture::list_devices)
|
||||
.await
|
||||
.map_err(|e| format!("join error: {e}"))?
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// Shared state for native microphone capture.
|
||||
pub struct NativeCaptureState {
|
||||
/// Stop signal sender — dropping this stops the accumulator task.
|
||||
stop_tx: Mutex<Option<tokio_mpsc::Sender<()>>>,
|
||||
/// All captured samples (16kHz mono) for save_audio.
|
||||
all_samples: Arc<Mutex<Vec<f32>>>,
|
||||
}
|
||||
|
||||
impl NativeCaptureState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
stop_tx: Mutex::new(None),
|
||||
all_samples: Arc::new(Mutex::new(Vec::new())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Start native microphone capture via cpal.
|
||||
/// Streams 16kHz mono PCM chunks to the frontend via `native-pcm` events.
|
||||
#[tauri::command]
|
||||
pub async fn start_native_capture(
|
||||
app: tauri::AppHandle,
|
||||
state: tauri::State<'_, NativeCaptureState>,
|
||||
) -> Result<(), String> {
|
||||
eprintln!("[native-capture] start_native_capture called");
|
||||
|
||||
// Stop any existing capture
|
||||
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
|
||||
drop(tx);
|
||||
}
|
||||
|
||||
let (capture, rx) = MicrophoneCapture::start().map_err(|e| {
|
||||
eprintln!("[native-capture] MicrophoneCapture::start failed: {e}");
|
||||
e.to_string()
|
||||
})?;
|
||||
eprintln!("[native-capture] cpal capture started successfully");
|
||||
|
||||
// Wrap capture in Arc<Mutex> so it can be moved into the blocking task
|
||||
let capture = Arc::new(Mutex::new(Some(capture)));
|
||||
let capture_clone = capture.clone();
|
||||
|
||||
let all_samples = state.all_samples.clone();
|
||||
all_samples.lock().unwrap().clear();
|
||||
|
||||
let (stop_tx, mut stop_rx) = tokio_mpsc::channel::<()>(1);
|
||||
*state.stop_tx.lock().unwrap() = Some(stop_tx);
|
||||
|
||||
let all_samples_clone = all_samples.clone();
|
||||
|
||||
// Spawn a task that reads cpal chunks, downsamples to 16kHz mono,
|
||||
// and emits events to the frontend
|
||||
tokio::spawn(async move {
|
||||
let mut pcm_buffer: Vec<f32> = Vec::new();
|
||||
let chunk_size = 8000_usize; // ~0.5s at 16kHz
|
||||
|
||||
loop {
|
||||
// Check for stop signal (non-blocking)
|
||||
if stop_rx.try_recv().is_ok() {
|
||||
break;
|
||||
}
|
||||
|
||||
// Drain available audio chunks from cpal (non-blocking)
|
||||
let mut got_data = false;
|
||||
while let Ok(chunk) = rx.try_recv() {
|
||||
got_data = true;
|
||||
let sample_rate = chunk.sample_rate;
|
||||
let channels = chunk.channels as usize;
|
||||
|
||||
// Downmix to mono if stereo
|
||||
let mono: Vec<f32> = if channels > 1 {
|
||||
chunk
|
||||
.samples
|
||||
.chunks(channels)
|
||||
.map(|frame| frame.iter().sum::<f32>() / channels as f32)
|
||||
.collect()
|
||||
} else {
|
||||
chunk.samples
|
||||
};
|
||||
|
||||
// Downsample to 16kHz using simple decimation
|
||||
// (acceptable quality for speech — same approach as pcm-processor.js)
|
||||
let ratio = sample_rate as f64 / WHISPER_SAMPLE_RATE as f64;
|
||||
if (ratio - 1.0).abs() < 0.01 {
|
||||
pcm_buffer.extend_from_slice(&mono);
|
||||
} else {
|
||||
let mut pos: f64 = 0.0;
|
||||
for &s in &mono {
|
||||
pos += 1.0;
|
||||
if pos >= ratio {
|
||||
pcm_buffer.push(s);
|
||||
pos -= ratio;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Emit chunks to frontend when we have enough
|
||||
while pcm_buffer.len() >= chunk_size {
|
||||
let chunk: Vec<f32> = pcm_buffer.drain(..chunk_size).collect();
|
||||
|
||||
// Store for save_audio
|
||||
if let Ok(mut all) = all_samples_clone.lock() {
|
||||
all.extend_from_slice(&chunk);
|
||||
}
|
||||
|
||||
let _ = app.emit("native-pcm", serde_json::json!({
|
||||
"samples": chunk,
|
||||
}));
|
||||
}
|
||||
|
||||
if !got_data {
|
||||
// Avoid busy-spinning when no audio data is available
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
// Emit any remaining samples
|
||||
if !pcm_buffer.is_empty() {
|
||||
if let Ok(mut all) = all_samples_clone.lock() {
|
||||
all.extend_from_slice(&pcm_buffer);
|
||||
}
|
||||
let _ = app.emit("native-pcm", serde_json::json!({
|
||||
"samples": pcm_buffer,
|
||||
}));
|
||||
}
|
||||
|
||||
// Drop the capture to stop the cpal stream
|
||||
if let Ok(mut cap) = capture_clone.lock() {
|
||||
cap.take();
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop native microphone capture. Returns all captured samples (16kHz mono).
|
||||
#[tauri::command]
|
||||
pub async fn stop_native_capture(
|
||||
state: tauri::State<'_, NativeCaptureState>,
|
||||
) -> Result<Vec<f32>, String> {
|
||||
// Extract the stop sender without holding the guard across an await
|
||||
let stop_tx = state.stop_tx.lock().unwrap().take();
|
||||
if let Some(tx) = stop_tx {
|
||||
let _ = tx.send(()).await;
|
||||
}
|
||||
|
||||
// Brief delay to let the accumulator flush
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
let samples = {
|
||||
let mut all = state.all_samples.lock().unwrap();
|
||||
std::mem::take(&mut *all)
|
||||
};
|
||||
|
||||
Ok(samples)
|
||||
}
|
||||
|
||||
pub async fn persist_audio_samples(
|
||||
app: &tauri::AppHandle,
|
||||
samples: Vec<f32>,
|
||||
output_folder: Option<String>,
|
||||
) -> Result<String, String> {
|
||||
@@ -47,3 +217,13 @@ pub async fn save_audio(
|
||||
|
||||
Ok(path.to_string_lossy().to_string())
|
||||
}
|
||||
|
||||
/// Save PCM f32 samples as a WAV file. Returns the file path.
|
||||
#[tauri::command]
|
||||
pub async fn save_audio(
|
||||
app: tauri::AppHandle,
|
||||
samples: Vec<f32>,
|
||||
output_folder: Option<String>,
|
||||
) -> Result<String, String> {
|
||||
persist_audio_samples(&app, samples, output_folder).await
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ mod commands;
|
||||
mod tray;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use sqlx::SqlitePool;
|
||||
use tauri::Manager;
|
||||
@@ -47,6 +48,7 @@ fn build_preferences_script(prefs_json: Option<String>) -> String {
|
||||
if (a.fontSize) el.style.setProperty('--font-size-body', a.fontSize + 'px');
|
||||
if (a.letterSpacing != null) el.style.setProperty('--letter-spacing-body', a.letterSpacing + 'em');
|
||||
if (a.lineHeight) el.style.setProperty('--line-height-body', String(a.lineHeight));
|
||||
if (a.transcriptSize) el.style.setProperty('--text-transcript', a.transcriptSize + 'px');
|
||||
if (a.bionicReading) el.dataset.bionicReading = 'true';
|
||||
if (a.reduceMotion === 'on' || (a.reduceMotion === 'system' && window.matchMedia('(prefers-reduced-motion: reduce)').matches)) el.dataset.reduceMotion = 'true';
|
||||
}}
|
||||
@@ -75,15 +77,19 @@ pub fn run() {
|
||||
.setup(|app| {
|
||||
// Initialise database (blocking in setup — runs once at startup)
|
||||
let db_path = database_path();
|
||||
let t0 = Instant::now();
|
||||
let db = tauri::async_runtime::block_on(async {
|
||||
init_db(&db_path).await
|
||||
})
|
||||
.map_err(|e| Box::new(e) as Box<dyn std::error::Error>)?;
|
||||
eprintln!("[startup] DB init: {:?}", t0.elapsed());
|
||||
|
||||
// Load saved preferences for webview injection
|
||||
let t1 = Instant::now();
|
||||
let prefs_json = tauri::async_runtime::block_on(async {
|
||||
get_setting(&db, "kon_preferences").await.unwrap_or(None)
|
||||
});
|
||||
eprintln!("[startup] Preferences load: {:?}", t1.elapsed());
|
||||
let init_script = build_preferences_script(prefs_json);
|
||||
|
||||
// Apply preferences to the main window (defined in tauri.conf.json)
|
||||
@@ -136,6 +142,8 @@ pub fn run() {
|
||||
app.manage(PreferencesScript(init_script));
|
||||
|
||||
app.manage(commands::hotkey::HotkeyState::new());
|
||||
app.manage(commands::audio::NativeCaptureState::new());
|
||||
app.manage(commands::live::LiveTranscriptionState::default());
|
||||
|
||||
app.manage(AppState {
|
||||
whisper_engine: Arc::new(LocalEngine::new(
|
||||
@@ -162,6 +170,7 @@ pub fn run() {
|
||||
commands::models::list_models,
|
||||
commands::models::load_model,
|
||||
commands::models::check_engine,
|
||||
commands::models::get_runtime_capabilities,
|
||||
// Parakeet model management
|
||||
commands::models::download_parakeet_model,
|
||||
commands::models::check_parakeet_model,
|
||||
@@ -174,6 +183,11 @@ pub fn run() {
|
||||
commands::transcription::transcribe_pcm_parakeet,
|
||||
// Audio
|
||||
commands::audio::save_audio,
|
||||
commands::audio::start_native_capture,
|
||||
commands::audio::stop_native_capture,
|
||||
commands::audio::list_audio_devices,
|
||||
commands::live::start_live_transcription_session,
|
||||
commands::live::stop_live_transcription_session,
|
||||
// Windows
|
||||
commands::windows::open_task_window,
|
||||
commands::windows::open_viewer_window,
|
||||
|
||||
Reference in New Issue
Block a user