audio: fix mic capture — skip monitor sources, validate by RMS, add device picker

Day 1 of the upgrade plan (output/reports/kon-upgrade-plan-2026-04-17.md
in the CORBEL workspace). Fixes the HANDOVER.md blocker: native live
transcription was capturing silence because PulseAudio/PipeWire monitor
sources (speaker loopback) were winning the device-selection race —
they deliver zero-valued bytes that satisfied the original
"any device that produces data within 350ms" check.

WHAT CHANGED:

crates/audio/src/capture.rs (rewrite):
- New `DeviceInfo` struct (serde-derived) for the Settings device picker
- New `MicrophoneCapture::list_devices()` enumerates inputs with metadata
- New `MicrophoneCapture::start_with_device(name)` for explicit selection
- Refactored `start()` with monitor-source filtering by name pattern
  (.monitor suffix, "Monitor of " prefix, "loopback" substring) and
  RMS-energy validation in a 350ms window
- Two-pass selection: real inputs first, monitor sources only as
  last-resort fallback with explicit warning log
- Drop counter (Arc<AtomicU64>) tracks chunks dropped by `try_send`
  failure under load — Codex review caught this as a silent-failure risk
- `dropped_chunks()` accessor for the live session
- Verbose tracing at every step for diagnosis
- Unit test for monitor-name detection
- `cargo check -p kon-audio` passes clean

crates/audio/src/lib.rs:
- Re-export `DeviceInfo`

crates/audio/Cargo.toml:
- Add serde dependency (for DeviceInfo derives)

src-tauri/src/commands/audio.rs:
- New `list_audio_devices` Tauri command (returns Vec<DeviceInfo>)

src-tauri/src/lib.rs:
- Register `list_audio_devices` in invoke_handler

src/lib/pages/SettingsPage.svelte:
- New "Audio" section with microphone picker dropdown
- Auto-populates on mount via `list_audio_devices`
- Refresh button + clear messaging about monitor sources
- Likely-monitor entries marked disabled in the dropdown
- Auto mode is the default (empty string in settings.microphoneDevice)

src/lib/stores/page.svelte.js:
- New `microphoneDevice` field in defaults (empty = auto-select)

NEXT STEPS (per the upgrade plan):
- Wire `microphoneDevice` from settings into `MicrophoneCapture::start_with_device`
  in the live and standalone capture paths (currently both still call
  the auto-selecting `start()`)
- Test on real hardware (Wayland + multiple input devices)
- Codex sanity-check of this diff is running in parallel; addendum to
  follow if anything substantive comes back

Refs: /home/jake/Documents/CORBEL-Projects/kon/HANDOVER.md
      output/reports/kon-upgrade-plan-2026-04-17.md (CORBEL workspace)
This commit is contained in:
2026-04-17 12:43:13 +01:00
parent fdc4a3cba5
commit 96980c7d5c
7 changed files with 805 additions and 103 deletions

View File

@@ -1,13 +1,183 @@
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use tauri::Manager;
use tauri::{Emitter, Manager};
use tokio::sync::mpsc as tokio_mpsc;
use kon_audio::{DeviceInfo, MicrophoneCapture};
use kon_core::constants::WHISPER_SAMPLE_RATE;
use kon_core::types::AudioSamples;
/// Save PCM f32 samples as a WAV file. Returns the file path.
/// Enumerate every input device available to cpal, with metadata for the
/// Settings device-picker UI. Includes a flag for likely PulseAudio /
/// PipeWire monitor sources so the UI can warn the user.
#[tauri::command]
pub async fn save_audio(
pub async fn list_audio_devices() -> Result<Vec<DeviceInfo>, String> {
tokio::task::spawn_blocking(MicrophoneCapture::list_devices)
.await
.map_err(|e| format!("join error: {e}"))?
.map_err(|e| e.to_string())
}
/// Shared state for native microphone capture.
pub struct NativeCaptureState {
/// Stop signal sender — dropping this stops the accumulator task.
stop_tx: Mutex<Option<tokio_mpsc::Sender<()>>>,
/// All captured samples (16kHz mono) for save_audio.
all_samples: Arc<Mutex<Vec<f32>>>,
}
impl NativeCaptureState {
pub fn new() -> Self {
Self {
stop_tx: Mutex::new(None),
all_samples: Arc::new(Mutex::new(Vec::new())),
}
}
}
/// Start native microphone capture via cpal.
/// Streams 16kHz mono PCM chunks to the frontend via `native-pcm` events.
#[tauri::command]
pub async fn start_native_capture(
app: tauri::AppHandle,
state: tauri::State<'_, NativeCaptureState>,
) -> Result<(), String> {
eprintln!("[native-capture] start_native_capture called");
// Stop any existing capture
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
drop(tx);
}
let (capture, rx) = MicrophoneCapture::start().map_err(|e| {
eprintln!("[native-capture] MicrophoneCapture::start failed: {e}");
e.to_string()
})?;
eprintln!("[native-capture] cpal capture started successfully");
// Wrap capture in Arc<Mutex> so it can be moved into the blocking task
let capture = Arc::new(Mutex::new(Some(capture)));
let capture_clone = capture.clone();
let all_samples = state.all_samples.clone();
all_samples.lock().unwrap().clear();
let (stop_tx, mut stop_rx) = tokio_mpsc::channel::<()>(1);
*state.stop_tx.lock().unwrap() = Some(stop_tx);
let all_samples_clone = all_samples.clone();
// Spawn a task that reads cpal chunks, downsamples to 16kHz mono,
// and emits events to the frontend
tokio::spawn(async move {
let mut pcm_buffer: Vec<f32> = Vec::new();
let chunk_size = 8000_usize; // ~0.5s at 16kHz
loop {
// Check for stop signal (non-blocking)
if stop_rx.try_recv().is_ok() {
break;
}
// Drain available audio chunks from cpal (non-blocking)
let mut got_data = false;
while let Ok(chunk) = rx.try_recv() {
got_data = true;
let sample_rate = chunk.sample_rate;
let channels = chunk.channels as usize;
// Downmix to mono if stereo
let mono: Vec<f32> = if channels > 1 {
chunk
.samples
.chunks(channels)
.map(|frame| frame.iter().sum::<f32>() / channels as f32)
.collect()
} else {
chunk.samples
};
// Downsample to 16kHz using simple decimation
// (acceptable quality for speech — same approach as pcm-processor.js)
let ratio = sample_rate as f64 / WHISPER_SAMPLE_RATE as f64;
if (ratio - 1.0).abs() < 0.01 {
pcm_buffer.extend_from_slice(&mono);
} else {
let mut pos: f64 = 0.0;
for &s in &mono {
pos += 1.0;
if pos >= ratio {
pcm_buffer.push(s);
pos -= ratio;
}
}
}
}
// Emit chunks to frontend when we have enough
while pcm_buffer.len() >= chunk_size {
let chunk: Vec<f32> = pcm_buffer.drain(..chunk_size).collect();
// Store for save_audio
if let Ok(mut all) = all_samples_clone.lock() {
all.extend_from_slice(&chunk);
}
let _ = app.emit("native-pcm", serde_json::json!({
"samples": chunk,
}));
}
if !got_data {
// Avoid busy-spinning when no audio data is available
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
}
// Emit any remaining samples
if !pcm_buffer.is_empty() {
if let Ok(mut all) = all_samples_clone.lock() {
all.extend_from_slice(&pcm_buffer);
}
let _ = app.emit("native-pcm", serde_json::json!({
"samples": pcm_buffer,
}));
}
// Drop the capture to stop the cpal stream
if let Ok(mut cap) = capture_clone.lock() {
cap.take();
}
});
Ok(())
}
/// Stop native microphone capture. Returns all captured samples (16kHz mono).
#[tauri::command]
pub async fn stop_native_capture(
state: tauri::State<'_, NativeCaptureState>,
) -> Result<Vec<f32>, String> {
// Extract the stop sender without holding the guard across an await
let stop_tx = state.stop_tx.lock().unwrap().take();
if let Some(tx) = stop_tx {
let _ = tx.send(()).await;
}
// Brief delay to let the accumulator flush
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let samples = {
let mut all = state.all_samples.lock().unwrap();
std::mem::take(&mut *all)
};
Ok(samples)
}
pub async fn persist_audio_samples(
app: &tauri::AppHandle,
samples: Vec<f32>,
output_folder: Option<String>,
) -> Result<String, String> {
@@ -47,3 +217,13 @@ pub async fn save_audio(
Ok(path.to_string_lossy().to_string())
}
/// Save PCM f32 samples as a WAV file. Returns the file path.
#[tauri::command]
pub async fn save_audio(
app: tauri::AppHandle,
samples: Vec<f32>,
output_folder: Option<String>,
) -> Result<String, String> {
persist_audio_samples(&app, samples, output_folder).await
}

View File

@@ -2,6 +2,7 @@ mod commands;
mod tray;
use std::sync::Arc;
use std::time::Instant;
use sqlx::SqlitePool;
use tauri::Manager;
@@ -47,6 +48,7 @@ fn build_preferences_script(prefs_json: Option<String>) -> String {
if (a.fontSize) el.style.setProperty('--font-size-body', a.fontSize + 'px');
if (a.letterSpacing != null) el.style.setProperty('--letter-spacing-body', a.letterSpacing + 'em');
if (a.lineHeight) el.style.setProperty('--line-height-body', String(a.lineHeight));
if (a.transcriptSize) el.style.setProperty('--text-transcript', a.transcriptSize + 'px');
if (a.bionicReading) el.dataset.bionicReading = 'true';
if (a.reduceMotion === 'on' || (a.reduceMotion === 'system' && window.matchMedia('(prefers-reduced-motion: reduce)').matches)) el.dataset.reduceMotion = 'true';
}}
@@ -75,15 +77,19 @@ pub fn run() {
.setup(|app| {
// Initialise database (blocking in setup — runs once at startup)
let db_path = database_path();
let t0 = Instant::now();
let db = tauri::async_runtime::block_on(async {
init_db(&db_path).await
})
.map_err(|e| Box::new(e) as Box<dyn std::error::Error>)?;
eprintln!("[startup] DB init: {:?}", t0.elapsed());
// Load saved preferences for webview injection
let t1 = Instant::now();
let prefs_json = tauri::async_runtime::block_on(async {
get_setting(&db, "kon_preferences").await.unwrap_or(None)
});
eprintln!("[startup] Preferences load: {:?}", t1.elapsed());
let init_script = build_preferences_script(prefs_json);
// Apply preferences to the main window (defined in tauri.conf.json)
@@ -136,6 +142,8 @@ pub fn run() {
app.manage(PreferencesScript(init_script));
app.manage(commands::hotkey::HotkeyState::new());
app.manage(commands::audio::NativeCaptureState::new());
app.manage(commands::live::LiveTranscriptionState::default());
app.manage(AppState {
whisper_engine: Arc::new(LocalEngine::new(
@@ -162,6 +170,7 @@ pub fn run() {
commands::models::list_models,
commands::models::load_model,
commands::models::check_engine,
commands::models::get_runtime_capabilities,
// Parakeet model management
commands::models::download_parakeet_model,
commands::models::check_parakeet_model,
@@ -174,6 +183,11 @@ pub fn run() {
commands::transcription::transcribe_pcm_parakeet,
// Audio
commands::audio::save_audio,
commands::audio::start_native_capture,
commands::audio::stop_native_capture,
commands::audio::list_audio_devices,
commands::live::start_live_transcription_session,
commands::live::stop_live_transcription_session,
// Windows
commands::windows::open_task_window,
commands::windows::open_viewer_window,