audio: fix mic capture — skip monitor sources, validate by RMS, add device picker
Day 1 of the upgrade plan (output/reports/kon-upgrade-plan-2026-04-17.md
in the CORBEL workspace). Fixes the HANDOVER.md blocker: native live
transcription was capturing silence because PulseAudio/PipeWire monitor
sources (speaker loopback) were winning the device-selection race —
they deliver zero-valued bytes that satisfied the original
"any device that produces data within 350ms" check.
WHAT CHANGED:
crates/audio/src/capture.rs (rewrite):
- New `DeviceInfo` struct (serde-derived) for the Settings device picker
- New `MicrophoneCapture::list_devices()` enumerates inputs with metadata
- New `MicrophoneCapture::start_with_device(name)` for explicit selection
- Refactored `start()` with monitor-source filtering by name pattern
(.monitor suffix, "Monitor of " prefix, "loopback" substring) and
RMS-energy validation in a 350ms window
- Two-pass selection: real inputs first, monitor sources only as
last-resort fallback with explicit warning log
- Drop counter (Arc<AtomicU64>) tracks chunks dropped by `try_send`
failure under load — Codex review caught this as a silent-failure risk
- `dropped_chunks()` accessor for the live session
- Verbose tracing at every step for diagnosis
- Unit test for monitor-name detection
- `cargo check -p kon-audio` passes clean
crates/audio/src/lib.rs:
- Re-export `DeviceInfo`
crates/audio/Cargo.toml:
- Add serde dependency (for DeviceInfo derives)
src-tauri/src/commands/audio.rs:
- New `list_audio_devices` Tauri command (returns Vec<DeviceInfo>)
src-tauri/src/lib.rs:
- Register `list_audio_devices` in invoke_handler
src/lib/pages/SettingsPage.svelte:
- New "Audio" section with microphone picker dropdown
- Auto-populates on mount via `list_audio_devices`
- Refresh button + clear messaging about monitor sources
- Likely-monitor entries marked disabled in the dropdown
- Auto mode is the default (empty string in settings.microphoneDevice)
src/lib/stores/page.svelte.js:
- New `microphoneDevice` field in defaults (empty = auto-select)
NEXT STEPS (per the upgrade plan):
- Wire `microphoneDevice` from settings into `MicrophoneCapture::start_with_device`
in the live and standalone capture paths (currently both still call
the auto-selecting `start()`)
- Test on real hardware (Wayland + multiple input devices)
- Codex sanity-check of this diff is running in parallel; addendum to
follow if anything substantive comes back
Refs: /home/jake/Documents/CORBEL-Projects/kon/HANDOVER.md
output/reports/kon-upgrade-plan-2026-04-17.md (CORBEL workspace)
This commit is contained in:
@@ -1,13 +1,183 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use tauri::Manager;
|
||||
use tauri::{Emitter, Manager};
|
||||
use tokio::sync::mpsc as tokio_mpsc;
|
||||
|
||||
use kon_audio::{DeviceInfo, MicrophoneCapture};
|
||||
use kon_core::constants::WHISPER_SAMPLE_RATE;
|
||||
use kon_core::types::AudioSamples;
|
||||
|
||||
/// Save PCM f32 samples as a WAV file. Returns the file path.
|
||||
/// Enumerate every input device available to cpal, with metadata for the
|
||||
/// Settings device-picker UI. Includes a flag for likely PulseAudio /
|
||||
/// PipeWire monitor sources so the UI can warn the user.
|
||||
#[tauri::command]
|
||||
pub async fn save_audio(
|
||||
pub async fn list_audio_devices() -> Result<Vec<DeviceInfo>, String> {
|
||||
tokio::task::spawn_blocking(MicrophoneCapture::list_devices)
|
||||
.await
|
||||
.map_err(|e| format!("join error: {e}"))?
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// Shared state for native microphone capture.
|
||||
pub struct NativeCaptureState {
|
||||
/// Stop signal sender — dropping this stops the accumulator task.
|
||||
stop_tx: Mutex<Option<tokio_mpsc::Sender<()>>>,
|
||||
/// All captured samples (16kHz mono) for save_audio.
|
||||
all_samples: Arc<Mutex<Vec<f32>>>,
|
||||
}
|
||||
|
||||
impl NativeCaptureState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
stop_tx: Mutex::new(None),
|
||||
all_samples: Arc::new(Mutex::new(Vec::new())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Start native microphone capture via cpal.
|
||||
/// Streams 16kHz mono PCM chunks to the frontend via `native-pcm` events.
|
||||
#[tauri::command]
|
||||
pub async fn start_native_capture(
|
||||
app: tauri::AppHandle,
|
||||
state: tauri::State<'_, NativeCaptureState>,
|
||||
) -> Result<(), String> {
|
||||
eprintln!("[native-capture] start_native_capture called");
|
||||
|
||||
// Stop any existing capture
|
||||
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
|
||||
drop(tx);
|
||||
}
|
||||
|
||||
let (capture, rx) = MicrophoneCapture::start().map_err(|e| {
|
||||
eprintln!("[native-capture] MicrophoneCapture::start failed: {e}");
|
||||
e.to_string()
|
||||
})?;
|
||||
eprintln!("[native-capture] cpal capture started successfully");
|
||||
|
||||
// Wrap capture in Arc<Mutex> so it can be moved into the blocking task
|
||||
let capture = Arc::new(Mutex::new(Some(capture)));
|
||||
let capture_clone = capture.clone();
|
||||
|
||||
let all_samples = state.all_samples.clone();
|
||||
all_samples.lock().unwrap().clear();
|
||||
|
||||
let (stop_tx, mut stop_rx) = tokio_mpsc::channel::<()>(1);
|
||||
*state.stop_tx.lock().unwrap() = Some(stop_tx);
|
||||
|
||||
let all_samples_clone = all_samples.clone();
|
||||
|
||||
// Spawn a task that reads cpal chunks, downsamples to 16kHz mono,
|
||||
// and emits events to the frontend
|
||||
tokio::spawn(async move {
|
||||
let mut pcm_buffer: Vec<f32> = Vec::new();
|
||||
let chunk_size = 8000_usize; // ~0.5s at 16kHz
|
||||
|
||||
loop {
|
||||
// Check for stop signal (non-blocking)
|
||||
if stop_rx.try_recv().is_ok() {
|
||||
break;
|
||||
}
|
||||
|
||||
// Drain available audio chunks from cpal (non-blocking)
|
||||
let mut got_data = false;
|
||||
while let Ok(chunk) = rx.try_recv() {
|
||||
got_data = true;
|
||||
let sample_rate = chunk.sample_rate;
|
||||
let channels = chunk.channels as usize;
|
||||
|
||||
// Downmix to mono if stereo
|
||||
let mono: Vec<f32> = if channels > 1 {
|
||||
chunk
|
||||
.samples
|
||||
.chunks(channels)
|
||||
.map(|frame| frame.iter().sum::<f32>() / channels as f32)
|
||||
.collect()
|
||||
} else {
|
||||
chunk.samples
|
||||
};
|
||||
|
||||
// Downsample to 16kHz using simple decimation
|
||||
// (acceptable quality for speech — same approach as pcm-processor.js)
|
||||
let ratio = sample_rate as f64 / WHISPER_SAMPLE_RATE as f64;
|
||||
if (ratio - 1.0).abs() < 0.01 {
|
||||
pcm_buffer.extend_from_slice(&mono);
|
||||
} else {
|
||||
let mut pos: f64 = 0.0;
|
||||
for &s in &mono {
|
||||
pos += 1.0;
|
||||
if pos >= ratio {
|
||||
pcm_buffer.push(s);
|
||||
pos -= ratio;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Emit chunks to frontend when we have enough
|
||||
while pcm_buffer.len() >= chunk_size {
|
||||
let chunk: Vec<f32> = pcm_buffer.drain(..chunk_size).collect();
|
||||
|
||||
// Store for save_audio
|
||||
if let Ok(mut all) = all_samples_clone.lock() {
|
||||
all.extend_from_slice(&chunk);
|
||||
}
|
||||
|
||||
let _ = app.emit("native-pcm", serde_json::json!({
|
||||
"samples": chunk,
|
||||
}));
|
||||
}
|
||||
|
||||
if !got_data {
|
||||
// Avoid busy-spinning when no audio data is available
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
// Emit any remaining samples
|
||||
if !pcm_buffer.is_empty() {
|
||||
if let Ok(mut all) = all_samples_clone.lock() {
|
||||
all.extend_from_slice(&pcm_buffer);
|
||||
}
|
||||
let _ = app.emit("native-pcm", serde_json::json!({
|
||||
"samples": pcm_buffer,
|
||||
}));
|
||||
}
|
||||
|
||||
// Drop the capture to stop the cpal stream
|
||||
if let Ok(mut cap) = capture_clone.lock() {
|
||||
cap.take();
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop native microphone capture. Returns all captured samples (16kHz mono).
|
||||
#[tauri::command]
|
||||
pub async fn stop_native_capture(
|
||||
state: tauri::State<'_, NativeCaptureState>,
|
||||
) -> Result<Vec<f32>, String> {
|
||||
// Extract the stop sender without holding the guard across an await
|
||||
let stop_tx = state.stop_tx.lock().unwrap().take();
|
||||
if let Some(tx) = stop_tx {
|
||||
let _ = tx.send(()).await;
|
||||
}
|
||||
|
||||
// Brief delay to let the accumulator flush
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
let samples = {
|
||||
let mut all = state.all_samples.lock().unwrap();
|
||||
std::mem::take(&mut *all)
|
||||
};
|
||||
|
||||
Ok(samples)
|
||||
}
|
||||
|
||||
pub async fn persist_audio_samples(
|
||||
app: &tauri::AppHandle,
|
||||
samples: Vec<f32>,
|
||||
output_folder: Option<String>,
|
||||
) -> Result<String, String> {
|
||||
@@ -47,3 +217,13 @@ pub async fn save_audio(
|
||||
|
||||
Ok(path.to_string_lossy().to_string())
|
||||
}
|
||||
|
||||
/// Save PCM f32 samples as a WAV file. Returns the file path.
|
||||
#[tauri::command]
|
||||
pub async fn save_audio(
|
||||
app: tauri::AppHandle,
|
||||
samples: Vec<f32>,
|
||||
output_folder: Option<String>,
|
||||
) -> Result<String, String> {
|
||||
persist_audio_samples(&app, samples, output_folder).await
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user