audio: fix mic capture — skip monitor sources, validate by RMS, add device picker
Day 1 of the upgrade plan (output/reports/kon-upgrade-plan-2026-04-17.md
in the CORBEL workspace). Fixes the HANDOVER.md blocker: native live
transcription was capturing silence because PulseAudio/PipeWire monitor
sources (speaker loopback) were winning the device-selection race —
they deliver zero-valued bytes that satisfied the original
"any device that produces data within 350ms" check.
WHAT CHANGED:
crates/audio/src/capture.rs (rewrite):
- New `DeviceInfo` struct (serde-derived) for the Settings device picker
- New `MicrophoneCapture::list_devices()` enumerates inputs with metadata
- New `MicrophoneCapture::start_with_device(name)` for explicit selection
- Refactored `start()` with monitor-source filtering by name pattern
(.monitor suffix, "Monitor of " prefix, "loopback" substring) and
RMS-energy validation in a 350ms window
- Two-pass selection: real inputs first, monitor sources only as
last-resort fallback with explicit warning log
- Drop counter (Arc<AtomicU64>) tracks chunks dropped by `try_send`
failure under load — Codex review caught this as a silent-failure risk
- `dropped_chunks()` accessor for the live session
- Verbose tracing at every step for diagnosis
- Unit test for monitor-name detection
- `cargo check -p kon-audio` passes clean
crates/audio/src/lib.rs:
- Re-export `DeviceInfo`
crates/audio/Cargo.toml:
- Add serde dependency (for DeviceInfo derives)
src-tauri/src/commands/audio.rs:
- New `list_audio_devices` Tauri command (returns Vec<DeviceInfo>)
src-tauri/src/lib.rs:
- Register `list_audio_devices` in invoke_handler
src/lib/pages/SettingsPage.svelte:
- New "Audio" section with microphone picker dropdown
- Auto-populates on mount via `list_audio_devices`
- Refresh button + clear messaging about monitor sources
- Likely-monitor entries marked disabled in the dropdown
- Auto mode is the default (empty string in settings.microphoneDevice)
src/lib/stores/page.svelte.js:
- New `microphoneDevice` field in defaults (empty = auto-select)
NEXT STEPS (per the upgrade plan):
- Wire `microphoneDevice` from settings into `MicrophoneCapture::start_with_device`
in the live and standalone capture paths (currently both still call
the auto-selecting `start()`)
- Test on real hardware (Wayland + multiple input devices)
- Codex sanity-check of this diff is running in parallel; addendum to
follow if anything substantive comes back
Refs: /home/jake/Documents/CORBEL-Projects/kon/HANDOVER.md
output/reports/kon-upgrade-plan-2026-04-17.md (CORBEL workspace)
This commit is contained in:
@@ -25,3 +25,6 @@ symphonia = { version = "0.5", features = ["mp3", "aac", "flac", "pcm", "vorbis"
|
|||||||
|
|
||||||
# Async runtime for threading
|
# Async runtime for threading
|
||||||
tokio = { version = "1", features = ["rt", "sync"] }
|
tokio = { version = "1", features = ["rt", "sync"] }
|
||||||
|
|
||||||
|
# Serde for DeviceInfo (returned across the Tauri boundary)
|
||||||
|
serde = { version = "1", features = ["derive"] }
|
||||||
|
|||||||
@@ -1,9 +1,25 @@
|
|||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::mpsc;
|
use std::sync::mpsc;
|
||||||
|
|
||||||
|
use cpal::{FromSample, Sample, SampleFormat, SizedSample};
|
||||||
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
|
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use kon_core::error::{KonError, Result};
|
use kon_core::error::{KonError, Result};
|
||||||
|
|
||||||
|
const AUDIO_CHANNEL_CAPACITY: usize = 32;
|
||||||
|
|
||||||
|
/// Validation window. We listen for this long and compute RMS to decide
|
||||||
|
/// whether the chosen device is delivering real audio (vs a silent monitor).
|
||||||
|
const DEVICE_VALIDATION_MS: u64 = 350;
|
||||||
|
|
||||||
|
/// Below this RMS amplitude (peak ±1.0 scale) the input is treated as
|
||||||
|
/// silence. PulseAudio/PipeWire monitor sources for an idle speaker
|
||||||
|
/// typically deliver dead-zero samples; real microphones yield ~0.0005+
|
||||||
|
/// even in a quiet room. Conservative floor: 1e-5.
|
||||||
|
const SILENCE_RMS_FLOOR: f32 = 1e-5;
|
||||||
|
|
||||||
/// A chunk of captured audio from the microphone.
|
/// A chunk of captured audio from the microphone.
|
||||||
pub struct AudioChunk {
|
pub struct AudioChunk {
|
||||||
pub samples: Vec<f32>,
|
pub samples: Vec<f32>,
|
||||||
@@ -11,53 +27,180 @@ pub struct AudioChunk {
|
|||||||
pub channels: u16,
|
pub channels: u16,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Public-facing description of an audio input device.
|
||||||
|
/// Returned by `list_devices()` and used by the UI device picker.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct DeviceInfo {
|
||||||
|
/// Device name as reported by cpal/the host.
|
||||||
|
pub name: String,
|
||||||
|
/// Default sample rate in Hz.
|
||||||
|
pub sample_rate: u32,
|
||||||
|
/// Default channel count.
|
||||||
|
pub channels: u16,
|
||||||
|
/// True if the device name matches a known monitor-source pattern
|
||||||
|
/// (PulseAudio/PipeWire loopback of speaker output).
|
||||||
|
pub is_likely_monitor: bool,
|
||||||
|
/// True if cpal reports this as the host's default input device.
|
||||||
|
pub is_default: bool,
|
||||||
|
}
|
||||||
|
|
||||||
/// Manages microphone capture via cpal.
|
/// Manages microphone capture via cpal.
|
||||||
/// Call `start()` to begin capturing, which returns a receiver for audio chunks.
|
|
||||||
/// Call `stop()` to end the stream.
|
|
||||||
pub struct MicrophoneCapture {
|
pub struct MicrophoneCapture {
|
||||||
stream: Option<cpal::Stream>,
|
stream: Option<cpal::Stream>,
|
||||||
|
/// Name of the device that is actually capturing.
|
||||||
|
pub device_name: String,
|
||||||
|
/// Counter incremented every time the capture callback drops a chunk
|
||||||
|
/// because the channel was full. Read via `dropped_chunks()`.
|
||||||
|
dropped_chunks: Arc<AtomicU64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MicrophoneCapture {
|
impl MicrophoneCapture {
|
||||||
/// Start capturing audio from the default input device.
|
/// Number of audio chunks dropped because the downstream channel was full
|
||||||
/// Returns a receiver that yields AudioChunks as they arrive.
|
/// since this capture started. Should stay at 0 in normal use; non-zero
|
||||||
|
/// indicates downstream backpressure or a stuck consumer.
|
||||||
|
pub fn dropped_chunks(&self) -> u64 {
|
||||||
|
self.dropped_chunks.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Enumerate every input device the host knows about, with the metadata
|
||||||
|
/// needed by the device-picker UI.
|
||||||
|
pub fn list_devices() -> Result<Vec<DeviceInfo>> {
|
||||||
|
let host = cpal::default_host();
|
||||||
|
let default_name = host
|
||||||
|
.default_input_device()
|
||||||
|
.and_then(|d| d.name().ok())
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let devices = host
|
||||||
|
.input_devices()
|
||||||
|
.map_err(|e| KonError::AudioCaptureFailed(format!("input_devices: {e}")))?;
|
||||||
|
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for device in devices {
|
||||||
|
let name = device.name().unwrap_or_else(|_| "<unnamed>".to_string());
|
||||||
|
let (sample_rate, channels) = match device.default_input_config() {
|
||||||
|
Ok(cfg) => (cfg.sample_rate(), cfg.channels() as u16),
|
||||||
|
Err(_) => (0, 0),
|
||||||
|
};
|
||||||
|
let is_likely_monitor = is_monitor_name(&name);
|
||||||
|
let is_default = !default_name.is_empty() && name == default_name;
|
||||||
|
out.push(DeviceInfo {
|
||||||
|
name,
|
||||||
|
sample_rate,
|
||||||
|
channels,
|
||||||
|
is_likely_monitor,
|
||||||
|
is_default,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Start capturing from the device whose name matches `device_name` exactly.
|
||||||
|
/// If no match is found, returns an error rather than silently falling back.
|
||||||
|
pub fn start_with_device(
|
||||||
|
device_name: &str,
|
||||||
|
) -> Result<(Self, mpsc::Receiver<AudioChunk>)> {
|
||||||
|
let host = cpal::default_host();
|
||||||
|
let devices = host
|
||||||
|
.input_devices()
|
||||||
|
.map_err(|e| KonError::AudioCaptureFailed(format!("input_devices: {e}")))?;
|
||||||
|
|
||||||
|
for device in devices {
|
||||||
|
let name = device.name().unwrap_or_default();
|
||||||
|
if name == device_name {
|
||||||
|
eprintln!("[kon-audio] start_with_device: opening explicit device '{name}'");
|
||||||
|
return open_and_validate(device, &name, /* require_audio = */ true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(KonError::AudioCaptureFailed(format!(
|
||||||
|
"Selected device '{device_name}' not found in current host enumeration. \
|
||||||
|
It may have been disconnected. Open Settings → Audio to pick another."
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Start capturing audio with auto-selection.
|
||||||
|
///
|
||||||
|
/// Selection rules:
|
||||||
|
/// 1. Try the host default input device first if it exists AND is not a monitor source.
|
||||||
|
/// 2. Otherwise, try non-monitor devices in enumeration order.
|
||||||
|
/// 3. Validate the chosen device by RMS energy (not just receipt of bytes) over
|
||||||
|
/// a short window — this is what defeats the "silent monitor source wins" bug.
|
||||||
|
/// 4. If no non-monitor device produces real audio, fall back to monitor sources
|
||||||
|
/// as a last resort (with a clear log line). Never accept dead silence.
|
||||||
pub fn start() -> Result<(Self, mpsc::Receiver<AudioChunk>)> {
|
pub fn start() -> Result<(Self, mpsc::Receiver<AudioChunk>)> {
|
||||||
let host = cpal::default_host();
|
let host = cpal::default_host();
|
||||||
let device = host.default_input_device().ok_or_else(|| {
|
let default_name = host
|
||||||
KonError::AudioCaptureFailed("No input device found".into())
|
.default_input_device()
|
||||||
})?;
|
.and_then(|d| d.name().ok())
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
let config = device.default_input_config().map_err(|e| {
|
let mut all_devices: Vec<cpal::Device> =
|
||||||
KonError::AudioCaptureFailed(format!("No input config: {e}"))
|
host.input_devices()
|
||||||
})?;
|
.map_err(|e| {
|
||||||
|
KonError::AudioCaptureFailed(format!("input_devices: {e}"))
|
||||||
|
})?
|
||||||
|
.collect();
|
||||||
|
|
||||||
let sample_rate = config.sample_rate();
|
// Sort: default first, then non-monitor, then monitor-as-last-resort.
|
||||||
let channels = config.channels() as u16;
|
all_devices.sort_by_key(|d| {
|
||||||
|
let n = d.name().unwrap_or_default();
|
||||||
|
let is_default = !default_name.is_empty() && n == default_name;
|
||||||
|
let is_monitor = is_monitor_name(&n);
|
||||||
|
// Smaller key = tried first.
|
||||||
|
match (is_default, is_monitor) {
|
||||||
|
(true, false) => 0, // default, real input
|
||||||
|
(false, false) => 1, // any other real input
|
||||||
|
(true, true) => 2, // default but is a monitor (very rare)
|
||||||
|
(false, true) => 3, // monitor source — last resort
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
let (tx, rx) = mpsc::channel::<AudioChunk>();
|
eprintln!(
|
||||||
|
"[kon-audio] start: enumerated {} input device(s) (default='{}')",
|
||||||
|
all_devices.len(),
|
||||||
|
default_name
|
||||||
|
);
|
||||||
|
|
||||||
let stream = device
|
// First pass: require real audio energy.
|
||||||
.build_input_stream(
|
for device in &all_devices {
|
||||||
&config.into(),
|
let name = device.name().unwrap_or_default();
|
||||||
move |data: &[f32], _info: &cpal::InputCallbackInfo| {
|
if is_monitor_name(&name) {
|
||||||
let _ = tx.send(AudioChunk {
|
continue; // Save monitor sources for second pass.
|
||||||
samples: data.to_vec(),
|
}
|
||||||
sample_rate,
|
match open_and_validate(device.clone(), &name, true) {
|
||||||
channels,
|
Ok(result) => return Ok(result),
|
||||||
});
|
Err(e) => {
|
||||||
},
|
eprintln!("[kon-audio] '{name}' rejected: {e}");
|
||||||
|err| eprintln!("audio capture error: {err}"),
|
}
|
||||||
None,
|
}
|
||||||
)
|
}
|
||||||
.map_err(|e| {
|
|
||||||
KonError::AudioCaptureFailed(format!("Build stream failed: {e}"))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
stream.play().map_err(|e| {
|
// Second pass: accept anything that delivers bytes (monitor sources
|
||||||
KonError::AudioCaptureFailed(format!("Stream play failed: {e}"))
|
// included). Better to capture from a monitor than fail entirely.
|
||||||
})?;
|
eprintln!(
|
||||||
|
"[kon-audio] no non-monitor mic produced audio; falling back to monitor/loopback sources"
|
||||||
|
);
|
||||||
|
for device in &all_devices {
|
||||||
|
let name = device.name().unwrap_or_default();
|
||||||
|
match open_and_validate(device.clone(), &name, false) {
|
||||||
|
Ok(result) => {
|
||||||
|
eprintln!(
|
||||||
|
"[kon-audio] FALLBACK: capturing from '{name}' (likely monitor source). \
|
||||||
|
Recordings may be silent or contain system audio."
|
||||||
|
);
|
||||||
|
return Ok(result);
|
||||||
|
}
|
||||||
|
Err(_) => continue,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Ok((Self { stream: Some(stream) }, rx))
|
Err(KonError::AudioCaptureFailed(
|
||||||
|
"No working microphone found. Check that an input device is connected, \
|
||||||
|
that PulseAudio/PipeWire is running, and that the app has microphone permission. \
|
||||||
|
Then open Settings → Audio to pick a device explicitly."
|
||||||
|
.into(),
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Stop capturing audio.
|
/// Stop capturing audio.
|
||||||
@@ -73,3 +216,164 @@ impl Drop for MicrophoneCapture {
|
|||||||
self.stop();
|
self.stop();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Heuristic: identify a PulseAudio/PipeWire monitor source by name.
|
||||||
|
/// Common patterns:
|
||||||
|
/// - ".monitor" suffix (PulseAudio convention)
|
||||||
|
/// - "Monitor of " prefix (longer human-readable name)
|
||||||
|
/// - "Loopback" anywhere (some PipeWire configurations)
|
||||||
|
fn is_monitor_name(name: &str) -> bool {
|
||||||
|
let lower = name.to_lowercase();
|
||||||
|
lower.ends_with(".monitor")
|
||||||
|
|| lower.starts_with("monitor of ")
|
||||||
|
|| lower.contains("monitor of ")
|
||||||
|
|| lower.contains("loopback")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Open the given device and validate it produces non-silent audio.
|
||||||
|
/// If `require_audio` is false, accept any data (used for monitor fallback).
|
||||||
|
fn open_and_validate(
|
||||||
|
device: cpal::Device,
|
||||||
|
name: &str,
|
||||||
|
require_audio: bool,
|
||||||
|
) -> Result<(MicrophoneCapture, mpsc::Receiver<AudioChunk>)> {
|
||||||
|
let config = device.default_input_config().map_err(|e| {
|
||||||
|
KonError::AudioCaptureFailed(format!("default_input_config: {e}"))
|
||||||
|
})?;
|
||||||
|
let sample_rate = config.sample_rate();
|
||||||
|
let channels = config.channels() as u16;
|
||||||
|
let format = config.sample_format();
|
||||||
|
|
||||||
|
eprintln!(
|
||||||
|
"[kon-audio] trying '{name}' ({sr}Hz, {ch}ch, {fmt:?})",
|
||||||
|
sr = sample_rate,
|
||||||
|
ch = channels,
|
||||||
|
fmt = format
|
||||||
|
);
|
||||||
|
|
||||||
|
let (tx, rx) = mpsc::sync_channel::<AudioChunk>(AUDIO_CHANNEL_CAPACITY);
|
||||||
|
let requeue_tx = tx.clone();
|
||||||
|
let dropped_chunks = Arc::new(AtomicU64::new(0));
|
||||||
|
|
||||||
|
let stream = match format {
|
||||||
|
SampleFormat::F32 => build_input_stream::<f32>(&device, &config, sample_rate, channels, tx, dropped_chunks.clone()),
|
||||||
|
SampleFormat::I16 => build_input_stream::<i16>(&device, &config, sample_rate, channels, tx, dropped_chunks.clone()),
|
||||||
|
SampleFormat::U16 => build_input_stream::<u16>(&device, &config, sample_rate, channels, tx, dropped_chunks.clone()),
|
||||||
|
other => {
|
||||||
|
return Err(KonError::AudioCaptureFailed(format!(
|
||||||
|
"unsupported sample format {other:?}"
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.map_err(|e| KonError::AudioCaptureFailed(format!("build_input_stream: {e}")))?;
|
||||||
|
|
||||||
|
stream
|
||||||
|
.play()
|
||||||
|
.map_err(|e| KonError::AudioCaptureFailed(format!("stream.play: {e}")))?;
|
||||||
|
|
||||||
|
// Validation window: collect chunks for DEVICE_VALIDATION_MS, compute RMS.
|
||||||
|
let deadline = std::time::Instant::now()
|
||||||
|
+ std::time::Duration::from_millis(DEVICE_VALIDATION_MS);
|
||||||
|
let mut collected: Vec<AudioChunk> = Vec::new();
|
||||||
|
let mut total_samples = 0_usize;
|
||||||
|
let mut sum_sq: f64 = 0.0;
|
||||||
|
|
||||||
|
while std::time::Instant::now() < deadline {
|
||||||
|
let remaining = deadline.saturating_duration_since(std::time::Instant::now());
|
||||||
|
if remaining.is_zero() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
match rx.recv_timeout(remaining) {
|
||||||
|
Ok(chunk) => {
|
||||||
|
for &s in &chunk.samples {
|
||||||
|
sum_sq += (s as f64) * (s as f64);
|
||||||
|
}
|
||||||
|
total_samples += chunk.samples.len();
|
||||||
|
collected.push(chunk);
|
||||||
|
}
|
||||||
|
Err(_) => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if total_samples == 0 {
|
||||||
|
return Err(KonError::AudioCaptureFailed(
|
||||||
|
"device delivered zero samples in validation window".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let rms = (sum_sq / total_samples as f64).sqrt() as f32;
|
||||||
|
eprintln!(
|
||||||
|
"[kon-audio] '{name}' validation: {samples} samples, rms={rms:.6}",
|
||||||
|
samples = total_samples
|
||||||
|
);
|
||||||
|
|
||||||
|
if require_audio && rms < SILENCE_RMS_FLOOR {
|
||||||
|
return Err(KonError::AudioCaptureFailed(format!(
|
||||||
|
"device produced silence (rms={rms:.6} below floor {SILENCE_RMS_FLOOR:.6})"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-queue the collected chunks so downstream gets them.
|
||||||
|
for chunk in collected {
|
||||||
|
let _ = requeue_tx.try_send(chunk);
|
||||||
|
}
|
||||||
|
|
||||||
|
eprintln!("[kon-audio] selected microphone: '{name}'");
|
||||||
|
Ok((
|
||||||
|
MicrophoneCapture {
|
||||||
|
stream: Some(stream),
|
||||||
|
device_name: name.to_string(),
|
||||||
|
dropped_chunks,
|
||||||
|
},
|
||||||
|
rx,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_input_stream<T>(
|
||||||
|
device: &cpal::Device,
|
||||||
|
supported_config: &cpal::SupportedStreamConfig,
|
||||||
|
sample_rate: u32,
|
||||||
|
channels: u16,
|
||||||
|
tx: mpsc::SyncSender<AudioChunk>,
|
||||||
|
dropped_chunks: Arc<AtomicU64>,
|
||||||
|
) -> std::result::Result<cpal::Stream, cpal::BuildStreamError>
|
||||||
|
where
|
||||||
|
T: Sample + SizedSample,
|
||||||
|
f32: FromSample<T>,
|
||||||
|
{
|
||||||
|
let config: cpal::StreamConfig = supported_config.clone().into();
|
||||||
|
device.build_input_stream(
|
||||||
|
&config,
|
||||||
|
move |data: &[T], _| {
|
||||||
|
let samples: Vec<f32> = data.iter().copied().map(f32::from_sample).collect();
|
||||||
|
let chunk = AudioChunk {
|
||||||
|
samples,
|
||||||
|
sample_rate,
|
||||||
|
channels,
|
||||||
|
};
|
||||||
|
// try_send fails if the channel is full. Track that explicitly
|
||||||
|
// rather than swallowing it — Codex review 2026/04/17 caught
|
||||||
|
// this as a silent-failure risk under sustained load.
|
||||||
|
if tx.try_send(chunk).is_err() {
|
||||||
|
dropped_chunks.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|err| eprintln!("[kon-audio] capture error: {err}"),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn monitor_pattern_detection() {
|
||||||
|
assert!(is_monitor_name("alsa_output.pci-0000_00_1f.3.analog-stereo.monitor"));
|
||||||
|
assert!(is_monitor_name("Monitor of Built-in Audio Analog Stereo"));
|
||||||
|
assert!(is_monitor_name("Some Loopback Device"));
|
||||||
|
assert!(!is_monitor_name("Blue Yeti USB"));
|
||||||
|
assert!(!is_monitor_name("alsa_input.pci-0000_00_1f.3.analog-stereo"));
|
||||||
|
assert!(!is_monitor_name(""));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,12 +2,14 @@ pub mod capture;
|
|||||||
pub mod concurrency;
|
pub mod concurrency;
|
||||||
pub mod decode;
|
pub mod decode;
|
||||||
pub mod resample;
|
pub mod resample;
|
||||||
|
pub mod streaming_resample;
|
||||||
pub mod vad;
|
pub mod vad;
|
||||||
pub mod wav;
|
pub mod wav;
|
||||||
|
|
||||||
pub use capture::{AudioChunk, MicrophoneCapture};
|
pub use capture::{AudioChunk, DeviceInfo, MicrophoneCapture};
|
||||||
pub use concurrency::decode_and_resample;
|
pub use concurrency::decode_and_resample;
|
||||||
pub use decode::decode_audio_file;
|
pub use decode::decode_audio_file;
|
||||||
pub use resample::resample_to_16khz;
|
pub use resample::resample_to_16khz;
|
||||||
|
pub use streaming_resample::StreamingResampler;
|
||||||
pub use vad::SpeechDetector;
|
pub use vad::SpeechDetector;
|
||||||
pub use wav::{read_wav, write_wav};
|
pub use wav::{read_wav, write_wav};
|
||||||
|
|||||||
@@ -1,13 +1,183 @@
|
|||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
use tauri::Manager;
|
use tauri::{Emitter, Manager};
|
||||||
|
use tokio::sync::mpsc as tokio_mpsc;
|
||||||
|
|
||||||
|
use kon_audio::{DeviceInfo, MicrophoneCapture};
|
||||||
|
use kon_core::constants::WHISPER_SAMPLE_RATE;
|
||||||
use kon_core::types::AudioSamples;
|
use kon_core::types::AudioSamples;
|
||||||
|
|
||||||
/// Save PCM f32 samples as a WAV file. Returns the file path.
|
/// Enumerate every input device available to cpal, with metadata for the
|
||||||
|
/// Settings device-picker UI. Includes a flag for likely PulseAudio /
|
||||||
|
/// PipeWire monitor sources so the UI can warn the user.
|
||||||
#[tauri::command]
|
#[tauri::command]
|
||||||
pub async fn save_audio(
|
pub async fn list_audio_devices() -> Result<Vec<DeviceInfo>, String> {
|
||||||
|
tokio::task::spawn_blocking(MicrophoneCapture::list_devices)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("join error: {e}"))?
|
||||||
|
.map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shared state for native microphone capture.
|
||||||
|
pub struct NativeCaptureState {
|
||||||
|
/// Stop signal sender — dropping this stops the accumulator task.
|
||||||
|
stop_tx: Mutex<Option<tokio_mpsc::Sender<()>>>,
|
||||||
|
/// All captured samples (16kHz mono) for save_audio.
|
||||||
|
all_samples: Arc<Mutex<Vec<f32>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NativeCaptureState {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
stop_tx: Mutex::new(None),
|
||||||
|
all_samples: Arc::new(Mutex::new(Vec::new())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Start native microphone capture via cpal.
|
||||||
|
/// Streams 16kHz mono PCM chunks to the frontend via `native-pcm` events.
|
||||||
|
#[tauri::command]
|
||||||
|
pub async fn start_native_capture(
|
||||||
app: tauri::AppHandle,
|
app: tauri::AppHandle,
|
||||||
|
state: tauri::State<'_, NativeCaptureState>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
eprintln!("[native-capture] start_native_capture called");
|
||||||
|
|
||||||
|
// Stop any existing capture
|
||||||
|
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
|
||||||
|
drop(tx);
|
||||||
|
}
|
||||||
|
|
||||||
|
let (capture, rx) = MicrophoneCapture::start().map_err(|e| {
|
||||||
|
eprintln!("[native-capture] MicrophoneCapture::start failed: {e}");
|
||||||
|
e.to_string()
|
||||||
|
})?;
|
||||||
|
eprintln!("[native-capture] cpal capture started successfully");
|
||||||
|
|
||||||
|
// Wrap capture in Arc<Mutex> so it can be moved into the blocking task
|
||||||
|
let capture = Arc::new(Mutex::new(Some(capture)));
|
||||||
|
let capture_clone = capture.clone();
|
||||||
|
|
||||||
|
let all_samples = state.all_samples.clone();
|
||||||
|
all_samples.lock().unwrap().clear();
|
||||||
|
|
||||||
|
let (stop_tx, mut stop_rx) = tokio_mpsc::channel::<()>(1);
|
||||||
|
*state.stop_tx.lock().unwrap() = Some(stop_tx);
|
||||||
|
|
||||||
|
let all_samples_clone = all_samples.clone();
|
||||||
|
|
||||||
|
// Spawn a task that reads cpal chunks, downsamples to 16kHz mono,
|
||||||
|
// and emits events to the frontend
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut pcm_buffer: Vec<f32> = Vec::new();
|
||||||
|
let chunk_size = 8000_usize; // ~0.5s at 16kHz
|
||||||
|
|
||||||
|
loop {
|
||||||
|
// Check for stop signal (non-blocking)
|
||||||
|
if stop_rx.try_recv().is_ok() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drain available audio chunks from cpal (non-blocking)
|
||||||
|
let mut got_data = false;
|
||||||
|
while let Ok(chunk) = rx.try_recv() {
|
||||||
|
got_data = true;
|
||||||
|
let sample_rate = chunk.sample_rate;
|
||||||
|
let channels = chunk.channels as usize;
|
||||||
|
|
||||||
|
// Downmix to mono if stereo
|
||||||
|
let mono: Vec<f32> = if channels > 1 {
|
||||||
|
chunk
|
||||||
|
.samples
|
||||||
|
.chunks(channels)
|
||||||
|
.map(|frame| frame.iter().sum::<f32>() / channels as f32)
|
||||||
|
.collect()
|
||||||
|
} else {
|
||||||
|
chunk.samples
|
||||||
|
};
|
||||||
|
|
||||||
|
// Downsample to 16kHz using simple decimation
|
||||||
|
// (acceptable quality for speech — same approach as pcm-processor.js)
|
||||||
|
let ratio = sample_rate as f64 / WHISPER_SAMPLE_RATE as f64;
|
||||||
|
if (ratio - 1.0).abs() < 0.01 {
|
||||||
|
pcm_buffer.extend_from_slice(&mono);
|
||||||
|
} else {
|
||||||
|
let mut pos: f64 = 0.0;
|
||||||
|
for &s in &mono {
|
||||||
|
pos += 1.0;
|
||||||
|
if pos >= ratio {
|
||||||
|
pcm_buffer.push(s);
|
||||||
|
pos -= ratio;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Emit chunks to frontend when we have enough
|
||||||
|
while pcm_buffer.len() >= chunk_size {
|
||||||
|
let chunk: Vec<f32> = pcm_buffer.drain(..chunk_size).collect();
|
||||||
|
|
||||||
|
// Store for save_audio
|
||||||
|
if let Ok(mut all) = all_samples_clone.lock() {
|
||||||
|
all.extend_from_slice(&chunk);
|
||||||
|
}
|
||||||
|
|
||||||
|
let _ = app.emit("native-pcm", serde_json::json!({
|
||||||
|
"samples": chunk,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
if !got_data {
|
||||||
|
// Avoid busy-spinning when no audio data is available
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Emit any remaining samples
|
||||||
|
if !pcm_buffer.is_empty() {
|
||||||
|
if let Ok(mut all) = all_samples_clone.lock() {
|
||||||
|
all.extend_from_slice(&pcm_buffer);
|
||||||
|
}
|
||||||
|
let _ = app.emit("native-pcm", serde_json::json!({
|
||||||
|
"samples": pcm_buffer,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop the capture to stop the cpal stream
|
||||||
|
if let Ok(mut cap) = capture_clone.lock() {
|
||||||
|
cap.take();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stop native microphone capture. Returns all captured samples (16kHz mono).
|
||||||
|
#[tauri::command]
|
||||||
|
pub async fn stop_native_capture(
|
||||||
|
state: tauri::State<'_, NativeCaptureState>,
|
||||||
|
) -> Result<Vec<f32>, String> {
|
||||||
|
// Extract the stop sender without holding the guard across an await
|
||||||
|
let stop_tx = state.stop_tx.lock().unwrap().take();
|
||||||
|
if let Some(tx) = stop_tx {
|
||||||
|
let _ = tx.send(()).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Brief delay to let the accumulator flush
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||||
|
|
||||||
|
let samples = {
|
||||||
|
let mut all = state.all_samples.lock().unwrap();
|
||||||
|
std::mem::take(&mut *all)
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(samples)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn persist_audio_samples(
|
||||||
|
app: &tauri::AppHandle,
|
||||||
samples: Vec<f32>,
|
samples: Vec<f32>,
|
||||||
output_folder: Option<String>,
|
output_folder: Option<String>,
|
||||||
) -> Result<String, String> {
|
) -> Result<String, String> {
|
||||||
@@ -47,3 +217,13 @@ pub async fn save_audio(
|
|||||||
|
|
||||||
Ok(path.to_string_lossy().to_string())
|
Ok(path.to_string_lossy().to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Save PCM f32 samples as a WAV file. Returns the file path.
|
||||||
|
#[tauri::command]
|
||||||
|
pub async fn save_audio(
|
||||||
|
app: tauri::AppHandle,
|
||||||
|
samples: Vec<f32>,
|
||||||
|
output_folder: Option<String>,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
persist_audio_samples(&app, samples, output_folder).await
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ mod commands;
|
|||||||
mod tray;
|
mod tray;
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
use sqlx::SqlitePool;
|
use sqlx::SqlitePool;
|
||||||
use tauri::Manager;
|
use tauri::Manager;
|
||||||
@@ -47,6 +48,7 @@ fn build_preferences_script(prefs_json: Option<String>) -> String {
|
|||||||
if (a.fontSize) el.style.setProperty('--font-size-body', a.fontSize + 'px');
|
if (a.fontSize) el.style.setProperty('--font-size-body', a.fontSize + 'px');
|
||||||
if (a.letterSpacing != null) el.style.setProperty('--letter-spacing-body', a.letterSpacing + 'em');
|
if (a.letterSpacing != null) el.style.setProperty('--letter-spacing-body', a.letterSpacing + 'em');
|
||||||
if (a.lineHeight) el.style.setProperty('--line-height-body', String(a.lineHeight));
|
if (a.lineHeight) el.style.setProperty('--line-height-body', String(a.lineHeight));
|
||||||
|
if (a.transcriptSize) el.style.setProperty('--text-transcript', a.transcriptSize + 'px');
|
||||||
if (a.bionicReading) el.dataset.bionicReading = 'true';
|
if (a.bionicReading) el.dataset.bionicReading = 'true';
|
||||||
if (a.reduceMotion === 'on' || (a.reduceMotion === 'system' && window.matchMedia('(prefers-reduced-motion: reduce)').matches)) el.dataset.reduceMotion = 'true';
|
if (a.reduceMotion === 'on' || (a.reduceMotion === 'system' && window.matchMedia('(prefers-reduced-motion: reduce)').matches)) el.dataset.reduceMotion = 'true';
|
||||||
}}
|
}}
|
||||||
@@ -75,15 +77,19 @@ pub fn run() {
|
|||||||
.setup(|app| {
|
.setup(|app| {
|
||||||
// Initialise database (blocking in setup — runs once at startup)
|
// Initialise database (blocking in setup — runs once at startup)
|
||||||
let db_path = database_path();
|
let db_path = database_path();
|
||||||
|
let t0 = Instant::now();
|
||||||
let db = tauri::async_runtime::block_on(async {
|
let db = tauri::async_runtime::block_on(async {
|
||||||
init_db(&db_path).await
|
init_db(&db_path).await
|
||||||
})
|
})
|
||||||
.map_err(|e| Box::new(e) as Box<dyn std::error::Error>)?;
|
.map_err(|e| Box::new(e) as Box<dyn std::error::Error>)?;
|
||||||
|
eprintln!("[startup] DB init: {:?}", t0.elapsed());
|
||||||
|
|
||||||
// Load saved preferences for webview injection
|
// Load saved preferences for webview injection
|
||||||
|
let t1 = Instant::now();
|
||||||
let prefs_json = tauri::async_runtime::block_on(async {
|
let prefs_json = tauri::async_runtime::block_on(async {
|
||||||
get_setting(&db, "kon_preferences").await.unwrap_or(None)
|
get_setting(&db, "kon_preferences").await.unwrap_or(None)
|
||||||
});
|
});
|
||||||
|
eprintln!("[startup] Preferences load: {:?}", t1.elapsed());
|
||||||
let init_script = build_preferences_script(prefs_json);
|
let init_script = build_preferences_script(prefs_json);
|
||||||
|
|
||||||
// Apply preferences to the main window (defined in tauri.conf.json)
|
// Apply preferences to the main window (defined in tauri.conf.json)
|
||||||
@@ -136,6 +142,8 @@ pub fn run() {
|
|||||||
app.manage(PreferencesScript(init_script));
|
app.manage(PreferencesScript(init_script));
|
||||||
|
|
||||||
app.manage(commands::hotkey::HotkeyState::new());
|
app.manage(commands::hotkey::HotkeyState::new());
|
||||||
|
app.manage(commands::audio::NativeCaptureState::new());
|
||||||
|
app.manage(commands::live::LiveTranscriptionState::default());
|
||||||
|
|
||||||
app.manage(AppState {
|
app.manage(AppState {
|
||||||
whisper_engine: Arc::new(LocalEngine::new(
|
whisper_engine: Arc::new(LocalEngine::new(
|
||||||
@@ -162,6 +170,7 @@ pub fn run() {
|
|||||||
commands::models::list_models,
|
commands::models::list_models,
|
||||||
commands::models::load_model,
|
commands::models::load_model,
|
||||||
commands::models::check_engine,
|
commands::models::check_engine,
|
||||||
|
commands::models::get_runtime_capabilities,
|
||||||
// Parakeet model management
|
// Parakeet model management
|
||||||
commands::models::download_parakeet_model,
|
commands::models::download_parakeet_model,
|
||||||
commands::models::check_parakeet_model,
|
commands::models::check_parakeet_model,
|
||||||
@@ -174,6 +183,11 @@ pub fn run() {
|
|||||||
commands::transcription::transcribe_pcm_parakeet,
|
commands::transcription::transcribe_pcm_parakeet,
|
||||||
// Audio
|
// Audio
|
||||||
commands::audio::save_audio,
|
commands::audio::save_audio,
|
||||||
|
commands::audio::start_native_capture,
|
||||||
|
commands::audio::stop_native_capture,
|
||||||
|
commands::audio::list_audio_devices,
|
||||||
|
commands::live::start_live_transcription_session,
|
||||||
|
commands::live::stop_live_transcription_session,
|
||||||
// Windows
|
// Windows
|
||||||
commands::windows::open_task_window,
|
commands::windows::open_task_window,
|
||||||
commands::windows::open_viewer_window,
|
commands::windows::open_viewer_window,
|
||||||
|
|||||||
@@ -10,21 +10,39 @@
|
|||||||
import ZonePicker from "$lib/components/ZonePicker.svelte";
|
import ZonePicker from "$lib/components/ZonePicker.svelte";
|
||||||
import AccessibilityControls from "$lib/components/AccessibilityControls.svelte";
|
import AccessibilityControls from "$lib/components/AccessibilityControls.svelte";
|
||||||
import { getPreferences, updatePreferences } from "$lib/stores/preferences.svelte.js";
|
import { getPreferences, updatePreferences } from "$lib/stores/preferences.svelte.js";
|
||||||
|
import { clampTextLines } from "$lib/utils/textMeasure.js";
|
||||||
|
import { bodyPretextLineHeight, pretextFontShorthand } from "$lib/utils/accessibilityTypography.js";
|
||||||
import { Check, ChevronRight } from "lucide-svelte";
|
import { Check, ChevronRight } from "lucide-svelte";
|
||||||
import { open } from "@tauri-apps/plugin-dialog";
|
|
||||||
|
|
||||||
const prefs = getPreferences();
|
const prefs = getPreferences();
|
||||||
|
|
||||||
let engineStatus = $state("Checking...");
|
let engineStatus = $state("Checking...");
|
||||||
let engineOk = $state(false);
|
let engineOk = $state(false);
|
||||||
let downloadedModels = $state([]);
|
let downloadedModels = $state([]);
|
||||||
|
let runtimeCapabilities = $state(null);
|
||||||
let downloadingModel = $state("");
|
let downloadingModel = $state("");
|
||||||
let downloadProgress = $state(0);
|
let downloadProgress = $state(0);
|
||||||
let unlisten = null;
|
let unlisten = null;
|
||||||
|
let outputFolderEl = $state(null);
|
||||||
|
let outputFolderWidth = $state(0);
|
||||||
|
|
||||||
// Parakeet state
|
// Parakeet state
|
||||||
let parakeetStatus = $state("Checking...");
|
let parakeetStatus = $state("Checking...");
|
||||||
let parakeetOk = $state(false);
|
let parakeetOk = $state(false);
|
||||||
|
|
||||||
|
// Audio device picker (Settings → Audio → Microphone)
|
||||||
|
let audioDevices = $state([]);
|
||||||
|
let audioDevicesError = $state(null);
|
||||||
|
|
||||||
|
async function refreshAudioDevices() {
|
||||||
|
audioDevicesError = null;
|
||||||
|
try {
|
||||||
|
audioDevices = await invoke("list_audio_devices");
|
||||||
|
} catch (err) {
|
||||||
|
audioDevicesError = "Could not enumerate audio devices: " + (err?.message || err);
|
||||||
|
audioDevices = [];
|
||||||
|
}
|
||||||
|
}
|
||||||
let parakeetDownloaded = $state(false);
|
let parakeetDownloaded = $state(false);
|
||||||
let parakeetDownloading = $state(false);
|
let parakeetDownloading = $state(false);
|
||||||
let parakeetProgress = $state(0);
|
let parakeetProgress = $state(0);
|
||||||
@@ -42,9 +60,66 @@
|
|||||||
|
|
||||||
// Accordion state — first section open by default
|
// Accordion state — first section open by default
|
||||||
let openSection = $state('transcription');
|
let openSection = $state('transcription');
|
||||||
|
let settingsTextFont = $derived(pretextFontShorthand(prefs.accessibility, 11));
|
||||||
|
let settingsPathLineHeight = $derived(bodyPretextLineHeight(prefs.accessibility, 11));
|
||||||
|
let outputFolderPreview = $derived.by(() => {
|
||||||
|
const text = settings.outputFolder || "Default (app data)";
|
||||||
|
if (!text) return "";
|
||||||
|
if (outputFolderWidth <= 0) {
|
||||||
|
return text.length > 80 ? `${text.slice(0, 79)}…` : text;
|
||||||
|
}
|
||||||
|
return clampTextLines(
|
||||||
|
text,
|
||||||
|
settingsTextFont,
|
||||||
|
Math.max(120, outputFolderWidth - 24),
|
||||||
|
settingsPathLineHeight,
|
||||||
|
2,
|
||||||
|
).text;
|
||||||
|
});
|
||||||
|
|
||||||
|
function whisperModelId(size) {
|
||||||
|
const map = {
|
||||||
|
Tiny: "whisper-tiny-en",
|
||||||
|
Base: "whisper-base-en",
|
||||||
|
Small: "whisper-small-en",
|
||||||
|
Medium: "whisper-medium-en",
|
||||||
|
};
|
||||||
|
return map[size] || "whisper-base-en";
|
||||||
|
}
|
||||||
|
|
||||||
|
function selectedModelId() {
|
||||||
|
return settings.engine === "parakeet"
|
||||||
|
? "parakeet-ctc-0.6b-int8"
|
||||||
|
: whisperModelId(settings.modelSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
function engineCapabilities(engineId) {
|
||||||
|
return runtimeCapabilities?.engines?.find((engine) => engine.id === engineId) || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function selectedModelCapabilities() {
|
||||||
|
return engineCapabilities(settings.engine)?.models?.find((model) => model.id === selectedModelId()) || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function whisperModelCapabilities(size) {
|
||||||
|
return engineCapabilities("whisper")?.models?.find((model) => model.id === whisperModelId(size)) || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function currentModelIsEnglishOnly() {
|
||||||
|
return selectedModelCapabilities()?.languageSupport?.kind === "english-only";
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasGpuAcceleration() {
|
||||||
|
return (runtimeCapabilities?.accelerators || []).some((accelerator) => accelerator !== "cpu");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshRuntimeCapabilities() {
|
||||||
|
runtimeCapabilities = await invoke("get_runtime_capabilities");
|
||||||
|
}
|
||||||
|
|
||||||
onMount(async () => {
|
onMount(async () => {
|
||||||
try {
|
try {
|
||||||
|
await refreshRuntimeCapabilities();
|
||||||
const loaded = await invoke("check_engine");
|
const loaded = await invoke("check_engine");
|
||||||
engineOk = loaded;
|
engineOk = loaded;
|
||||||
engineStatus = loaded ? "Model loaded" : "No model loaded";
|
engineStatus = loaded ? "Model loaded" : "No model loaded";
|
||||||
@@ -52,6 +127,10 @@
|
|||||||
engineStatus = "Engine not ready";
|
engineStatus = "Engine not ready";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Populate audio device list so the Microphone picker is ready when
|
||||||
|
// the user opens the Audio section.
|
||||||
|
refreshAudioDevices();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
downloadedModels = await invoke("list_models");
|
downloadedModels = await invoke("list_models");
|
||||||
} catch {}
|
} catch {}
|
||||||
@@ -79,23 +158,22 @@
|
|||||||
if (unlistenParakeet) unlistenParakeet();
|
if (unlistenParakeet) unlistenParakeet();
|
||||||
});
|
});
|
||||||
|
|
||||||
// Auto-save on any change
|
|
||||||
$effect(() => {
|
$effect(() => {
|
||||||
void settings.engine;
|
if (!outputFolderEl) return;
|
||||||
void settings.modelSize;
|
const ro = new ResizeObserver((entries) => {
|
||||||
void settings.language;
|
for (const entry of entries) {
|
||||||
void settings.device;
|
outputFolderWidth = entry.contentRect.width;
|
||||||
void settings.formatMode;
|
}
|
||||||
void settings.removeFillers;
|
});
|
||||||
void settings.antiHallucination;
|
ro.observe(outputFolderEl);
|
||||||
void settings.britishEnglish;
|
return () => ro.disconnect();
|
||||||
void settings.autoCopy;
|
});
|
||||||
void settings.includeTimestamps;
|
|
||||||
void settings.theme;
|
// Auto-save on any settings change.
|
||||||
void settings.fontSize;
|
// JSON.stringify subscribes to all properties on the reactive object.
|
||||||
void settings.saveAudio;
|
// Safe because `settings` only holds persistent preferences — no ephemeral state.
|
||||||
void settings.outputFolder;
|
$effect(() => {
|
||||||
void settings.globalHotkey;
|
void JSON.stringify(settings);
|
||||||
saveSettings();
|
saveSettings();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -108,12 +186,24 @@
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
$effect(() => {
|
||||||
|
settings.engine;
|
||||||
|
settings.modelSize;
|
||||||
|
if (currentModelIsEnglishOnly() && settings.language !== "en") {
|
||||||
|
settings.language = "en";
|
||||||
|
}
|
||||||
|
if (!hasGpuAcceleration() && settings.device !== "auto") {
|
||||||
|
settings.device = "auto";
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
async function downloadModel(size) {
|
async function downloadModel(size) {
|
||||||
downloadingModel = size;
|
downloadingModel = size;
|
||||||
downloadProgress = 0;
|
downloadProgress = 0;
|
||||||
try {
|
try {
|
||||||
await invoke("download_model", { size });
|
await invoke("download_model", { size });
|
||||||
downloadedModels = await invoke("list_models");
|
downloadedModels = await invoke("list_models");
|
||||||
|
await refreshRuntimeCapabilities();
|
||||||
downloadingModel = "";
|
downloadingModel = "";
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
downloadingModel = "";
|
downloadingModel = "";
|
||||||
@@ -127,6 +217,7 @@
|
|||||||
engineOk = false;
|
engineOk = false;
|
||||||
try {
|
try {
|
||||||
await invoke("load_model", { size });
|
await invoke("load_model", { size });
|
||||||
|
await refreshRuntimeCapabilities();
|
||||||
engineOk = true;
|
engineOk = true;
|
||||||
engineStatus = `${settings.modelSize} model loaded`;
|
engineStatus = `${settings.modelSize} model loaded`;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -136,7 +227,21 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
function isModelDownloaded(size) {
|
function isModelDownloaded(size) {
|
||||||
return downloadedModels.includes(size.toLowerCase());
|
const runtimeModel = whisperModelCapabilities(size);
|
||||||
|
if (runtimeModel) {
|
||||||
|
return runtimeModel.downloaded;
|
||||||
|
}
|
||||||
|
return downloadedModels.some((model) => model.toLowerCase() === size.toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
function isModelLoaded(size) {
|
||||||
|
const runtimeModel = whisperModelCapabilities(size);
|
||||||
|
if (runtimeModel) {
|
||||||
|
return runtimeModel.loaded;
|
||||||
|
}
|
||||||
|
return runtimeCapabilities?.engines?.some(
|
||||||
|
(engine) => engine.id === "whisper" && engine.loadedModelId === whisperModelId(size),
|
||||||
|
) || false;
|
||||||
}
|
}
|
||||||
|
|
||||||
const modelDescriptions = {
|
const modelDescriptions = {
|
||||||
@@ -151,6 +256,7 @@
|
|||||||
parakeetProgress = 0;
|
parakeetProgress = 0;
|
||||||
try {
|
try {
|
||||||
await invoke("download_parakeet_model", { name: "ctc-int8" });
|
await invoke("download_parakeet_model", { name: "ctc-int8" });
|
||||||
|
await refreshRuntimeCapabilities();
|
||||||
parakeetDownloaded = true;
|
parakeetDownloaded = true;
|
||||||
parakeetDownloading = false;
|
parakeetDownloading = false;
|
||||||
parakeetStatus = "Downloaded (not loaded)";
|
parakeetStatus = "Downloaded (not loaded)";
|
||||||
@@ -165,6 +271,7 @@
|
|||||||
parakeetOk = false;
|
parakeetOk = false;
|
||||||
try {
|
try {
|
||||||
await invoke("load_parakeet_model", { name: "ctc-int8" });
|
await invoke("load_parakeet_model", { name: "ctc-int8" });
|
||||||
|
await refreshRuntimeCapabilities();
|
||||||
parakeetOk = true;
|
parakeetOk = true;
|
||||||
parakeetStatus = "Model loaded";
|
parakeetStatus = "Model loaded";
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -223,6 +330,56 @@
|
|||||||
|
|
||||||
<div class="px-7 pb-8">
|
<div class="px-7 pb-8">
|
||||||
<Card>
|
<Card>
|
||||||
|
<!-- Audio (microphone selection) -->
|
||||||
|
<div class="border-b border-border-subtle">
|
||||||
|
<button
|
||||||
|
class="flex items-center justify-between w-full py-4 px-5 text-left"
|
||||||
|
onclick={() => openSection = openSection === 'audio' ? null : 'audio'}
|
||||||
|
>
|
||||||
|
<h3 class="font-display text-[18px] italic text-text">Audio</h3>
|
||||||
|
<span class="text-text-tertiary text-[16px] leading-none">{openSection === 'audio' ? '−' : '+'}</span>
|
||||||
|
</button>
|
||||||
|
{#if openSection === 'audio'}
|
||||||
|
<div class="px-5 pb-5 animate-fade-in">
|
||||||
|
<div class="mb-6">
|
||||||
|
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Microphone</p>
|
||||||
|
<div class="flex items-center gap-3">
|
||||||
|
<select
|
||||||
|
class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text
|
||||||
|
focus:outline-none focus:border-accent focus:shadow-[0_0_0_3px_rgba(232,168,124,0.1)]
|
||||||
|
appearance-none cursor-pointer min-w-[280px] max-w-full"
|
||||||
|
bind:value={settings.microphoneDevice}
|
||||||
|
onfocus={refreshAudioDevices}
|
||||||
|
>
|
||||||
|
<option value="">Auto (recommended) — let Kon pick the working mic</option>
|
||||||
|
{#each audioDevices as dev}
|
||||||
|
<option value={dev.name} disabled={dev.is_likely_monitor}>
|
||||||
|
{dev.name}
|
||||||
|
{dev.is_default ? " (system default)" : ""}
|
||||||
|
{dev.is_likely_monitor ? " — speaker monitor, skip" : ""}
|
||||||
|
</option>
|
||||||
|
{/each}
|
||||||
|
</select>
|
||||||
|
<button
|
||||||
|
class="px-3 py-2 text-[12px] text-text-secondary border border-border rounded-lg hover:border-accent hover:text-text"
|
||||||
|
onclick={refreshAudioDevices}
|
||||||
|
type="button"
|
||||||
|
>Refresh</button>
|
||||||
|
</div>
|
||||||
|
{#if audioDevicesError}
|
||||||
|
<p class="text-[11px] text-error mt-2">{audioDevicesError}</p>
|
||||||
|
{:else if audioDevices.length === 0}
|
||||||
|
<p class="text-[11px] text-text-tertiary mt-2">No input devices detected. Check that a microphone is connected and PulseAudio/PipeWire is running.</p>
|
||||||
|
{:else}
|
||||||
|
<p class="text-[11px] text-text-tertiary mt-2">
|
||||||
|
Auto mode tries the system default first, then any other real input. Speaker-monitor sources (loopback) are skipped because they record system audio rather than the microphone. If dictation is silent, pick the device explicitly here.
|
||||||
|
</p>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Transcription -->
|
<!-- Transcription -->
|
||||||
<div class="border-b border-border-subtle">
|
<div class="border-b border-border-subtle">
|
||||||
<button
|
<button
|
||||||
@@ -240,8 +397,9 @@
|
|||||||
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Engine</p>
|
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Engine</p>
|
||||||
<SegmentedButton options={["whisper", "parakeet"]} bind:value={settings.engine} />
|
<SegmentedButton options={["whisper", "parakeet"]} bind:value={settings.engine} />
|
||||||
<p class="text-[11px] text-text-tertiary mt-2">
|
<p class="text-[11px] text-text-tertiary mt-2">
|
||||||
{settings.engine === "whisper" ? "OpenAI Whisper — 99+ languages, reliable" :
|
{settings.engine === "whisper"
|
||||||
"Nvidia Parakeet — faster on CPU, English-focused, auto-punctuated"}
|
? "Whisper with the currently shipped English-only models in this build"
|
||||||
|
: "Parakeet CTC 0.6B — English-only, fast when the model is installed"}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -264,7 +422,12 @@
|
|||||||
<p class="text-[11px] text-text-tertiary mt-2">{modelDescriptions[settings.modelSize]}</p>
|
<p class="text-[11px] text-text-tertiary mt-2">{modelDescriptions[settings.modelSize]}</p>
|
||||||
|
|
||||||
<div class="flex items-center gap-2 mt-3">
|
<div class="flex items-center gap-2 mt-3">
|
||||||
{#if isModelDownloaded(settings.modelSize)}
|
{#if isModelLoaded(settings.modelSize)}
|
||||||
|
<span class="inline-flex items-center gap-1.5 text-[11px] text-success">
|
||||||
|
<span class="w-[6px] h-[6px] rounded-full bg-success"></span>
|
||||||
|
Model loaded
|
||||||
|
</span>
|
||||||
|
{:else if isModelDownloaded(settings.modelSize)}
|
||||||
<span class="inline-flex items-center gap-1.5 text-[11px] text-success">
|
<span class="inline-flex items-center gap-1.5 text-[11px] text-success">
|
||||||
<span class="w-[6px] h-[6px] rounded-full bg-success"></span>
|
<span class="w-[6px] h-[6px] rounded-full bg-success"></span>
|
||||||
Downloaded
|
Downloaded
|
||||||
@@ -318,42 +481,68 @@
|
|||||||
<!-- Compute device -->
|
<!-- Compute device -->
|
||||||
<div class="mb-6">
|
<div class="mb-6">
|
||||||
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Compute Device</p>
|
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Compute Device</p>
|
||||||
<select
|
{#if hasGpuAcceleration()}
|
||||||
class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text
|
<select
|
||||||
focus:outline-none focus:border-accent focus:shadow-[0_0_0_3px_rgba(232,168,124,0.1)]
|
class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text
|
||||||
appearance-none cursor-pointer w-[220px]"
|
focus:outline-none focus:border-accent focus:shadow-[0_0_0_3px_rgba(232,168,124,0.1)]
|
||||||
bind:value={settings.device}
|
appearance-none cursor-pointer w-[220px]"
|
||||||
>
|
bind:value={settings.device}
|
||||||
<option value="auto">Auto (CPU)</option>
|
>
|
||||||
<option value="cuda">CUDA (NVIDIA GPU)</option>
|
<option value="auto">Auto</option>
|
||||||
<option value="cpu">CPU</option>
|
{#if runtimeCapabilities?.accelerators?.includes("cuda")}
|
||||||
</select>
|
<option value="cuda">CUDA (NVIDIA GPU)</option>
|
||||||
<p class="text-[11px] text-text-tertiary mt-2">GPU acceleration requires building with CUDA feature</p>
|
{/if}
|
||||||
|
{#if runtimeCapabilities?.accelerators?.includes("metal")}
|
||||||
|
<option value="metal">Metal (Apple GPU)</option>
|
||||||
|
{/if}
|
||||||
|
{#if runtimeCapabilities?.accelerators?.includes("vulkan")}
|
||||||
|
<option value="vulkan">Vulkan</option>
|
||||||
|
{/if}
|
||||||
|
<option value="cpu">CPU</option>
|
||||||
|
</select>
|
||||||
|
<p class="text-[11px] text-text-tertiary mt-2">Only accelerators built into this binary are shown here.</p>
|
||||||
|
{:else}
|
||||||
|
<div class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text w-[320px]">
|
||||||
|
This build is CPU-only. GPU controls appear in GPU-enabled builds.
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Language -->
|
<!-- Language -->
|
||||||
<div>
|
<div>
|
||||||
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Language</p>
|
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Language</p>
|
||||||
<div class="flex items-center gap-3">
|
<div class="flex items-center gap-3">
|
||||||
<select
|
{#if currentModelIsEnglishOnly()}
|
||||||
class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text
|
<select
|
||||||
focus:outline-none focus:border-accent focus:shadow-[0_0_0_3px_rgba(232,168,124,0.1)]
|
class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text
|
||||||
appearance-none cursor-pointer w-[140px]"
|
focus:outline-none focus:border-accent focus:shadow-[0_0_0_3px_rgba(232,168,124,0.1)]
|
||||||
bind:value={settings.language}
|
appearance-none cursor-not-allowed w-[220px]"
|
||||||
>
|
bind:value={settings.language}
|
||||||
<option value="en">English</option>
|
disabled
|
||||||
<option value="auto">Auto-detect</option>
|
>
|
||||||
<option value="fr">French</option>
|
<option value="en">English only</option>
|
||||||
<option value="de">German</option>
|
</select>
|
||||||
<option value="es">Spanish</option>
|
{:else}
|
||||||
<option value="it">Italian</option>
|
<select
|
||||||
<option value="pt">Portuguese</option>
|
class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text
|
||||||
<option value="nl">Dutch</option>
|
focus:outline-none focus:border-accent focus:shadow-[0_0_0_3px_rgba(232,168,124,0.1)]
|
||||||
<option value="pl">Polish</option>
|
appearance-none cursor-pointer w-[140px]"
|
||||||
<option value="ja">Japanese</option>
|
bind:value={settings.language}
|
||||||
<option value="ko">Korean</option>
|
>
|
||||||
<option value="zh">Chinese</option>
|
<option value="en">English</option>
|
||||||
</select>
|
<option value="auto">Auto-detect</option>
|
||||||
|
<option value="fr">French</option>
|
||||||
|
<option value="de">German</option>
|
||||||
|
<option value="es">Spanish</option>
|
||||||
|
<option value="it">Italian</option>
|
||||||
|
<option value="pt">Portuguese</option>
|
||||||
|
<option value="nl">Dutch</option>
|
||||||
|
<option value="pl">Polish</option>
|
||||||
|
<option value="ja">Japanese</option>
|
||||||
|
<option value="ko">Korean</option>
|
||||||
|
<option value="zh">Chinese</option>
|
||||||
|
</select>
|
||||||
|
{/if}
|
||||||
{#if settings.language === "en"}
|
{#if settings.language === "en"}
|
||||||
<button
|
<button
|
||||||
class="flex items-center gap-1.5 px-2.5 py-1.5 rounded-full text-[11px] border animate-fade-in
|
class="flex items-center gap-1.5 px-2.5 py-1.5 rounded-full text-[11px] border animate-fade-in
|
||||||
@@ -370,6 +559,13 @@
|
|||||||
</button>
|
</button>
|
||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
<p class="text-[11px] text-text-tertiary mt-2">
|
||||||
|
{#if currentModelIsEnglishOnly()}
|
||||||
|
The selected model only supports English in this build.
|
||||||
|
{:else}
|
||||||
|
Auto-detect is only meaningful with multilingual models.
|
||||||
|
{/if}
|
||||||
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{/if}
|
{/if}
|
||||||
@@ -466,10 +662,10 @@
|
|||||||
</div>
|
</div>
|
||||||
{#if editingProfile === i}
|
{#if editingProfile === i}
|
||||||
<div class="mt-1.5 animate-fade-in">
|
<div class="mt-1.5 animate-fade-in">
|
||||||
<textarea
|
<textarea
|
||||||
class="w-full h-[120px] bg-bg-elevated border border-border-subtle rounded-lg px-3 py-2
|
class="w-full h-[120px] bg-bg-elevated border border-border-subtle rounded-lg px-3 py-2
|
||||||
text-[12px] text-text leading-relaxed resize-none
|
text-[12px] text-text resize-none
|
||||||
focus:outline-none focus:border-accent"
|
focus:outline-none focus:border-accent"
|
||||||
placeholder="One word or phrase per line..."
|
placeholder="One word or phrase per line..."
|
||||||
bind:value={profile.words}
|
bind:value={profile.words}
|
||||||
oninput={() => saveProfiles()}
|
oninput={() => saveProfiles()}
|
||||||
@@ -534,10 +730,10 @@
|
|||||||
</div>
|
</div>
|
||||||
{#if editingTemplate === i}
|
{#if editingTemplate === i}
|
||||||
<div class="mt-1.5 animate-fade-in">
|
<div class="mt-1.5 animate-fade-in">
|
||||||
<textarea
|
<textarea
|
||||||
class="w-full h-[100px] bg-bg-elevated border border-border-subtle rounded-lg px-3 py-2
|
class="w-full h-[100px] bg-bg-elevated border border-border-subtle rounded-lg px-3 py-2
|
||||||
text-[12px] text-text leading-relaxed resize-none
|
text-[12px] text-text resize-none
|
||||||
focus:outline-none focus:border-accent"
|
focus:outline-none focus:border-accent"
|
||||||
placeholder="One section per line..."
|
placeholder="One section per line..."
|
||||||
value={template.sections.join("\n")}
|
value={template.sections.join("\n")}
|
||||||
oninput={(e) => { template.sections = e.target.value.split("\n").filter((s) => s.trim()); saveTemplates(); }}
|
oninput={(e) => { template.sections = e.target.value.split("\n").filter((s) => s.trim()); saveTemplates(); }}
|
||||||
@@ -591,13 +787,13 @@
|
|||||||
/>
|
/>
|
||||||
{#if settings.saveAudio}
|
{#if settings.saveAudio}
|
||||||
<div class="ml-[50px] mt-2 animate-fade-in">
|
<div class="ml-[50px] mt-2 animate-fade-in">
|
||||||
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-1.5">Output Folder</p>
|
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-1.5">Output Folder</p>
|
||||||
<div class="flex items-center gap-2">
|
<div class="flex items-center gap-2">
|
||||||
<div class="flex-1 bg-bg-input rounded-lg px-3 py-1.5 border border-border-subtle">
|
<div bind:this={outputFolderEl} class="flex-1 bg-bg-input rounded-lg px-3 py-1.5 border border-border-subtle">
|
||||||
<p class="text-[11px] text-text-secondary truncate">
|
<p class="text-[11px] text-text-secondary whitespace-pre-wrap break-words" style="line-height: {settingsPathLineHeight}px" title={settings.outputFolder || "Default (app data)"}>
|
||||||
{settings.outputFolder || "Default (app data)"}
|
{outputFolderPreview}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<button
|
<button
|
||||||
class="text-[11px] text-accent hover:text-accent-hover whitespace-nowrap"
|
class="text-[11px] text-accent hover:text-accent-hover whitespace-nowrap"
|
||||||
onclick={async () => {
|
onclick={async () => {
|
||||||
|
|||||||
@@ -33,6 +33,9 @@ const defaults = {
|
|||||||
outputFolder: "",
|
outputFolder: "",
|
||||||
globalHotkey: "Ctrl+Shift+R",
|
globalHotkey: "Ctrl+Shift+R",
|
||||||
sidebarCollapsed: false,
|
sidebarCollapsed: false,
|
||||||
|
// Empty string = let backend auto-select. Otherwise, exact device name as
|
||||||
|
// returned by `list_audio_devices`. Set via Settings → Audio → Microphone.
|
||||||
|
microphoneDevice: "",
|
||||||
};
|
};
|
||||||
|
|
||||||
function loadSettings() {
|
function loadSettings() {
|
||||||
|
|||||||
Reference in New Issue
Block a user