diff --git a/crates/audio/src/streaming_resample.rs b/crates/audio/src/streaming_resample.rs new file mode 100644 index 0000000..21acfb4 --- /dev/null +++ b/crates/audio/src/streaming_resample.rs @@ -0,0 +1,211 @@ +// Streaming resampler used by the live transcription session. +// +// Microphones expose whatever native rate the device supports (commonly +// 44 100 or 48 000 Hz). whisper.cpp wants 16 kHz mono `f32`. The live +// session calls `push_samples()` with each capture chunk as it arrives +// and gets back zero-or-more 16 kHz samples to enqueue into the model +// input buffer. At end-of-session it calls `flush()` once to drain any +// residual input and the resampler's internal tail. +// +// Implementation notes: +// +// - We use rubato's `SincFixedIn` (same engine the file-level +// `resample::resample_to_16khz` uses) so behaviour stays consistent +// across live + file paths. +// - rubato's fixed-in API requires a constant-size input chunk. We +// buffer captured samples in a residual `Vec` and only feed +// the resampler when we have a full chunk. +// - When the input rate already matches 16 kHz we skip rubato +// entirely and pass samples straight through (zero allocations +// beyond the returned `Vec`). +// - `flush()` zero-pads the residual to one final chunk, processes +// it, then truncates the output to the proportion that came from +// real (non-padded) samples — otherwise the trailing silence +// produced by the padding leaks into the saved audio file. + +use rubato::{ + Resampler, SincFixedIn, SincInterpolationParameters, SincInterpolationType, + WindowFunction, +}; + +use kon_core::constants::WHISPER_SAMPLE_RATE; +use kon_core::error::{KonError, Result}; + +/// Number of input samples the rubato resampler consumes per `process()` +/// call. Matches the chunk size used in `resample::resample_to_16khz`. +const INPUT_CHUNK: usize = 1024; + +pub enum StreamingResampler { + /// Source is already at 16 kHz — emit input verbatim. + Passthrough, + /// Source is at some other rate — feed via rubato. + Sinc { + resampler: SincFixedIn, + residual: Vec, + ratio: f64, + }, +} + +impl StreamingResampler { + /// Construct a resampler that converts `from_rate` Hz mono input to + /// 16 kHz mono output. Returns an error if `from_rate` is zero or if + /// rubato rejects the requested ratio. + pub fn new(from_rate: u32) -> Result { + if from_rate == 0 { + return Err(KonError::AudioDecodeFailed( + "StreamingResampler: input sample rate is 0".into(), + )); + } + + if from_rate == WHISPER_SAMPLE_RATE { + return Ok(Self::Passthrough); + } + + let ratio = WHISPER_SAMPLE_RATE as f64 / from_rate as f64; + + let params = SincInterpolationParameters { + sinc_len: 256, + f_cutoff: 0.95, + oversampling_factor: 128, + interpolation: SincInterpolationType::Cubic, + window: WindowFunction::Blackman, + }; + + let resampler = SincFixedIn::::new( + ratio, + 1.1, // max relative jitter; mirrors the file-level resampler + params, + INPUT_CHUNK, + 1, // mono + ) + .map_err(|e| { + KonError::AudioDecodeFailed(format!( + "StreamingResampler init failed: {e}" + )) + })?; + + Ok(Self::Sinc { + resampler, + residual: Vec::new(), + ratio, + }) + } + + /// Feed a fresh capture chunk and return any 16 kHz samples that are + /// ready to dispatch. The caller may pass any length; samples that + /// don't yet form a complete `INPUT_CHUNK` are buffered internally + /// and emitted on a later call (or on `flush()`). + pub fn push_samples(&mut self, mono: &[f32]) -> Result> { + match self { + Self::Passthrough => Ok(mono.to_vec()), + Self::Sinc { resampler, residual, .. } => { + if mono.is_empty() { + return Ok(Vec::new()); + } + residual.extend_from_slice(mono); + + let mut out: Vec = Vec::new(); + while residual.len() >= INPUT_CHUNK { + let chunk: Vec = residual.drain(..INPUT_CHUNK).collect(); + let input = vec![chunk]; + let result = resampler.process(&input, None).map_err(|e| { + KonError::AudioDecodeFailed(format!( + "StreamingResampler process failed: {e}" + )) + })?; + if let Some(channel) = result.into_iter().next() { + out.extend_from_slice(&channel); + } + } + Ok(out) + } + } + } + + /// Drain any residual samples and return the final 16 kHz output. + /// Called once when the live session is stopping. Subsequent calls + /// return an empty `Vec`. + pub fn flush(&mut self) -> Result> { + match self { + Self::Passthrough => Ok(Vec::new()), + Self::Sinc { resampler, residual, ratio } => { + if residual.is_empty() { + return Ok(Vec::new()); + } + + let leftover = residual.len(); + let mut chunk = std::mem::take(residual); + chunk.resize(INPUT_CHUNK, 0.0); + + let input = vec![chunk]; + let result = resampler.process(&input, None).map_err(|e| { + KonError::AudioDecodeFailed(format!( + "StreamingResampler flush failed: {e}" + )) + })?; + + let Some(mut out) = result.into_iter().next() else { + return Ok(Vec::new()); + }; + + // Trim padding-induced output: keep only the proportion + // of samples that came from real input, not from the + // zeros we used to fill the chunk. + let real_out = ((leftover as f64) * *ratio).round() as usize; + if real_out < out.len() { + out.truncate(real_out); + } + Ok(out) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn passthrough_at_16khz() { + let mut r = StreamingResampler::new(16_000).unwrap(); + let out = r.push_samples(&[0.1, 0.2, 0.3]).unwrap(); + assert_eq!(out, vec![0.1, 0.2, 0.3]); + assert!(r.flush().unwrap().is_empty()); + } + + #[test] + fn rejects_zero_rate() { + assert!(StreamingResampler::new(0).is_err()); + } + + #[test] + fn streaming_48k_to_16k_preserves_duration() { + let from_rate = 48_000u32; + let secs = 1.0; + let n = (from_rate as f64 * secs) as usize; + let samples: Vec = + (0..n).map(|i| (i as f32 * 0.001).sin()).collect(); + + let mut r = StreamingResampler::new(from_rate).unwrap(); + + // Push in irregular chunks to exercise the residual buffer. + let mut produced: Vec = Vec::new(); + for window in samples.chunks(700) { + produced.extend(r.push_samples(window).unwrap()); + } + produced.extend(r.flush().unwrap()); + + let out_secs = produced.len() as f64 / WHISPER_SAMPLE_RATE as f64; + assert!( + (out_secs - secs).abs() < 0.05, + "expected ~{secs}s of 16 kHz output, got {out_secs}s ({} samples)", + produced.len(), + ); + } + + #[test] + fn flush_after_no_input_is_empty() { + let mut r = StreamingResampler::new(48_000).unwrap(); + assert!(r.flush().unwrap().is_empty()); + } +} diff --git a/crates/hotkey/src/linux.rs b/crates/hotkey/src/linux.rs index d46d563..c8380a5 100644 --- a/crates/hotkey/src/linux.rs +++ b/crates/hotkey/src/linux.rs @@ -72,10 +72,15 @@ impl EvdevHotkeyListener { tokio::spawn(async move { let (notify_tx, mut notify_rx) = mpsc::channel::(32); - // notify watcher runs on a blocking thread internally + // notify watcher runs on a blocking thread internally. + // If inotify itself is unavailable (rare: minimal containers, + // some BSDs misconfigured as Linux) we degrade to "no + // hotplug detection" rather than panicking the task — the + // initial scan_and_attach pass above still picks up all + // devices that exist at startup. let _watcher = { let notify_tx = notify_tx.clone(); - let mut w = recommended_watcher(move |res: Result| { + let watcher = recommended_watcher(move |res: Result| { if let Ok(event) = res { if matches!(event.kind, EventKind::Create(_)) { for path in event.paths { @@ -85,11 +90,27 @@ impl EvdevHotkeyListener { } } } - }) - .expect("failed to create inotify watcher"); - w.watch(Path::new("/dev/input"), RecursiveMode::NonRecursive) - .expect("failed to watch /dev/input"); - w + }); + match watcher { + Ok(mut w) => match w.watch(Path::new("/dev/input"), RecursiveMode::NonRecursive) { + Ok(()) => Some(w), + Err(e) => { + eprintln!( + "[kon-hotkey] cannot watch /dev/input ({e}); \ + hotplug detection disabled, devices present \ + at startup still work", + ); + None + } + }, + Err(e) => { + eprintln!( + "[kon-hotkey] cannot create inotify watcher ({e}); \ + hotplug detection disabled", + ); + None + } + } }; loop { diff --git a/crates/storage/src/database.rs b/crates/storage/src/database.rs index c9b2180..06de39f 100644 --- a/crates/storage/src/database.rs +++ b/crates/storage/src/database.rs @@ -44,8 +44,8 @@ pub struct InsertTranscriptParams<'a> { pub engine: Option<&'a str>, pub model_id: Option<&'a str>, pub inference_ms: Option, - pub sample_rate: Option, - pub audio_channels: Option, + pub sample_rate: Option, + pub audio_channels: Option, pub format_mode: Option<&'a str>, pub remove_fillers: bool, pub british_english: bool, @@ -197,7 +197,7 @@ pub async fn search_transcripts( FROM transcripts t \ JOIN transcripts_fts fts ON fts.rowid = t.rowid \ WHERE transcripts_fts MATCH ? \ - ORDER BY rank LIMIT ?", + ORDER BY fts.rank LIMIT ?", ) .bind(query) .bind(limit) @@ -237,13 +237,22 @@ pub async fn add_dictionary_entry( term: &str, note: Option<&str>, ) -> Result { - let res = sqlx::query("INSERT OR IGNORE INTO dictionary (term, note) VALUES (?, ?)") - .bind(term) - .bind(note) - .execute(pool) - .await - .map_err(|e| KonError::StorageError(format!("Insert dictionary entry failed: {e}")))?; - Ok(res.last_insert_rowid()) + // ON CONFLICT … DO UPDATE so we always RETURN the actual row id. + // The previous `INSERT OR IGNORE` returned 0 (or a stale auto-increment + // counter) for duplicate terms, which lied to the frontend about which + // row it had just upserted. Bumping `note` to `excluded.note` also lets + // the user edit a term's note by re-adding it. + let row = sqlx::query( + "INSERT INTO dictionary (term, note) VALUES (?, ?) \ + ON CONFLICT(term) DO UPDATE SET note = excluded.note \ + RETURNING id", + ) + .bind(term) + .bind(note) + .fetch_one(pool) + .await + .map_err(|e| KonError::StorageError(format!("Insert dictionary entry failed: {e}")))?; + Ok(row.get("id")) } pub async fn delete_dictionary_entry(pool: &SqlitePool, id: i64) -> Result<()> { @@ -350,8 +359,8 @@ pub struct TranscriptRow { pub engine: Option, pub model_id: Option, pub inference_ms: Option, - pub sample_rate: Option, - pub audio_channels: Option, + pub sample_rate: Option, + pub audio_channels: Option, pub format_mode: Option, pub remove_fillers: bool, pub british_english: bool, diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 4c1d8df..135642c 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -35,8 +35,11 @@ serde_json = "1" # Async runtime (spawn_blocking for inference) tokio = { version = "1", features = ["rt", "sync"] } arboard = "3.6.1" -tauri-plugin-mcp = "0.7.1" + +# SqlitePool is named directly from src-tauri/src/lib.rs (the AppState +# stores it). Must be unconditional, not Linux-only — naming a type from +# a transitive dep requires the dep be listed here too. +sqlx = { version = "0.8", features = ["sqlite", "runtime-tokio"] } [target.'cfg(target_os = "linux")'.dependencies] webkit2gtk = "2.0" -sqlx = { version = "0.8", features = ["sqlite", "runtime-tokio"] } diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json index f94b23d..eda44c8 100644 --- a/src-tauri/capabilities/default.json +++ b/src-tauri/capabilities/default.json @@ -2,7 +2,7 @@ "$schema": "../gen/schemas/desktop-schema.json", "identifier": "default", "description": "Capability for the main window", - "windows": ["main", "tasks-float"], + "windows": ["main", "tasks-float", "transcript-viewer"], "permissions": [ "core:default", "core:window:allow-start-dragging", diff --git a/src-tauri/src/commands/live.rs b/src-tauri/src/commands/live.rs index dd21a09..220bd9f 100644 --- a/src-tauri/src/commands/live.rs +++ b/src-tauri/src/commands/live.rs @@ -254,11 +254,16 @@ fn run_live_session( status_channel: Channel, stop_flag: Arc, ) -> Result { - let (capture, rx) = match config.microphone_device.as_deref() { + let (mut capture, rx) = match config.microphone_device.as_deref() { Some(name) if !name.is_empty() => MicrophoneCapture::start_with_device(name), _ => MicrophoneCapture::start(), } .map_err(|e| e.to_string())?; + // Drain runtime stream errors into the status channel so the user + // gets a toast when the device disconnects mid-recording instead of + // silently producing empty transcripts. The `_capture` binding keeps + // the cpal stream alive for the duration of the session. + let mic_error_rx = capture.take_error_rx(); let _capture = capture; let mut resampler: Option = None; @@ -283,6 +288,22 @@ fn run_live_session( &status_channel, )? {} + // Surface any cpal runtime errors as warnings. Non-fatal: a hard + // disconnect will also drop the audio sender and be caught by + // the `Disconnected` arm below. This lets the user see a toast + // even when cpal recovers without tearing the stream down. + if let Some(err_rx) = &mic_error_rx { + while let Ok(err) = err_rx.try_recv() { + let _ = status_channel.send(LiveStatusMessage::Warning { + session_id, + message: format!( + "Microphone '{}' reported an error: {}", + err.device_name, err.message + ), + }); + } + } + match rx.recv_timeout(Duration::from_millis(25)) { Ok(chunk) => { let mono = downmix_chunk(chunk.samples, chunk.channels as usize); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 0da8061..490b08c 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -174,7 +174,18 @@ pub fn run() { request.allow(); true }); - }).expect("Failed to configure webview media permissions"); + }) + .unwrap_or_else(|e| { + // Non-fatal: WebKitGTK may already have media + // capture wired by some compositors, or the + // signal binding may fail on unusual builds. + // Falling back means getUserMedia() prompts (or + // silently denies) instead of auto-granting, + // which is degraded but recoverable. + eprintln!( + "[startup] failed to configure webview media permissions: {e}", + ); + }); } // Close-to-tray: hide window instead of exiting diff --git a/src/lib/stores/preferences.svelte.js b/src/lib/stores/preferences.svelte.js index ae58804..0fa65f9 100644 --- a/src/lib/stores/preferences.svelte.js +++ b/src/lib/stores/preferences.svelte.js @@ -1,5 +1,6 @@ // src/lib/stores/preferences.svelte.js import { invoke } from '@tauri-apps/api/core'; +import { toasts } from './toasts.svelte.js'; const DEFAULTS = { theme: 'dark', @@ -74,13 +75,25 @@ function applyToDOM(prefs) { } let saveTimeout = null; +// Show the failure toast at most once per process so a stuck SQLite path +// doesn't spam the user every time they nudge a slider. +let saveFailureToastShown = false; function persistToSQLite(prefs) { clearTimeout(saveTimeout); saveTimeout = setTimeout(async () => { try { await invoke('save_preferences', { preferences: JSON.stringify(prefs) }); + saveFailureToastShown = false; } catch (e) { console.error('Failed to save preferences:', e); + if (!saveFailureToastShown) { + const msg = typeof e === 'string' ? e : (e?.message ?? String(e)); + toasts.warn( + 'Could not save preferences', + `${msg}. Your changes still apply for this session.`, + ); + saveFailureToastShown = true; + } } }, 500); } diff --git a/src/lib/utils/runtime.js b/src/lib/utils/runtime.js new file mode 100644 index 0000000..a16268c --- /dev/null +++ b/src/lib/utils/runtime.js @@ -0,0 +1,18 @@ +// Detects whether the Tauri runtime is present in the current window. +// +// In Tauri v2 the bootstrap script injects `window.__TAURI_INTERNALS__` +// before the page script runs, and also sets `window.isTauri = true` as +// a convenience marker. Either is sufficient evidence we are inside a +// Tauri webview. +// +// In a plain browser (vite preview, `npm run dev` outside `tauri dev`) +// neither global is set, so `hasTauriRuntime()` returns false. Callers +// use this to gate `invoke()` calls and fall back to localStorage-only +// behaviour for browser-preview mode. + +export function hasTauriRuntime() { + if (typeof window === 'undefined') return false; + if (window.__TAURI_INTERNALS__) return true; + if (window.isTauri === true) return true; + return false; +} diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 6dbf0e6..e5dd84a 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -10,7 +10,6 @@ import { loadOsInfo } from "$lib/utils/osInfo.js"; import { page, settings, saveSettings } from "$lib/stores/page.svelte.js"; import { getPreferences, updatePreferences } from "$lib/stores/preferences.svelte.js"; - import { hasTauriRuntime } from "$lib/utils/runtime.js"; import { page as sveltePage } from "$app/stores"; @@ -171,6 +170,9 @@ // log_frontend_error. Best-effort: never let the error handler itself // throw, never crash the app over a logging failure. // (Diagnostics layer 1 — local only, never transmitted) + let onWindowError = null; + let onUnhandledRejection = null; + function installGlobalErrorCapture() { if (!hasTauriRuntime()) return; @@ -181,19 +183,22 @@ } catch { /* same */ } }; - window.addEventListener("error", (ev) => { + onWindowError = (ev) => { safeLog( "window.onerror", ev?.message || ev?.error?.message || "Unknown error", ev?.error?.stack || null, ); - }); + }; - window.addEventListener("unhandledrejection", (ev) => { + onUnhandledRejection = (ev) => { const reason = ev?.reason; const msg = (reason && (reason.message || String(reason))) || "Unhandled rejection"; safeLog("unhandledrejection", msg, reason?.stack || null); - }); + }; + + window.addEventListener("error", onWindowError); + window.addEventListener("unhandledrejection", onUnhandledRejection); } onMount(async () => { @@ -229,6 +234,8 @@ onDestroy(() => { window.removeEventListener("resize", handleResize); + if (onWindowError) window.removeEventListener("error", onWindowError); + if (onUnhandledRejection) window.removeEventListener("unhandledrejection", onUnhandledRejection); if (!tauriRuntimeAvailable) { return; }