audio: fix mic capture — skip monitor sources, validate by RMS, add device picker

Day 1 of the upgrade plan (output/reports/kon-upgrade-plan-2026-04-17.md
in the CORBEL workspace). Fixes the HANDOVER.md blocker: native live
transcription was capturing silence because PulseAudio/PipeWire monitor
sources (speaker loopback) were winning the device-selection race —
they deliver zero-valued bytes that satisfied the original
"any device that produces data within 350ms" check.

WHAT CHANGED:

crates/audio/src/capture.rs (rewrite):
- New `DeviceInfo` struct (serde-derived) for the Settings device picker
- New `MicrophoneCapture::list_devices()` enumerates inputs with metadata
- New `MicrophoneCapture::start_with_device(name)` for explicit selection
- Refactored `start()` with monitor-source filtering by name pattern
  (.monitor suffix, "Monitor of " prefix, "loopback" substring) and
  RMS-energy validation in a 350ms window
- Two-pass selection: real inputs first, monitor sources only as
  last-resort fallback with explicit warning log
- Drop counter (Arc<AtomicU64>) tracks chunks dropped by `try_send`
  failure under load — Codex review caught this as a silent-failure risk
- `dropped_chunks()` accessor for the live session
- Verbose tracing at every step for diagnosis
- Unit test for monitor-name detection
- `cargo check -p kon-audio` passes clean

crates/audio/src/lib.rs:
- Re-export `DeviceInfo`

crates/audio/Cargo.toml:
- Add serde dependency (for DeviceInfo derives)

src-tauri/src/commands/audio.rs:
- New `list_audio_devices` Tauri command (returns Vec<DeviceInfo>)

src-tauri/src/lib.rs:
- Register `list_audio_devices` in invoke_handler

src/lib/pages/SettingsPage.svelte:
- New "Audio" section with microphone picker dropdown
- Auto-populates on mount via `list_audio_devices`
- Refresh button + clear messaging about monitor sources
- Likely-monitor entries marked disabled in the dropdown
- Auto mode is the default (empty string in settings.microphoneDevice)

src/lib/stores/page.svelte.js:
- New `microphoneDevice` field in defaults (empty = auto-select)

NEXT STEPS (per the upgrade plan):
- Wire `microphoneDevice` from settings into `MicrophoneCapture::start_with_device`
  in the live and standalone capture paths (currently both still call
  the auto-selecting `start()`)
- Test on real hardware (Wayland + multiple input devices)
- Codex sanity-check of this diff is running in parallel; addendum to
  follow if anything substantive comes back

Refs: /home/jake/Documents/CORBEL-Projects/kon/HANDOVER.md
      output/reports/kon-upgrade-plan-2026-04-17.md (CORBEL workspace)
This commit is contained in:
2026-04-17 12:43:13 +01:00
parent fdc4a3cba5
commit 96980c7d5c
7 changed files with 805 additions and 103 deletions

View File

@@ -10,21 +10,39 @@
import ZonePicker from "$lib/components/ZonePicker.svelte";
import AccessibilityControls from "$lib/components/AccessibilityControls.svelte";
import { getPreferences, updatePreferences } from "$lib/stores/preferences.svelte.js";
import { clampTextLines } from "$lib/utils/textMeasure.js";
import { bodyPretextLineHeight, pretextFontShorthand } from "$lib/utils/accessibilityTypography.js";
import { Check, ChevronRight } from "lucide-svelte";
import { open } from "@tauri-apps/plugin-dialog";
const prefs = getPreferences();
let engineStatus = $state("Checking...");
let engineOk = $state(false);
let downloadedModels = $state([]);
let runtimeCapabilities = $state(null);
let downloadingModel = $state("");
let downloadProgress = $state(0);
let unlisten = null;
let outputFolderEl = $state(null);
let outputFolderWidth = $state(0);
// Parakeet state
let parakeetStatus = $state("Checking...");
let parakeetOk = $state(false);
// Audio device picker (Settings → Audio → Microphone)
let audioDevices = $state([]);
let audioDevicesError = $state(null);
async function refreshAudioDevices() {
audioDevicesError = null;
try {
audioDevices = await invoke("list_audio_devices");
} catch (err) {
audioDevicesError = "Could not enumerate audio devices: " + (err?.message || err);
audioDevices = [];
}
}
let parakeetDownloaded = $state(false);
let parakeetDownloading = $state(false);
let parakeetProgress = $state(0);
@@ -42,9 +60,66 @@
// Accordion state — first section open by default
let openSection = $state('transcription');
let settingsTextFont = $derived(pretextFontShorthand(prefs.accessibility, 11));
let settingsPathLineHeight = $derived(bodyPretextLineHeight(prefs.accessibility, 11));
let outputFolderPreview = $derived.by(() => {
const text = settings.outputFolder || "Default (app data)";
if (!text) return "";
if (outputFolderWidth <= 0) {
return text.length > 80 ? `${text.slice(0, 79)}…` : text;
}
return clampTextLines(
text,
settingsTextFont,
Math.max(120, outputFolderWidth - 24),
settingsPathLineHeight,
2,
).text;
});
function whisperModelId(size) {
const map = {
Tiny: "whisper-tiny-en",
Base: "whisper-base-en",
Small: "whisper-small-en",
Medium: "whisper-medium-en",
};
return map[size] || "whisper-base-en";
}
function selectedModelId() {
return settings.engine === "parakeet"
? "parakeet-ctc-0.6b-int8"
: whisperModelId(settings.modelSize);
}
function engineCapabilities(engineId) {
return runtimeCapabilities?.engines?.find((engine) => engine.id === engineId) || null;
}
function selectedModelCapabilities() {
return engineCapabilities(settings.engine)?.models?.find((model) => model.id === selectedModelId()) || null;
}
function whisperModelCapabilities(size) {
return engineCapabilities("whisper")?.models?.find((model) => model.id === whisperModelId(size)) || null;
}
function currentModelIsEnglishOnly() {
return selectedModelCapabilities()?.languageSupport?.kind === "english-only";
}
function hasGpuAcceleration() {
return (runtimeCapabilities?.accelerators || []).some((accelerator) => accelerator !== "cpu");
}
async function refreshRuntimeCapabilities() {
runtimeCapabilities = await invoke("get_runtime_capabilities");
}
onMount(async () => {
try {
await refreshRuntimeCapabilities();
const loaded = await invoke("check_engine");
engineOk = loaded;
engineStatus = loaded ? "Model loaded" : "No model loaded";
@@ -52,6 +127,10 @@
engineStatus = "Engine not ready";
}
// Populate audio device list so the Microphone picker is ready when
// the user opens the Audio section.
refreshAudioDevices();
try {
downloadedModels = await invoke("list_models");
} catch {}
@@ -79,23 +158,22 @@
if (unlistenParakeet) unlistenParakeet();
});
// Auto-save on any change
$effect(() => {
void settings.engine;
void settings.modelSize;
void settings.language;
void settings.device;
void settings.formatMode;
void settings.removeFillers;
void settings.antiHallucination;
void settings.britishEnglish;
void settings.autoCopy;
void settings.includeTimestamps;
void settings.theme;
void settings.fontSize;
void settings.saveAudio;
void settings.outputFolder;
void settings.globalHotkey;
if (!outputFolderEl) return;
const ro = new ResizeObserver((entries) => {
for (const entry of entries) {
outputFolderWidth = entry.contentRect.width;
}
});
ro.observe(outputFolderEl);
return () => ro.disconnect();
});
// Auto-save on any settings change.
// JSON.stringify subscribes to all properties on the reactive object.
// Safe because `settings` only holds persistent preferences — no ephemeral state.
$effect(() => {
void JSON.stringify(settings);
saveSettings();
});
@@ -108,12 +186,24 @@
}
});
$effect(() => {
settings.engine;
settings.modelSize;
if (currentModelIsEnglishOnly() && settings.language !== "en") {
settings.language = "en";
}
if (!hasGpuAcceleration() && settings.device !== "auto") {
settings.device = "auto";
}
});
async function downloadModel(size) {
downloadingModel = size;
downloadProgress = 0;
try {
await invoke("download_model", { size });
downloadedModels = await invoke("list_models");
await refreshRuntimeCapabilities();
downloadingModel = "";
} catch (err) {
downloadingModel = "";
@@ -127,6 +217,7 @@
engineOk = false;
try {
await invoke("load_model", { size });
await refreshRuntimeCapabilities();
engineOk = true;
engineStatus = `${settings.modelSize} model loaded`;
} catch (err) {
@@ -136,7 +227,21 @@
}
function isModelDownloaded(size) {
return downloadedModels.includes(size.toLowerCase());
const runtimeModel = whisperModelCapabilities(size);
if (runtimeModel) {
return runtimeModel.downloaded;
}
return downloadedModels.some((model) => model.toLowerCase() === size.toLowerCase());
}
function isModelLoaded(size) {
const runtimeModel = whisperModelCapabilities(size);
if (runtimeModel) {
return runtimeModel.loaded;
}
return runtimeCapabilities?.engines?.some(
(engine) => engine.id === "whisper" && engine.loadedModelId === whisperModelId(size),
) || false;
}
const modelDescriptions = {
@@ -151,6 +256,7 @@
parakeetProgress = 0;
try {
await invoke("download_parakeet_model", { name: "ctc-int8" });
await refreshRuntimeCapabilities();
parakeetDownloaded = true;
parakeetDownloading = false;
parakeetStatus = "Downloaded (not loaded)";
@@ -165,6 +271,7 @@
parakeetOk = false;
try {
await invoke("load_parakeet_model", { name: "ctc-int8" });
await refreshRuntimeCapabilities();
parakeetOk = true;
parakeetStatus = "Model loaded";
} catch (err) {
@@ -223,6 +330,56 @@
<div class="px-7 pb-8">
<Card>
<!-- Audio (microphone selection) -->
<div class="border-b border-border-subtle">
<button
class="flex items-center justify-between w-full py-4 px-5 text-left"
onclick={() => openSection = openSection === 'audio' ? null : 'audio'}
>
<h3 class="font-display text-[18px] italic text-text">Audio</h3>
<span class="text-text-tertiary text-[16px] leading-none">{openSection === 'audio' ? '' : '+'}</span>
</button>
{#if openSection === 'audio'}
<div class="px-5 pb-5 animate-fade-in">
<div class="mb-6">
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Microphone</p>
<div class="flex items-center gap-3">
<select
class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text
focus:outline-none focus:border-accent focus:shadow-[0_0_0_3px_rgba(232,168,124,0.1)]
appearance-none cursor-pointer min-w-[280px] max-w-full"
bind:value={settings.microphoneDevice}
onfocus={refreshAudioDevices}
>
<option value="">Auto (recommended) — let Kon pick the working mic</option>
{#each audioDevices as dev}
<option value={dev.name} disabled={dev.is_likely_monitor}>
{dev.name}
{dev.is_default ? " (system default)" : ""}
{dev.is_likely_monitor ? " — speaker monitor, skip" : ""}
</option>
{/each}
</select>
<button
class="px-3 py-2 text-[12px] text-text-secondary border border-border rounded-lg hover:border-accent hover:text-text"
onclick={refreshAudioDevices}
type="button"
>Refresh</button>
</div>
{#if audioDevicesError}
<p class="text-[11px] text-error mt-2">{audioDevicesError}</p>
{:else if audioDevices.length === 0}
<p class="text-[11px] text-text-tertiary mt-2">No input devices detected. Check that a microphone is connected and PulseAudio/PipeWire is running.</p>
{:else}
<p class="text-[11px] text-text-tertiary mt-2">
Auto mode tries the system default first, then any other real input. Speaker-monitor sources (loopback) are skipped because they record system audio rather than the microphone. If dictation is silent, pick the device explicitly here.
</p>
{/if}
</div>
</div>
{/if}
</div>
<!-- Transcription -->
<div class="border-b border-border-subtle">
<button
@@ -240,8 +397,9 @@
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Engine</p>
<SegmentedButton options={["whisper", "parakeet"]} bind:value={settings.engine} />
<p class="text-[11px] text-text-tertiary mt-2">
{settings.engine === "whisper" ? "OpenAI Whisper — 99+ languages, reliable" :
"Nvidia Parakeet — faster on CPU, English-focused, auto-punctuated"}
{settings.engine === "whisper"
? "Whisper with the currently shipped English-only models in this build"
: "Parakeet CTC 0.6B — English-only, fast when the model is installed"}
</p>
</div>
@@ -264,7 +422,12 @@
<p class="text-[11px] text-text-tertiary mt-2">{modelDescriptions[settings.modelSize]}</p>
<div class="flex items-center gap-2 mt-3">
{#if isModelDownloaded(settings.modelSize)}
{#if isModelLoaded(settings.modelSize)}
<span class="inline-flex items-center gap-1.5 text-[11px] text-success">
<span class="w-[6px] h-[6px] rounded-full bg-success"></span>
Model loaded
</span>
{:else if isModelDownloaded(settings.modelSize)}
<span class="inline-flex items-center gap-1.5 text-[11px] text-success">
<span class="w-[6px] h-[6px] rounded-full bg-success"></span>
Downloaded
@@ -318,42 +481,68 @@
<!-- Compute device -->
<div class="mb-6">
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Compute Device</p>
<select
class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text
focus:outline-none focus:border-accent focus:shadow-[0_0_0_3px_rgba(232,168,124,0.1)]
appearance-none cursor-pointer w-[220px]"
bind:value={settings.device}
>
<option value="auto">Auto (CPU)</option>
<option value="cuda">CUDA (NVIDIA GPU)</option>
<option value="cpu">CPU</option>
</select>
<p class="text-[11px] text-text-tertiary mt-2">GPU acceleration requires building with CUDA feature</p>
{#if hasGpuAcceleration()}
<select
class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text
focus:outline-none focus:border-accent focus:shadow-[0_0_0_3px_rgba(232,168,124,0.1)]
appearance-none cursor-pointer w-[220px]"
bind:value={settings.device}
>
<option value="auto">Auto</option>
{#if runtimeCapabilities?.accelerators?.includes("cuda")}
<option value="cuda">CUDA (NVIDIA GPU)</option>
{/if}
{#if runtimeCapabilities?.accelerators?.includes("metal")}
<option value="metal">Metal (Apple GPU)</option>
{/if}
{#if runtimeCapabilities?.accelerators?.includes("vulkan")}
<option value="vulkan">Vulkan</option>
{/if}
<option value="cpu">CPU</option>
</select>
<p class="text-[11px] text-text-tertiary mt-2">Only accelerators built into this binary are shown here.</p>
{:else}
<div class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text w-[320px]">
This build is CPU-only. GPU controls appear in GPU-enabled builds.
</div>
{/if}
</div>
<!-- Language -->
<div>
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Language</p>
<div class="flex items-center gap-3">
<select
class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text
focus:outline-none focus:border-accent focus:shadow-[0_0_0_3px_rgba(232,168,124,0.1)]
appearance-none cursor-pointer w-[140px]"
bind:value={settings.language}
>
<option value="en">English</option>
<option value="auto">Auto-detect</option>
<option value="fr">French</option>
<option value="de">German</option>
<option value="es">Spanish</option>
<option value="it">Italian</option>
<option value="pt">Portuguese</option>
<option value="nl">Dutch</option>
<option value="pl">Polish</option>
<option value="ja">Japanese</option>
<option value="ko">Korean</option>
<option value="zh">Chinese</option>
</select>
{#if currentModelIsEnglishOnly()}
<select
class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text
focus:outline-none focus:border-accent focus:shadow-[0_0_0_3px_rgba(232,168,124,0.1)]
appearance-none cursor-not-allowed w-[220px]"
bind:value={settings.language}
disabled
>
<option value="en">English only</option>
</select>
{:else}
<select
class="bg-bg-input border border-border rounded-lg px-3 py-2 text-[13px] text-text
focus:outline-none focus:border-accent focus:shadow-[0_0_0_3px_rgba(232,168,124,0.1)]
appearance-none cursor-pointer w-[140px]"
bind:value={settings.language}
>
<option value="en">English</option>
<option value="auto">Auto-detect</option>
<option value="fr">French</option>
<option value="de">German</option>
<option value="es">Spanish</option>
<option value="it">Italian</option>
<option value="pt">Portuguese</option>
<option value="nl">Dutch</option>
<option value="pl">Polish</option>
<option value="ja">Japanese</option>
<option value="ko">Korean</option>
<option value="zh">Chinese</option>
</select>
{/if}
{#if settings.language === "en"}
<button
class="flex items-center gap-1.5 px-2.5 py-1.5 rounded-full text-[11px] border animate-fade-in
@@ -370,6 +559,13 @@
</button>
{/if}
</div>
<p class="text-[11px] text-text-tertiary mt-2">
{#if currentModelIsEnglishOnly()}
The selected model only supports English in this build.
{:else}
Auto-detect is only meaningful with multilingual models.
{/if}
</p>
</div>
</div>
{/if}
@@ -466,10 +662,10 @@
</div>
{#if editingProfile === i}
<div class="mt-1.5 animate-fade-in">
<textarea
class="w-full h-[120px] bg-bg-elevated border border-border-subtle rounded-lg px-3 py-2
text-[12px] text-text leading-relaxed resize-none
focus:outline-none focus:border-accent"
<textarea
class="w-full h-[120px] bg-bg-elevated border border-border-subtle rounded-lg px-3 py-2
text-[12px] text-text resize-none
focus:outline-none focus:border-accent"
placeholder="One word or phrase per line..."
bind:value={profile.words}
oninput={() => saveProfiles()}
@@ -534,10 +730,10 @@
</div>
{#if editingTemplate === i}
<div class="mt-1.5 animate-fade-in">
<textarea
class="w-full h-[100px] bg-bg-elevated border border-border-subtle rounded-lg px-3 py-2
text-[12px] text-text leading-relaxed resize-none
focus:outline-none focus:border-accent"
<textarea
class="w-full h-[100px] bg-bg-elevated border border-border-subtle rounded-lg px-3 py-2
text-[12px] text-text resize-none
focus:outline-none focus:border-accent"
placeholder="One section per line..."
value={template.sections.join("\n")}
oninput={(e) => { template.sections = e.target.value.split("\n").filter((s) => s.trim()); saveTemplates(); }}
@@ -591,13 +787,13 @@
/>
{#if settings.saveAudio}
<div class="ml-[50px] mt-2 animate-fade-in">
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-1.5">Output Folder</p>
<div class="flex items-center gap-2">
<div class="flex-1 bg-bg-input rounded-lg px-3 py-1.5 border border-border-subtle">
<p class="text-[11px] text-text-secondary truncate">
{settings.outputFolder || "Default (app data)"}
</p>
</div>
<p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-1.5">Output Folder</p>
<div class="flex items-center gap-2">
<div bind:this={outputFolderEl} class="flex-1 bg-bg-input rounded-lg px-3 py-1.5 border border-border-subtle">
<p class="text-[11px] text-text-secondary whitespace-pre-wrap break-words" style="line-height: {settingsPathLineHeight}px" title={settings.outputFolder || "Default (app data)"}>
{outputFolderPreview}
</p>
</div>
<button
class="text-[11px] text-accent hover:text-accent-hover whitespace-nowrap"
onclick={async () => {