jake/Lumotia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

audio: wire user's microphone choice through start_native_capture + live session

Browse Source 
Day 1 follow-up to 96980c7. The device-picker UI in Settings now
actually takes effect: settings.microphoneDevice flows from the Svelte
store, through the Tauri invoke, into MicrophoneCapture::start_with_device
on the Rust side.

Touched paths (back-to-front):
- src-tauri/src/commands/audio.rs:start_native_capture — new optional
  `device_name: Option<String>` parameter; routes to start_with_device
  when set, falls back to auto-select start() when None or empty.
- src-tauri/src/commands/live.rs:StartLiveTranscriptionConfig — new
  optional `microphone_device: Option<String>` field with same
  semantics (rename_all = "camelCase" maps it to microphoneDevice on
  the wire).
- src-tauri/src/commands/live.rs:run_live_session — picks
  start_with_device when an explicit name is provided.
- src/lib/pages/DictationPage.svelte — passes
  microphoneDevice: settings.microphoneDevice || null in the invoke.

Behaviour:
- "Auto" in the picker (empty string) -> backend auto-selects, skipping
  monitor sources and validating by RMS energy.
- Specific device -> backend opens that device by exact name; if it has
  been disconnected the user gets a clear error pointing them back at
  Settings.

cargo check -p kon-audio passes clean. Tauri-crate cargo check requires
cmake (pre-existing infra dependency for whisper-rs-sys); install via
`sudo dnf install cmake clang-devel`.
This commit is contained in:
Jake
2026-04-17 12:45:19 +01:00
parent 96980c7d5c
commit 41db162041
3 changed files with 1002 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src-tauri/src/commands/audio.rs
View File

@@ -38,23 +38,38 @@ impl NativeCaptureState {
/// Start native microphone capture via cpal.
/// Streams 16kHz mono PCM chunks to the frontend via `native-pcm` events.
///
/// `device_name`: explicit device name (from `list_audio_devices`) or None / ""
/// to auto-select. The frontend passes `settings.microphoneDevice` here so the
/// user's pick from Settings → Audio → Microphone takes effect.
#[tauri::command]
pub async fn start_native_capture(
app: tauri::AppHandle,
state: tauri::State<'_, NativeCaptureState>,
device_name: Option<String>,
) -> Result<(), String> {
eprintln!("[native-capture] start_native_capture called");
eprintln!(
"[native-capture] start_native_capture called (device='{}')",
device_name.as_deref().unwrap_or("<auto>")
);
// Stop any existing capture
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
drop(tx);
}
let (capture, rx) = MicrophoneCapture::start().map_err(|e| {
let (capture, rx) = match device_name.as_deref() {
Some(name) if !name.is_empty() => MicrophoneCapture::start_with_device(name),
_ => MicrophoneCapture::start(),
}
.map_err(|e| {
eprintln!("[native-capture] MicrophoneCapture::start failed: {e}");
e.to_string()
})?;
eprintln!("[native-capture] cpal capture started successfully");
eprintln!(
"[native-capture] cpal capture started successfully on '{}'",
capture.device_name
);
// Wrap capture in Arc<Mutex> so it can be moved into the blocking task
let capture = Arc::new(Mutex::new(Some(capture)));

614
src-tauri/src/commands/live.rs Normal file
View File

@@ -0,0 +1,614 @@
#![allow(clippy::too_many_arguments)]
use std::sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc, Mutex,
};
use std::thread;
use std::time::{Duration, Instant};
use serde::{Deserialize, Serialize};
use tauri::ipc::Channel;
use crate::commands::audio::persist_audio_samples;
use crate::commands::models::{default_model_id_for_engine, ensure_model_loaded};
use crate::AppState;
use kon_ai_formatting::{
post_process_segments, FormatMode, PostProcessOptions,
};
use kon_audio::{MicrophoneCapture, StreamingResampler};
use kon_core::constants::WHISPER_SAMPLE_RATE;
use kon_core::types::{AudioSamples, Segment, TranscriptionOptions};
use kon_transcription::LocalEngine;
const CHUNK_SAMPLES: usize = 32_000; // 2s at 16kHz
const OVERLAP_SAMPLES: usize = 4_000; // 0.25s at 16kHz
const FINAL_CHUNK_MIN_SAMPLES: usize = 4_000; // 0.25s
const MAX_PENDING_SAMPLES: usize = CHUNK_SAMPLES;
const SPEECH_FRAME_SAMPLES: usize = 800; // 50ms
const MIN_SPEECH_FRAMES: usize = 1; // any plausible speech-like frame
const RMS_SPEECH_THRESHOLD: f32 = 0.001;
const PEAK_SPEECH_THRESHOLD: f32 = 0.004;
const FLATLINE_PEAK_THRESHOLD: f32 = 0.0005;
#[derive(Default)]
pub struct LiveTranscriptionState {
next_session_id: AtomicU64,
running: Mutex<Option<RunningLiveSession>>,
}
struct RunningLiveSession {
id: u64,
output_folder: Option<String>,
stop_flag: Arc<AtomicBool>,
handle: tokio::task::JoinHandle<Result<LiveSessionSummary, String>>,
status_channel: Channel<LiveStatusMessage>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct StartLiveTranscriptionConfig {
pub engine: String,
pub model_id: Option<String>,
pub language: Option<String>,
pub initial_prompt: Option<String>,
pub save_audio: bool,
pub output_folder: Option<String>,
pub remove_fillers: bool,
pub british_english: bool,
pub anti_hallucination: bool,
pub format_mode: String,
/// Optional explicit microphone device name (from `list_audio_devices`).
/// None or empty string = let `MicrophoneCapture::start` auto-select.
pub microphone_device: Option<String>,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct StartLiveTranscriptionResponse {
pub session_id: u64,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct StopLiveTranscriptionResponse {
pub session_id: u64,
pub audio_path: Option<String>,
pub dropped_audio_ms: u64,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct LiveResultMessage {
pub session_id: u64,
pub chunk_id: u32,
pub chunk_start_secs: f64,
pub duration: f64,
pub language: String,
pub inference_ms: u64,
pub segments: Vec<Segment>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase", tag = "type")]
#[allow(dead_code)]
pub enum LiveStatusMessage {
Warning {
session_id: u64,
message: String,
},
Overload {
session_id: u64,
dropped_audio_ms: u64,
message: String,
},
Error {
session_id: u64,
message: String,
},
Finished {
session_id: u64,
audio_path: Option<String>,
dropped_audio_ms: u64,
},
}
struct LiveSessionSummary {
session_id: u64,
dropped_audio_ms: u64,
audio_samples: Option<Vec<f32>>,
}
struct InferenceTask {
chunk_id: u32,
chunk_start_sample: u64,
trim_before_secs: f64,
duration_secs: f64,
rx: std::sync::mpsc::Receiver<Result<kon_transcription::TimedTranscript, String>>,
}
#[tauri::command]
pub async fn start_live_transcription_session(
state: tauri::State<'_, AppState>,
live_state: tauri::State<'_, LiveTranscriptionState>,
config: StartLiveTranscriptionConfig,
result_channel: Channel<LiveResultMessage>,
status_channel: Channel<LiveStatusMessage>,
) -> Result<StartLiveTranscriptionResponse, String> {
{
let running = live_state.running.lock().unwrap();
if running.is_some() {
return Err("A live transcription session is already running".into());
}
}
let model_id = config
.model_id
.clone()
.unwrap_or_else(|| default_model_id_for_engine(&config.engine).to_string());
eprintln!(
"[live] starting session: engine={}, model={}, language={:?}, save_audio={}",
config.engine,
model_id,
config.language,
config.save_audio
);
ensure_model_loaded(&state, &config.engine, &model_id).await?;
let session_id = live_state
.next_session_id
.fetch_add(1, Ordering::Relaxed)
.saturating_add(1);
let stop_flag = Arc::new(AtomicBool::new(false));
let engine = pick_engine(&state, &config.engine)?;
let output_folder = config.output_folder.clone();
let worker_stop = stop_flag.clone();
let worker_status = status_channel.clone();
let worker_results = result_channel.clone();
let handle = tokio::task::spawn_blocking(move || {
run_live_session(
session_id,
engine,
config,
worker_results,
worker_status,
worker_stop,
)
});
*live_state.running.lock().unwrap() = Some(RunningLiveSession {
id: session_id,
output_folder,
stop_flag,
handle,
status_channel,
});
Ok(StartLiveTranscriptionResponse { session_id })
}
#[tauri::command]
pub async fn stop_live_transcription_session(
app: tauri::AppHandle,
live_state: tauri::State<'_, LiveTranscriptionState>,
session_id: u64,
) -> Result<StopLiveTranscriptionResponse, String> {
let running = live_state.running.lock().unwrap().take();
let Some(running) = running else {
return Err("No live transcription session is running".into());
};
if running.id != session_id {
*live_state.running.lock().unwrap() = Some(running);
return Err(format!("Session {session_id} is not active"));
}
running.stop_flag.store(true, Ordering::Relaxed);
let summary = running
.handle
.await
.map_err(|e| format!("Live session task failed: {e}"))??;
let audio_path = if let Some(samples) = summary.audio_samples {
Some(
persist_audio_samples(&app, samples, running.output_folder.clone())
.await?,
)
} else {
None
};
let response = StopLiveTranscriptionResponse {
session_id: summary.session_id,
audio_path: audio_path.clone(),
dropped_audio_ms: summary.dropped_audio_ms,
};
let _ = running.status_channel.send(LiveStatusMessage::Finished {
session_id: summary.session_id,
audio_path,
dropped_audio_ms: summary.dropped_audio_ms,
});
Ok(response)
}
fn pick_engine(
state: &AppState,
engine: &str,
) -> Result<Arc<LocalEngine>, String> {
match engine {
"whisper" => Ok(state.whisper_engine.clone()),
"parakeet" => Ok(state.parakeet_engine.clone()),
other => Err(format!("Unknown engine: {other}")),
}
}
fn run_live_session(
session_id: u64,
engine: Arc<LocalEngine>,
config: StartLiveTranscriptionConfig,
result_channel: Channel<LiveResultMessage>,
status_channel: Channel<LiveStatusMessage>,
stop_flag: Arc<AtomicBool>,
) -> Result<LiveSessionSummary, String> {
let (capture, rx) = match config.microphone_device.as_deref() {
Some(name) if !name.is_empty() => MicrophoneCapture::start_with_device(name),
_ => MicrophoneCapture::start(),
}
.map_err(|e| e.to_string())?;
let _capture = capture;
let mut resampler: Option<StreamingResampler> = None;
let mut capture_buffer: Vec<f32> = Vec::new();
let mut kept_audio = if config.save_audio {
Some(Vec::new())
} else {
None
};
let mut buffer_start_sample: u64 = 0;
let mut dropped_audio_ms: u64 = 0;
let mut chunk_id: u32 = 0;
let mut inflight: Option<InferenceTask> = None;
let mut resampler_flushed = false;
loop {
if let Some(_done) = poll_inference(
&mut inflight,
session_id,
&config,
&result_channel,
&status_channel,
)? {}
match rx.recv_timeout(Duration::from_millis(25)) {
Ok(chunk) => {
let mono = downmix_chunk(chunk.samples, chunk.channels as usize);
let resampler = match &mut resampler {
Some(resampler) => resampler,
None => {
resampler = Some(
StreamingResampler::new(chunk.sample_rate)
.map_err(|e| e.to_string())?,
);
resampler.as_mut().expect("resampler just set")
}
};
let resampled =
resampler.push_samples(&mono).map_err(|e| e.to_string())?;
append_resampled_audio(
&mut capture_buffer,
&mut kept_audio,
&resampled,
);
}
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
let message =
"Microphone capture disconnected unexpectedly".to_string();
let _ = status_channel.send(LiveStatusMessage::Error {
session_id,
message: message.clone(),
});
return Err(message);
}
}
if inflight.is_some() && capture_buffer.len() > MAX_PENDING_SAMPLES {
let overflow = capture_buffer.len() - MAX_PENDING_SAMPLES;
capture_buffer.drain(..overflow);
buffer_start_sample = buffer_start_sample.saturating_add(overflow as u64);
dropped_audio_ms = dropped_audio_ms
.saturating_add((overflow as u64 * 1000) / WHISPER_SAMPLE_RATE as u64);
let _ = status_channel.send(LiveStatusMessage::Overload {
session_id,
dropped_audio_ms,
message: "Kon dropped older audio to keep live dictation responsive".into(),
});
}
let stopping = stop_flag.load(Ordering::Relaxed);
if stopping && !resampler_flushed {
if let Some(resampler) = &mut resampler {
let tail = resampler.flush().map_err(|e| e.to_string())?;
append_resampled_audio(&mut capture_buffer, &mut kept_audio, &tail);
}
resampler_flushed = true;
}
if inflight.is_none() {
if let Some(task) = maybe_dispatch_chunk(
&engine,
&config,
&mut capture_buffer,
&mut buffer_start_sample,
&mut chunk_id,
stopping,
&status_channel,
session_id,
) {
inflight = Some(task);
continue;
}
if stopping && resampler_flushed {
break;
}
}
}
while inflight.is_some() {
poll_inference(
&mut inflight,
session_id,
&config,
&result_channel,
&status_channel,
)?;
thread::sleep(Duration::from_millis(10));
}
Ok(LiveSessionSummary {
session_id,
dropped_audio_ms,
audio_samples: kept_audio,
})
}
fn append_resampled_audio(
capture_buffer: &mut Vec<f32>,
kept_audio: &mut Option<Vec<f32>>,
resampled: &[f32],
) {
if resampled.is_empty() {
return;
}
capture_buffer.extend_from_slice(resampled);
if let Some(kept_audio) = kept_audio {
kept_audio.extend_from_slice(resampled);
}
}
fn maybe_dispatch_chunk(
engine: &Arc<LocalEngine>,
config: &StartLiveTranscriptionConfig,
capture_buffer: &mut Vec<f32>,
buffer_start_sample: &mut u64,
chunk_id: &mut u32,
stopping: bool,
status_channel: &Channel<LiveStatusMessage>,
session_id: u64,
) -> Option<InferenceTask> {
let target_len = if capture_buffer.len() >= CHUNK_SAMPLES {
CHUNK_SAMPLES
} else if stopping && capture_buffer.len() >= FINAL_CHUNK_MIN_SAMPLES {
capture_buffer.len()
} else {
return None;
};
let trim_before_secs = if *chunk_id > 0 && !stopping && target_len > OVERLAP_SAMPLES {
OVERLAP_SAMPLES as f64 / WHISPER_SAMPLE_RATE as f64
} else {
0.0
};
let speech_window = if trim_before_secs > 0.0 {
&capture_buffer[OVERLAP_SAMPLES..target_len]
} else {
&capture_buffer[..target_len]
};
if !has_enough_speech(speech_window) {
let skipped_ms =
(target_len as u64 * 1000) / WHISPER_SAMPLE_RATE as u64;
eprintln!(
"[live] session {session_id}: skipped {skipped_ms}ms chunk as near-silence"
);
let _ = status_channel.send(LiveStatusMessage::Warning {
session_id,
message: format!(
"Skipped {skipped_ms}ms of near-silent audio. If this keeps happening, try a louder mic level or move closer to the microphone."
),
});
capture_buffer.drain(..target_len);
*buffer_start_sample = buffer_start_sample.saturating_add(target_len as u64);
return None;
}
*chunk_id = chunk_id.saturating_add(1);
let current_chunk_id = *chunk_id;
let chunk_start_sample = *buffer_start_sample;
let duration_secs = target_len as f64 / WHISPER_SAMPLE_RATE as f64;
let chunk_samples = capture_buffer[..target_len].to_vec();
eprintln!(
"[live] session {session_id}: dispatching chunk {} ({duration_secs:.2}s, {} samples)",
current_chunk_id,
chunk_samples.len()
);
let advance_by = if stopping || target_len < CHUNK_SAMPLES {
target_len
} else {
target_len.saturating_sub(OVERLAP_SAMPLES)
};
capture_buffer.drain(..advance_by);
*buffer_start_sample = buffer_start_sample.saturating_add(advance_by as u64);
let options = TranscriptionOptions {
language: config.language.clone(),
initial_prompt: config.initial_prompt.clone(),
};
let engine = engine.clone();
let (tx, rx) = std::sync::mpsc::channel();
thread::spawn(move || {
let audio = AudioSamples::mono_16khz(chunk_samples);
let started = Instant::now();
let result = engine
.transcribe_sync(&audio, &options)
.map(|mut timed| {
timed.inference_ms = started.elapsed().as_millis() as u64;
timed
})
.map_err(|e| e.to_string());
let _ = tx.send(result);
});
Some(InferenceTask {
chunk_id: current_chunk_id,
chunk_start_sample,
trim_before_secs,
duration_secs,
rx,
})
}
fn poll_inference(
inflight: &mut Option<InferenceTask>,
session_id: u64,
config: &StartLiveTranscriptionConfig,
result_channel: &Channel<LiveResultMessage>,
status_channel: &Channel<LiveStatusMessage>,
) -> Result<Option<bool>, String> {
let Some(task) = inflight else {
return Ok(None);
};
match task.rx.try_recv() {
Ok(Ok(timed)) => {
let mut segments: Vec<Segment> =
timed.transcript.segments().to_vec();
trim_overlap_segments(&mut segments, task.trim_before_secs);
post_process_segments(
&mut segments,
&PostProcessOptions {
remove_fillers: config.remove_fillers,
british_english: config.british_english,
anti_hallucination: config.anti_hallucination,
format_mode: FormatMode::parse(&config.format_mode),
},
);
let segment_count = segments.len();
result_channel
.send(LiveResultMessage {
session_id,
chunk_id: task.chunk_id,
chunk_start_secs: task.chunk_start_sample as f64
/ WHISPER_SAMPLE_RATE as f64,
duration: task.duration_secs,
language: timed.transcript.language().to_string(),
inference_ms: timed.inference_ms,
segments,
})
.map_err(|e| e.to_string())?;
eprintln!(
"[live] session {session_id}: delivered chunk {} with {} segments in {}ms",
task.chunk_id,
segment_count,
timed.inference_ms
);
*inflight = None;
Ok(Some(true))
}
Ok(Err(err)) => {
eprintln!("[live] session {session_id}: inference error: {err}");
*inflight = None;
let _ = status_channel.send(LiveStatusMessage::Error {
session_id,
message: err.clone(),
});
Err(err)
}
Err(std::sync::mpsc::TryRecvError::Empty) => Ok(Some(false)),
Err(std::sync::mpsc::TryRecvError::Disconnected) => {
*inflight = None;
let message = "Inference worker disconnected unexpectedly".to_string();
eprintln!("[live] session {session_id}: {message}");
let _ = status_channel.send(LiveStatusMessage::Error {
session_id,
message: message.clone(),
});
Err(message)
}
}
}
fn trim_overlap_segments(segments: &mut Vec<Segment>, trim_before_secs: f64) {
if trim_before_secs <= 0.0 {
return;
}
segments.retain(|segment| segment.end > trim_before_secs);
for segment in segments.iter_mut() {
if segment.start < trim_before_secs {
segment.start = trim_before_secs;
}
}
}
fn has_enough_speech(samples: &[f32]) -> bool {
if samples.is_empty() {
return false;
}
let chunk_peak = samples
.iter()
.map(|sample| sample.abs())
.fold(0.0_f32, f32::max);
if chunk_peak < FLATLINE_PEAK_THRESHOLD {
return false;
}
let mut speech_frames = 0usize;
for frame in samples.chunks(SPEECH_FRAME_SAMPLES) {
let len = frame.len().max(1) as f32;
let rms = (frame.iter().map(|sample| sample * sample).sum::<f32>() / len)
.sqrt();
let peak = frame
.iter()
.map(|sample| sample.abs())
.fold(0.0_f32, f32::max);
if rms >= RMS_SPEECH_THRESHOLD || peak >= PEAK_SPEECH_THRESHOLD {
speech_frames += 1;
}
}
speech_frames >= MIN_SPEECH_FRAMES
}
fn downmix_chunk(samples: Vec<f32>, channels: usize) -> Vec<f32> {
if channels <= 1 {
return samples;
}
samples
.chunks(channels)
.map(|frame| frame.iter().sum::<f32>() / channels as f32)
.collect()
}

592
src/lib/pages/DictationPage.svelte
View File

@@ -1,25 +1,28 @@
<script>
import { onMount, onDestroy } from "svelte";
import { invoke } from "@tauri-apps/api/core";
import { listen } from "@tauri-apps/api/event";
import { page, settings, templates, addToHistory, addTask, tasks } from "$lib/stores/page.svelte.js";
import { Channel, invoke } from "@tauri-apps/api/core";
import { page, settings, templates, profiles, addToHistory, addTask, tasks } from "$lib/stores/page.svelte.js";
import Card from "$lib/components/Card.svelte";
import ModelDownloader from "$lib/components/ModelDownloader.svelte";
import { exportTranscript } from "$lib/utils/export.js";
import { extractTasks } from "$lib/utils/taskExtractor.js";
import { pad } from "$lib/utils/time.js";
import { MAX_PCM_SAMPLES, MIN_CHUNK_SAMPLES, CHUNK_INTERVAL_MS, FEEDBACK_TIMEOUT_MS } from "$lib/utils/constants.js";
import { FEEDBACK_TIMEOUT_MS } from "$lib/utils/constants.js";
import { Mic, Loader2, SquareCheck, AlertTriangle } from 'lucide-svelte';
import EmptyState from '$lib/components/EmptyState.svelte';
import { getPreferences } from '$lib/stores/preferences.svelte.js';
import { bionicReading } from '$lib/actions/bionicReading.js';
import { measurePreWrap } from '$lib/utils/textMeasure.js';
import { transcriptPretextFont, transcriptPretextLineHeight } from '$lib/utils/accessibilityTypography.js';
import { hasTauriRuntime } from '$lib/utils/runtime.js';
const prefs = getPreferences();
const tauriRuntimeAvailable = hasTauriRuntime();
const browserPreviewMessage = "You're viewing Kon in a normal browser. Local transcription only works in the Tauri desktop app window.";
let transcript = $state("");
let segments = $state([]);
let timerInterval = $state(null);
let startTime = $state(0);
let chunkId = $state(0);
let modelReady = $state(false);
let modelLoading = $state(false);
let needsDownload = $state(false);
@@ -30,8 +33,14 @@
let extractedCount = $state(0);
let aiProcessing = $state(false);
let aiStatus = $state("");
let unlisten = null;
let chunkTimeOffset = 0;
let runtimeCapabilities = $state(null);
let sessionId = $state(null);
let drainingSessionId = $state(null);
let liveWarning = $state("");
let lastResultAt = $state(0);
let lastLiveActivityAt = $state(0);
let resultChannel = null;
let statusChannel = null;
// Cursor-based insertion
let textareaEl = $state(null);
@@ -44,41 +53,23 @@
// Deduplication: track which chunk IDs have been processed
let processedChunks = new Set();
// AudioWorklet state
let audioContext = null;
let workletNode = null;
let mediaStream = null;
let pcmBuffer = [];
let chunkTimer = null;
let allSamples = []; // Accumulate all PCM for audio saving
// Global hotkey listener
let hotkeyHandler = () => toggleRecording();
onMount(async () => {
unlisten = await listen("transcription-result", (event) => {
let result;
try {
result = typeof event.payload === "string"
? JSON.parse(event.payload)
: event.payload;
} catch (e) {
console.error("Failed to parse transcription result:", e);
return;
}
handleResult(result);
});
window.addEventListener("kon:toggle-recording", hotkeyHandler);
if (!tauriRuntimeAvailable) {
error = browserPreviewMessage;
return;
}
await checkModelState();
});
onDestroy(() => {
if (unlisten) unlisten();
window.removeEventListener("kon:toggle-recording", hotkeyHandler);
clearInterval(timerInterval);
clearInterval(chunkTimer);
if (page.recording) {
page.recording = false;
page.status = "Ready";
@@ -87,74 +78,154 @@
cleanup();
});
function handleResult(result) {
if (result.status === "transcription" && result.segments) {
// Deduplication guard: skip if this chunk_id was already processed
if (result.chunk_id != null && processedChunks.has(result.chunk_id)) {
return;
$effect(() => {
settings.engine;
settings.modelSize;
if (!tauriRuntimeAvailable) return;
queueMicrotask(() => {
if (!page.recording) {
void checkModelState();
}
if (result.chunk_id != null) processedChunks.add(result.chunk_id);
const text = result.segments.map((s) => s.text).join(" ").trim();
if (text) {
if (insertPos >= 0) {
// Insert at cursor position
const before = transcript.slice(0, insertPos);
const after = transcript.slice(insertPos);
const spaceBefore = before && !before.endsWith(" ") && !before.endsWith("\n") ? " " : "";
const spaceAfter = after && !after.startsWith(" ") && !after.startsWith("\n") ? " " : "";
transcript = before + spaceBefore + text + spaceAfter + after;
insertPos += spaceBefore.length + text.length + spaceAfter.length;
// Move cursor to end of inserted text
requestAnimationFrame(() => {
if (textareaEl) {
textareaEl.selectionStart = insertPos;
textareaEl.selectionEnd = insertPos;
}
});
} else {
// Append mode
// First chunk (id=1) starts the transcript; subsequent chunks append to it.
// When transcript is empty, assign directly; otherwise prepend a space separator.
if (!transcript) {
transcript = text;
} else {
transcript += " " + text;
});
});
function whisperModelId(size) {
const map = {
Tiny: "whisper-tiny-en",
Base: "whisper-base-en",
Small: "whisper-small-en",
Medium: "whisper-medium-en",
};
return map[size] || "whisper-base-en";
}
function selectedModelId() {
return settings.engine === "parakeet"
? "parakeet-ctc-0.6b-int8"
: whisperModelId(settings.modelSize);
}
function currentEngineCapabilities() {
return runtimeCapabilities?.engines?.find((engine) => engine.id === settings.engine) || null;
}
function currentModelCapabilities() {
return currentEngineCapabilities()?.models?.find((model) => model.id === selectedModelId()) || null;
}
function currentModelIsEnglishOnly() {
return currentModelCapabilities()?.languageSupport?.kind === "english-only";
}
function effectiveLanguage() {
return currentModelIsEnglishOnly() ? "en" : settings.language;
}
function buildInitialPrompt() {
if (!page.activeProfile || page.activeProfile === "None") return "";
const profile = profiles.find((entry) => entry.name === page.activeProfile);
if (!profile?.words) return "";
const words = profile.words
.split("\n")
.map((word) => word.trim())
.filter(Boolean);
if (words.length === 0) return "";
return `Use these terms when they match the audio: ${words.join(", ")}`;
}
async function refreshRuntimeCapabilities() {
runtimeCapabilities = await invoke("get_runtime_capabilities");
}
function matchesLiveSession(candidateSessionId) {
return candidateSessionId != null
&& (candidateSessionId === sessionId || candidateSessionId === drainingSessionId);
}
function handleLiveResult(result) {
if (!result || !matchesLiveSession(result.sessionId) || !result.segments) {
return;
}
lastResultAt = Date.now();
lastLiveActivityAt = lastResultAt;
if (textareaEl) {
if (page.recording && userScrolledUp) {
shouldPreserveScrollAnchor = true;
} else if (page.recording) {
shouldStickToBottom = true;
}
}
if (result.chunkId != null && processedChunks.has(result.chunkId)) {
return;
}
if (result.chunkId != null) processedChunks.add(result.chunkId);
const text = result.segments.map((segment) => segment.text).join(" ").trim();
if (text) {
if (insertPos >= 0) {
const before = transcript.slice(0, insertPos);
const after = transcript.slice(insertPos);
const spaceBefore = before && !before.endsWith(" ") && !before.endsWith("\n") ? " " : "";
const spaceAfter = after && !after.startsWith(" ") && !after.startsWith("\n") ? " " : "";
transcript = before + spaceBefore + text + spaceAfter + after;
insertPos += spaceBefore.length + text.length + spaceAfter.length;
requestAnimationFrame(() => {
if (textareaEl) {
textareaEl.selectionStart = insertPos;
textareaEl.selectionEnd = insertPos;
}
}
// Offset segment timestamps to be absolute
const offset = chunkTimeOffset;
const adjusted = result.segments.map((s) => ({
...s,
start: s.start + offset,
end: s.end + offset,
}));
segments = [...segments, ...adjusted];
if (result.duration) {
chunkTimeOffset += result.duration;
}
});
} else {
transcript = transcript ? `${transcript} ${text}` : text;
}
}
if (!page.recording && result.chunk_id === chunkId) {
finaliseTranscription();
}
const absoluteSegments = result.segments.map((segment) => ({
...segment,
start: segment.start + result.chunkStartSecs,
end: segment.end + result.chunkStartSecs,
}));
segments = [...segments, ...absoluteSegments];
}
function handleLiveStatus(status) {
if (!status || !matchesLiveSession(status.sessionId)) return;
lastLiveActivityAt = Date.now();
if (status.type === "overload" || status.type === "warning") {
liveWarning = status.message || "Kon is dropping older audio to stay responsive.";
return;
}
if (status.type === "error") {
error = status.message || "Live transcription failed";
transcriptionFailed = true;
page.status = "Error";
page.statusColor = "#e87171";
return;
}
if (status.type === "finished" && status.droppedAudioMs > 0) {
liveWarning = `Dropped ${Math.round(status.droppedAudioMs / 1000)}s of older audio to keep up in real time.`;
}
}
async function checkModelState() {
if (!tauriRuntimeAvailable) {
modelReady = false;
needsDownload = false;
return;
}
try {
if (settings.engine === "parakeet") {
const loaded = await invoke("check_parakeet_engine");
if (loaded) { modelReady = true; return; }
const downloaded = await invoke("check_parakeet_model", { name: "ctc-int8" });
if (downloaded) { needsDownload = false; await loadModel(); }
else { needsDownload = true; }
} else {
const loaded = await invoke("check_engine");
if (loaded) { modelReady = true; return; }
const downloaded = await invoke("check_model", { size: settings.modelSize.toLowerCase() });
if (downloaded) { needsDownload = false; await loadModel(); }
else { needsDownload = true; }
await refreshRuntimeCapabilities();
const currentModel = currentModelCapabilities();
modelReady = !!currentModel?.loaded;
needsDownload = currentModel ? !currentModel.downloaded : true;
if (currentModelIsEnglishOnly() && settings.language !== "en") {
settings.language = "en";
}
} catch (err) {
error = typeof err === "string" ? err : err.message || "Failed to check model";
@@ -162,6 +233,10 @@
}
async function loadModel() {
if (!tauriRuntimeAvailable) {
error = browserPreviewMessage;
return;
}
modelLoading = true;
page.status = "Loading model...";
page.statusColor = "#e8c86e";
@@ -173,6 +248,7 @@
} else {
await invoke("load_model", { size: settings.modelSize.toLowerCase() });
}
await refreshRuntimeCapabilities();
modelReady = true;
modelLoading = false;
page.status = "Ready";
@@ -187,7 +263,7 @@
function onModelDownloaded() {
needsDownload = false;
loadModel();
void loadModel();
}
async function toggleRecording() {
@@ -200,8 +276,15 @@
async function startRecording() {
error = "";
liveWarning = "";
saved = false;
transcriptionFailed = false;
if (!tauriRuntimeAvailable) {
error = browserPreviewMessage;
page.status = "Desktop app required";
page.statusColor = "#e8c86e";
return;
}
if (!modelReady) {
if (needsDownload) return;
await loadModel();
@@ -216,48 +299,49 @@
}
try {
audioContext = new AudioContext({ sampleRate: 16000 });
await audioContext.audioWorklet.addModule("/pcm-processor.js");
mediaStream = await navigator.mediaDevices.getUserMedia({
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
});
const source = audioContext.createMediaStreamSource(mediaStream);
workletNode = new AudioWorkletNode(audioContext, "pcm-processor");
pcmBuffer = [];
chunkId = 0;
chunkTimeOffset = 0;
sessionId = null;
drainingSessionId = null;
lastResultAt = 0;
lastLiveActivityAt = 0;
processedChunks.clear();
// Only clear transcript if not in insert mode (fresh recording)
if (insertPos === -1) {
transcript = "";
segments = [];
previousMeasuredContentHeight = 0;
}
allSamples = [];
workletNode.port.onmessage = (e) => {
if (e.data.type === "pcm") {
pcmBuffer = pcmBuffer.concat(e.data.samples);
if (settings.saveAudio) {
allSamples = allSamples.concat(e.data.samples);
}
}
};
resultChannel = new Channel((message) => handleLiveResult(message));
statusChannel = new Channel((message) => handleLiveStatus(message));
source.connect(workletNode);
const response = await invoke("start_live_transcription_session", {
config: {
engine: settings.engine,
modelId: selectedModelId(),
language: effectiveLanguage(),
initialPrompt: buildInitialPrompt(),
saveAudio: settings.saveAudio,
outputFolder: settings.outputFolder || null,
removeFillers: settings.removeFillers,
britishEnglish: settings.britishEnglish,
antiHallucination: settings.antiHallucination,
formatMode: settings.formatMode,
// Honour Settings → Audio → Microphone choice. Empty = auto-select.
microphoneDevice: settings.microphoneDevice || null,
},
resultChannel,
statusChannel,
});
sessionId = response.sessionId;
startTime = Date.now();
page.recording = true;
page.status = "Recording...";
page.statusColor = "#e87171";
timerInterval = setInterval(updateTimer, 1000);
chunkTimer = setInterval(sendChunk, CHUNK_INTERVAL_MS);
} catch (err) {
error = `Microphone access denied: ${err.message}`;
error = typeof err === "string" ? err : err?.message || "Microphone error";
page.status = "Error";
page.statusColor = "#e87171";
cleanup();
@@ -266,115 +350,71 @@
async function stopRecording() {
clearInterval(timerInterval);
clearInterval(chunkTimer);
page.recording = false;
transcribing = true;
page.status = "Finalising...";
page.statusColor = "#e8c86e";
const waitForTranscription = () => new Promise((resolve) => {
const check = () => transcribing ? setTimeout(check, 100) : resolve();
check();
});
await waitForTranscription();
await sendChunk();
cleanup();
if (chunkId === 0) {
page.status = "Ready";
page.statusColor = "#7ec89a";
}
}
function cleanup() {
if (workletNode) {
workletNode.disconnect();
workletNode = null;
}
if (mediaStream) {
mediaStream.getTracks().forEach((t) => t.stop());
mediaStream = null;
}
if (audioContext) {
audioContext.close();
audioContext = null;
}
}
async function sendChunk() {
if (pcmBuffer.length < MIN_CHUNK_SAMPLES) return;
if (transcribing) return;
chunkId++;
const currentChunkId = chunkId;
const samples = [...pcmBuffer];
pcmBuffer = [];
if (samples.length > MAX_PCM_SAMPLES) {
console.warn(`PCM buffer truncated from ${samples.length} to ${MAX_PCM_SAMPLES} samples (~5 min at 16 kHz)`);
samples.length = MAX_PCM_SAMPLES;
}
transcribing = true;
try {
let initialPrompt = "";
if (page.activeProfile && page.activeProfile !== "None") {
initialPrompt = page.activeProfile;
}
const activityBeforeStop = lastLiveActivityAt;
drainingSessionId = sessionId;
const response = sessionId
? await invoke("stop_live_transcription_session", { sessionId })
: null;
await waitForResultDrain(activityBeforeStop);
cleanup();
if (settings.engine === "parakeet") {
await invoke("transcribe_pcm_parakeet", {
samples,
chunkId: currentChunkId,
removeFillers: settings.removeFillers,
britishEnglish: settings.britishEnglish,
antiHallucination: settings.antiHallucination,
formatMode: settings.formatMode,
});
if (processedChunks.size === 0 && !transcript.trim()) {
page.status = "Ready";
page.statusColor = "#7ec89a";
} else {
await invoke("transcribe_pcm", {
samples,
chunkId: currentChunkId,
language: settings.language,
initialPrompt,
removeFillers: settings.removeFillers,
britishEnglish: settings.britishEnglish,
antiHallucination: settings.antiHallucination,
formatMode: settings.formatMode,
});
await finaliseTranscription(response?.audioPath || null);
}
} catch (err) {
console.error("transcribe_pcm failed:", err);
error = typeof err === "string" ? err : err.message || "Transcription failed";
transcriptionFailed = true;
if (!page.recording) {
page.status = "Error";
page.statusColor = "#e87171";
}
error = typeof err === "string" ? err : err?.message || "Failed to stop live transcription";
page.status = "Error";
page.statusColor = "#e87171";
cleanup();
} finally {
transcribing = false;
}
}
async function finaliseTranscription() {
function cleanup() {
sessionId = null;
drainingSessionId = null;
resultChannel = null;
statusChannel = null;
}
async function waitForResultDrain(previousActivityAt = 0) {
const firstMessageDeadline = Date.now() + 400;
while (Date.now() < firstMessageDeadline) {
if (lastLiveActivityAt > previousActivityAt) {
break;
}
await new Promise((resolve) => setTimeout(resolve, 25));
}
if (lastLiveActivityAt <= previousActivityAt) {
return;
}
const idleDeadline = Date.now() + 1500;
while (Date.now() < idleDeadline) {
if (Date.now() - lastLiveActivityAt >= 200) {
return;
}
await new Promise((resolve) => setTimeout(resolve, 25));
}
}
async function finaliseTranscription(audioPath = null) {
if (transcript.trim()) {
if (settings.autoCopy) {
invoke("copy_to_clipboard", { text: transcript }).catch(() => {});
}
// Save audio if enabled — capture path for history replay
let audioPath = null;
if (settings.saveAudio && allSamples.length > 0) {
try {
audioPath = await invoke("save_audio", {
samples: allSamples,
outputFolder: settings.outputFolder || null,
});
} catch (err) {
console.error("save_audio failed:", err);
}
allSamples = [];
}
const historyId = crypto.randomUUID();
addToHistory({
id: historyId,
@@ -384,7 +424,7 @@
text: transcript,
segments: segments,
duration: (Date.now() - startTime) / 1000,
language: settings.language,
language: effectiveLanguage(),
template: activeTemplate || undefined,
audioPath,
});
@@ -425,8 +465,12 @@
function clearTranscript() {
transcript = "";
segments = [];
previousMeasuredContentHeight = 0;
shouldPreserveScrollAnchor = false;
shouldStickToBottom = false;
page.timerText = "00:00";
error = "";
liveWarning = "";
saved = false;
insertPos = -1;
activeTemplate = "";
@@ -452,6 +496,7 @@
showTemplateMenu = false;
transcript = template.sections.map((s) => `## ${s}\n\n`).join("\n");
segments = [];
previousMeasuredContentHeight = 0;
// Position cursor at first section body
requestAnimationFrame(() => {
if (textareaEl) {
@@ -504,6 +549,80 @@
return trimmed ? trimmed.split(/\s+/).length : 0;
});
// Pretext content height measurement (DOM-free).
// Used for scroll-to-bottom affordance during live transcription.
let textareaWidth = $state(0);
let userScrolledUp = $state(false);
let previousMeasuredContentHeight = $state(0);
let shouldPreserveScrollAnchor = $state(false);
let shouldStickToBottom = $state(false);
// Build font string matching .font-transcript CSS
let transcriptFont = $derived(transcriptPretextFont(prefs.accessibility));
let transcriptLineHeight = $derived(transcriptPretextLineHeight(prefs.accessibility));
let contentHeight = $derived.by(() => {
if (!transcript.trim() || textareaWidth <= 0) return 0;
// Subtract padding (p-6 = 24px each side)
const innerWidth = textareaWidth - 48;
if (innerWidth <= 0) return 0;
return measurePreWrap(transcript, transcriptFont, innerWidth, transcriptLineHeight).height;
});
function onTextareaScroll(e) {
const el = e.target;
// "scrolled up" = more than 1 line from the bottom
userScrolledUp = (el.scrollHeight - el.scrollTop - el.clientHeight) > transcriptLineHeight;
}
function scrollToBottom() {
if (textareaEl) {
textareaEl.scrollTop = textareaEl.scrollHeight;
userScrolledUp = false;
}
}
// Track textarea width for Pretext layout calculations
$effect(() => {
if (!textareaEl) return;
const ro = new ResizeObserver(entries => {
for (const entry of entries) textareaWidth = entry.contentRect.width;
});
ro.observe(textareaEl);
return () => ro.disconnect();
});
$effect(() => {
const measuredHeight = contentHeight;
const previousHeight = previousMeasuredContentHeight;
if (!textareaEl) {
previousMeasuredContentHeight = measuredHeight;
return;
}
if (shouldPreserveScrollAnchor && measuredHeight > previousHeight) {
const delta = measuredHeight - previousHeight;
requestAnimationFrame(() => {
if (textareaEl) {
textareaEl.scrollTop += delta;
}
});
shouldPreserveScrollAnchor = false;
} else if (shouldStickToBottom) {
requestAnimationFrame(() => {
if (textareaEl) {
textareaEl.scrollTop = textareaEl.scrollHeight;
}
});
shouldStickToBottom = false;
}
previousMeasuredContentHeight = measuredHeight;
});
let showScrollHint = $derived(page.recording && userScrolledUp && contentHeight > 0);
let reduceMotion = $derived(
prefs.accessibility.reduceMotion === 'on'
|| (prefs.accessibility.reduceMotion === 'system'
@@ -519,7 +638,11 @@
{#if page.recording}Recording started{/if}
</div>
{#if needsDownload}
<ModelDownloader modelSize={settings.modelSize.toLowerCase()} onComplete={onModelDownloaded} />
<ModelDownloader
engine={settings.engine}
modelSize={settings.modelSize.toLowerCase()}
onComplete={onModelDownloaded}
/>
{:else}
<!-- Control strip -->
<div class="flex items-center gap-3 px-5 h-[56px] border-b border-border-subtle flex-shrink-0">
@@ -531,9 +654,11 @@
? 'bg-danger animate-pulse-warm'
: modelLoading
? 'bg-warning opacity-60 cursor-wait'
: 'bg-accent hover:bg-accent-hover shadow-[0_4px_20px_rgba(232,168,124,0.3)]'}"
: !tauriRuntimeAvailable
? 'bg-bg-elevated text-text-tertiary cursor-not-allowed'
: 'bg-accent hover:bg-accent-hover shadow-[0_4px_20px_rgba(232,168,124,0.3)]'}"
onclick={toggleRecording}
disabled={modelLoading}
disabled={modelLoading || !tauriRuntimeAvailable}
aria-label={page.recording ? "Stop recording" : "Start recording"}
style="transition-duration: var(--duration-ui)"
>
@@ -571,6 +696,8 @@
<span class="text-[11px] text-text-tertiary">
{#if modelLoading}
Loading model...
{:else if !tauriRuntimeAvailable}
Desktop app required for local transcription
{:else if saved}
<span class="text-success animate-fade-in">
Saved{#if extractedCount > 0} · {extractedCount} task{extractedCount === 1 ? '' : 's'} extracted{/if}
@@ -612,7 +739,7 @@
aria-label="Toggle task sidebar"
>
<span class="flex items-center gap-1.5">
<SquareCheck size={16} class="{page.taskSidebarOpen ? 'text-accent' : 'text-text-tertiary'}" aria-hidden="true" />
<SquareCheck size={16} class={page.taskSidebarOpen ? 'text-accent' : 'text-text-tertiary'} aria-hidden="true" />
<span class="text-[11px] {page.taskSidebarOpen ? 'text-accent' : 'text-text-tertiary'}">Tasks</span>
</span>
{#if taskCount > 0}
@@ -704,6 +831,14 @@
</div>
{/if}
{#if liveWarning && !error}
<div class="px-5 pt-2 animate-fade-in flex-shrink-0">
<div class="px-4 py-2 rounded-lg bg-warning/10 border border-warning/20 text-[12px] text-warning">
{liveWarning}
</div>
</div>
{/if}
<!-- Transcript area -->
<div class="flex-1 px-5 pt-3 pb-3 min-h-0" style="--text-transcript: {prefs.accessibility.transcriptSize}px; font-size: var(--text-transcript)">
<Card classes="h-full flex flex-col">
@@ -727,17 +862,30 @@
aria-live="polite"
></textarea>
{:else}
<textarea
bind:this={textareaEl}
class="font-transcript flex-1 w-full bg-transparent text-text p-6
resize-none focus:outline-none placeholder:text-text-tertiary min-h-0"
placeholder={activeTemplate ? "Click a section above, then press record..." : "Your words will appear here..."}
bind:value={transcript}
onclick={() => { showExportMenu = false; showTemplateMenu = false; }}
data-no-transition
aria-label="Transcript"
aria-live="polite"
></textarea>
<div class="relative flex-1 min-h-0">
<textarea
bind:this={textareaEl}
class="font-transcript h-full w-full bg-transparent text-text p-6
resize-none focus:outline-none placeholder:text-text-tertiary"
placeholder={activeTemplate ? "Click a section above, then press record..." : "Your words will appear here..."}
bind:value={transcript}
onscroll={onTextareaScroll}
onclick={() => { showExportMenu = false; showTemplateMenu = false; }}
data-no-transition
aria-label="Transcript"
aria-live="polite"
></textarea>
{#if showScrollHint}
<button
class="absolute bottom-3 right-6 px-3 py-1.5 rounded-full bg-accent text-white text-[11px] font-medium shadow-md hover:bg-accent-hover animate-fade-in"
onclick={scrollToBottom}
aria-label="Scroll to latest"
style="transition-duration: var(--duration-ui)"
>
↓ New text
</button>
{/if}
</div>
{/if}
<!-- Status footer (inside transcript card) -->