diff --git a/crates/ai-formatting/src/lib.rs b/crates/ai-formatting/src/lib.rs index 7c7d246..7d1a758 100644 --- a/crates/ai-formatting/src/lib.rs +++ b/crates/ai-formatting/src/lib.rs @@ -4,6 +4,6 @@ pub mod pipeline; pub mod rule_based; pub use correction_learning::extract_corrections; -pub use llm_client::cleanup_text as llm_cleanup_text; +pub use llm_client::{cleanup_text as llm_cleanup_text, LlmPromptPreset}; pub use pipeline::{post_process_segments, FormatMode, PostProcessOptions}; pub use rule_based::{format_text, is_hallucination, remove_fillers, to_british_english}; diff --git a/crates/ai-formatting/src/llm_client.rs b/crates/ai-formatting/src/llm_client.rs index 754e0e6..66b06d9 100644 --- a/crates/ai-formatting/src/llm_client.rs +++ b/crates/ai-formatting/src/llm_client.rs @@ -67,19 +67,95 @@ pub fn format_dictionary_suffix(terms: &[String]) -> String { ) } +/// Named cleanup-style presets (brief item B.1 #15). Each preset adds a +/// short additional instruction to the translation contract so the same +/// underlying translator behaviour produces output appropriate for the +/// user's current context (email vs. meeting notes vs. code). +/// +/// Deliberately narrow set — four presets is small enough to pick from a +/// dropdown without becoming its own cognitive load. Users wanting more +/// nuance edit `profile.initial_prompt` instead; presets layer on top of +/// whatever the active profile specifies. +/// +/// The translator-not-editor framing from CLEANUP_PROMPT still governs — +/// presets shape tone and structure, never licence content editing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LlmPromptPreset { + /// No additional guidance beyond the profile's initial_prompt. + Default, + /// Format as an email paragraph — tight sentences, natural + /// paragraph breaks at topic shifts, no markdown. + Email, + /// Format as bulleted meeting notes. Lead action items with an + /// imperative verb; keep informational sentences as prose. + Notes, + /// Software-dictation mode. Preserve technical terms, variable + /// names, file paths, and symbols exactly as spoken. Do not reword + /// technical phrasing. + Code, +} + +impl LlmPromptPreset { + /// Parse a frontend-serialised preset identifier. Unknown or empty + /// strings collapse to Default so an outdated frontend can never + /// produce an unhandled enum variant — the user just sees baseline + /// behaviour. + pub fn parse(value: &str) -> Self { + match value.trim().to_ascii_lowercase().as_str() { + "email" => Self::Email, + "notes" | "meeting" | "meeting-notes" => Self::Notes, + "code" | "software" => Self::Code, + _ => Self::Default, + } + } + + /// Extra instruction appended to the system prompt. Empty string + /// for Default — no whitespace or leading newline — so the concat + /// with the dictionary suffix stays clean. + pub fn suffix(self) -> &'static str { + match self { + Self::Default => "", + Self::Email => concat!( + "\n\n", + "Context: the speaker is dictating an email. Produce a single ", + "coherent email paragraph (or two if the topic clearly shifts). ", + "Tight sentences, no markdown, no salutation or signature unless ", + "the speaker explicitly dictates one.", + ), + Self::Notes => concat!( + "\n\n", + "Context: the speaker is dictating meeting notes. Where the text ", + "contains a list of items or action items, render them as a ", + "markdown bullet list ('- '). Action items should lead with an ", + "imperative verb. Preserve prose informational sentences as prose; ", + "don't force bullets where narrative is clearer.", + ), + Self::Code => concat!( + "\n\n", + "Context: the speaker is dictating about software. Preserve ", + "technical terms, variable names, file paths, CLI flags, and ", + "symbols exactly as spoken. Do not reword technical phrasing or ", + "'translate' identifiers into natural English.", + ), + } + } +} + pub fn cleanup_text( engine: &LlmEngine, transcript: &str, dictionary_terms: &[String], + preset: LlmPromptPreset, ) -> Result { if transcript.trim().is_empty() { return Ok(String::new()); } let system_prompt = format!( - "{}{}", + "{}{}{}", CLEANUP_PROMPT, format_dictionary_suffix(dictionary_terms), + preset.suffix(), ); engine.cleanup_text(&system_prompt, transcript) } @@ -134,14 +210,39 @@ mod tests { #[test] fn cleanup_empty_returns_empty_string() { let engine = LlmEngine::new(); - let result = cleanup_text(&engine, "", &[]); + let result = cleanup_text(&engine, "", &[], LlmPromptPreset::Default); assert!(matches!(result, Ok(cleaned) if cleaned.is_empty())); } #[test] fn cleanup_unloaded_returns_not_loaded_error() { let engine = LlmEngine::new(); - let result = cleanup_text(&engine, "um hi there", &[]); + let result = cleanup_text(&engine, "um hi there", &[], LlmPromptPreset::Default); assert!(matches!(result, Err(EngineError::NotLoaded))); } + + #[test] + fn preset_parse_normalises_aliases() { + assert_eq!(LlmPromptPreset::parse("email"), LlmPromptPreset::Email); + assert_eq!(LlmPromptPreset::parse("EMAIL"), LlmPromptPreset::Email); + assert_eq!(LlmPromptPreset::parse("notes"), LlmPromptPreset::Notes); + assert_eq!(LlmPromptPreset::parse("meeting"), LlmPromptPreset::Notes); + assert_eq!(LlmPromptPreset::parse("meeting-notes"), LlmPromptPreset::Notes); + assert_eq!(LlmPromptPreset::parse("code"), LlmPromptPreset::Code); + assert_eq!(LlmPromptPreset::parse("software"), LlmPromptPreset::Code); + // Unknown values and explicit default fall back safely. + assert_eq!(LlmPromptPreset::parse("default"), LlmPromptPreset::Default); + assert_eq!(LlmPromptPreset::parse(""), LlmPromptPreset::Default); + assert_eq!(LlmPromptPreset::parse("random-unknown"), LlmPromptPreset::Default); + } + + #[test] + fn preset_suffix_shapes_tone_without_editing_licence() { + // Each non-default preset must add something; the Default must + // be empty so it composes cleanly with dictionary suffix. + assert!(LlmPromptPreset::Default.suffix().is_empty()); + assert!(LlmPromptPreset::Email.suffix().contains("email")); + assert!(LlmPromptPreset::Notes.suffix().to_lowercase().contains("bullet")); + assert!(LlmPromptPreset::Code.suffix().contains("technical")); + } } diff --git a/crates/ai-formatting/src/pipeline.rs b/crates/ai-formatting/src/pipeline.rs index c183a3b..2d5c935 100644 --- a/crates/ai-formatting/src/pipeline.rs +++ b/crates/ai-formatting/src/pipeline.rs @@ -76,7 +76,17 @@ pub fn post_process_segments( .join(" "); if !joined.is_empty() { - match llm_client::cleanup_text(engine, &joined, &options.dictionary_terms) { + // Pipeline-internal cleanup (used by file-based + live + // transcribe paths) runs with the Default preset. The + // named-preset UX (B.1 #15) flows through the explicit + // cleanup_transcript_text_cmd path instead, where the + // frontend decides which preset the user has selected. + match llm_client::cleanup_text( + engine, + &joined, + &options.dictionary_terms, + llm_client::LlmPromptPreset::Default, + ) { Ok(cleaned) if !cleaned.trim().is_empty() => { replace_segments_with_cleaned(segments, cleaned.trim()); } diff --git a/crates/transcription/src/local_engine.rs b/crates/transcription/src/local_engine.rs index 9f831bb..3803ac5 100644 --- a/crates/transcription/src/local_engine.rs +++ b/crates/transcription/src/local_engine.rs @@ -56,6 +56,23 @@ impl LocalEngine { *id_guard = Some(model_id); } + /// Drop the loaded model and free its backing resources (GPU VRAM, + /// CPU memory, mmap'd GGML tensors). Used by the sequential-GPU + /// guard (brief item A.1 #28) so loading the LLM on a tight-VRAM + /// system first frees the transcription engine, and vice versa. + /// + /// No-op when nothing is loaded. Thread-safe — the internal Mutex + /// serialises against concurrent transcribe_sync calls. + pub fn unload(&self) { + let mut guard = self.engine.lock().unwrap_or_else(|e| e.into_inner()); + *guard = None; + let mut id_guard = self + .loaded_model_id + .lock() + .unwrap_or_else(|e| e.into_inner()); + *id_guard = None; + } + pub fn name(&self) -> &EngineName { &self.engine_name } diff --git a/src-tauri/src/commands/live.rs b/src-tauri/src/commands/live.rs index 68aad38..8dac4b3 100644 --- a/src-tauri/src/commands/live.rs +++ b/src-tauri/src/commands/live.rs @@ -237,7 +237,9 @@ pub async fn start_live_transcription_session( "[live] starting session: engine={}, model={}, language={:?}, save_audio={}", config.engine, model_id, config.language, config.save_audio ); - ensure_model_loaded(&state, &config.engine, &model_id).await?; + // None: live-transcription model loads don't enforce sequential-GPU + // mode. The Settings-level load flow owns that guard. + ensure_model_loaded(&state, &config.engine, &model_id, None).await?; let session_id = live_state .next_session_id diff --git a/src-tauri/src/commands/llm.rs b/src-tauri/src/commands/llm.rs index 5d99017..5ba6095 100644 --- a/src-tauri/src/commands/llm.rs +++ b/src-tauri/src/commands/llm.rs @@ -2,7 +2,7 @@ use tauri::{Emitter, State}; use crate::commands::power::PowerAssertion; use crate::AppState; -use kon_ai_formatting::llm_cleanup_text; +use kon_ai_formatting::{llm_cleanup_text, LlmPromptPreset}; use kon_core::hardware; use kon_llm::model_manager::{self, model_info}; use kon_llm::LlmModelId; @@ -86,6 +86,7 @@ pub async fn load_llm_model( state: State<'_, AppState>, model_id: String, use_gpu: Option, + concurrent: Option, ) -> Result<(), String> { let id = parse_model_id(model_id)?; let path = model_manager::model_path(id); @@ -93,6 +94,15 @@ pub async fn load_llm_model( return Err("Model not downloaded — call download_llm_model first".to_string()); } + // Sequential-GPU guard (brief item A.1 #28): when the user has opted + // out of concurrent GPU residency, free the transcription engines + // before loading the LLM. Prevents VRAM OOM on tight-GPU setups. + // concurrent=None or Some(true) preserves legacy parallel behaviour. + if concurrent == Some(false) { + state.whisper_engine.unload(); + state.parakeet_engine.unload(); + } + let engine = state.llm_engine.clone(); let use_gpu = use_gpu.unwrap_or(true); tokio::task::spawn_blocking(move || engine.load_model(id, &path, use_gpu)) @@ -335,6 +345,7 @@ pub async fn cleanup_transcript_text_cmd( state: State<'_, AppState>, transcript: String, profile_id: Option, + preset: Option, ) -> Result { let resolved_profile_id = profile_id.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string()); @@ -346,13 +357,21 @@ pub async fn cleanup_transcript_text_cmd( .map(|term| term.term) .collect(); + // Named preset (brief item B.1 #15): Email / Notes / Code shape the + // output tone + structure without changing the translator-not-editor + // contract. None or unknown → Default (no additional guidance). + let resolved_preset = preset + .as_deref() + .map(LlmPromptPreset::parse) + .unwrap_or(LlmPromptPreset::Default); + let engine = state.llm_engine.clone(); tokio::task::spawn_blocking(move || { // macOS: pin a power assertion for the duration of the LLM // generation so App Nap can't decide to throttle us mid-token. // No-op on every other OS. Item #9. let _power_guard = PowerAssertion::begin("kon LLM cleanup"); - llm_cleanup_text(&engine, &transcript, &profile_terms) + llm_cleanup_text(&engine, &transcript, &profile_terms, resolved_preset) }) .await .map_err(|e| e.to_string())? diff --git a/src-tauri/src/commands/models.rs b/src-tauri/src/commands/models.rs index 2b644e4..5c5eaae 100644 --- a/src-tauri/src/commands/models.rs +++ b/src-tauri/src/commands/models.rs @@ -112,6 +112,7 @@ pub async fn ensure_model_loaded( state: &AppState, engine_name: &str, model_id: &str, + concurrent: Option, ) -> Result<(), String> { let model_id = ModelId::new(model_id); let entry = model_registry::find_model(&model_id) @@ -143,6 +144,15 @@ pub async fn ensure_model_loaded( return Ok(()); } + // Sequential-GPU guard (brief item A.1 #28): if the user opts out + // of concurrent GPU residency, free the LLM before bringing the + // transcription engine on. None / Some(true) leaves the LLM + // untouched (legacy parallel behaviour, safe on multi-GB VRAM + // setups). Inverse guard lives in commands::llm::load_llm_model. + if concurrent == Some(false) && state.llm_engine.is_loaded() { + state.llm_engine.unload().map_err(|e| e.to_string())?; + } + let engine_clone = engine.clone(); let model_id_clone = model_id.clone(); tokio::task::spawn_blocking(move || { @@ -510,9 +520,13 @@ pub fn list_models() -> Result, String> { } #[tauri::command] -pub async fn load_model(state: tauri::State<'_, AppState>, size: String) -> Result { +pub async fn load_model( + state: tauri::State<'_, AppState>, + size: String, + concurrent: Option, +) -> Result { let id = whisper_model_id(&size); - ensure_model_loaded(&state, "whisper", id.as_str()).await?; + ensure_model_loaded(&state, "whisper", id.as_str(), concurrent).await?; Ok(format!("Model {} loaded", size)) } @@ -561,9 +575,10 @@ pub fn list_parakeet_models() -> Result, String> { pub async fn load_parakeet_model( state: tauri::State<'_, AppState>, name: String, + concurrent: Option, ) -> Result { let id = parakeet_model_id(&name); - ensure_model_loaded(&state, "parakeet", id.as_str()).await?; + ensure_model_loaded(&state, "parakeet", id.as_str(), concurrent).await?; Ok(format!("Parakeet model {} loaded", name)) } diff --git a/src-tauri/src/commands/transcription.rs b/src-tauri/src/commands/transcription.rs index d216ea8..a63b49c 100644 --- a/src-tauri/src/commands/transcription.rs +++ b/src-tauri/src/commands/transcription.rs @@ -261,7 +261,9 @@ pub async fn transcribe_file( let engine_name = engine.unwrap_or_else(|| "whisper".to_string()); let model_id = model_id.unwrap_or_else(|| default_model_id_for_engine(&engine_name).to_string()); - ensure_model_loaded(&state, &engine_name, &model_id).await?; + // None: transcribe paths don't enforce sequential-GPU mode. That's + // owned by the Settings-level load flows (see models.rs). + ensure_model_loaded(&state, &engine_name, &model_id, None).await?; let engine = pick_engine(&state, &engine_name)?; let options = TranscriptionOptions { diff --git a/src/lib/pages/DictationPage.svelte b/src/lib/pages/DictationPage.svelte index 71e2437..7943a7e 100644 --- a/src/lib/pages/DictationPage.svelte +++ b/src/lib/pages/DictationPage.svelte @@ -239,7 +239,13 @@ try { const status = await invoke("check_llm_model", { modelId: settings.llmModelId }); if (status?.downloaded && !status.loaded) { - await invoke("load_llm_model", { modelId: settings.llmModelId }); + await invoke("load_llm_model", { + modelId: settings.llmModelId, + // Sequential-GPU guard (brief item A.1 #28): frees whisper + // before loading the LLM on tight-VRAM setups. "parallel" + // keeps both resident (default, safe on multi-GB cards). + concurrent: settings.aiGpuConcurrency === "parallel", + }); } } catch (err) { console.warn("ensureLlmModelLoaded failed", err); @@ -257,10 +263,15 @@ error = ""; try { + // concurrent flag mirrors settings.aiGpuConcurrency (brief A.1 #28). + const concurrent = settings.aiGpuConcurrency === "parallel"; if (settings.engine === "parakeet") { - await invoke("load_parakeet_model", { name: "ctc-int8" }); + await invoke("load_parakeet_model", { name: "ctc-int8", concurrent }); } else { - await invoke("load_model", { size: settings.modelSize.toLowerCase() }); + await invoke("load_model", { + size: settings.modelSize.toLowerCase(), + concurrent, + }); } await refreshRuntimeCapabilities(); modelReady = true; @@ -460,6 +471,7 @@ const cleaned = await invoke("cleanup_transcript_text_cmd", { transcript: text, profileId: profilesStore.activeProfileId, + preset: settings.llmPromptPreset, }); markGenerationDone(true); return cleaned?.trim() ? cleaned.trim() : text; diff --git a/src/lib/pages/SettingsPage.svelte b/src/lib/pages/SettingsPage.svelte index 9e03529..4e5f42d 100644 --- a/src/lib/pages/SettingsPage.svelte +++ b/src/lib/pages/SettingsPage.svelte @@ -531,7 +531,11 @@ const modelId = selectedLlmModelId(); llmStatus = "Loading..."; try { - await invoke("load_llm_model", { modelId }); + await invoke("load_llm_model", { + modelId, + // Sequential-GPU guard (brief item A.1 #28). + concurrent: settings.aiGpuConcurrency === "parallel", + }); await refreshLlmStatus(); await refreshGlobalLlmStatus(settings.aiTier); } catch (err) { @@ -740,7 +744,10 @@ engineStatus = "Loading..."; engineOk = false; try { - await invoke("load_model", { size }); + await invoke("load_model", { + size, + concurrent: settings.aiGpuConcurrency === "parallel", + }); await refreshRuntimeCapabilities(); engineOk = true; engineStatus = `${settings.modelSize} model loaded`; @@ -796,7 +803,10 @@ parakeetStatus = "Loading..."; parakeetOk = false; try { - await invoke("load_parakeet_model", { name: "ctc-int8" }); + await invoke("load_parakeet_model", { + name: "ctc-int8", + concurrent: settings.aiGpuConcurrency === "parallel", + }); await refreshRuntimeCapabilities(); parakeetOk = true; parakeetStatus = "Model loaded"; @@ -1505,6 +1515,48 @@ {/if}

+ + +
+

Cleanup preset

+ +

+ {#if settings.llmPromptPreset === "email"} + Formats output as an email paragraph — tight sentences, no markdown, no auto-added greeting or signoff. + {:else if settings.llmPromptPreset === "notes"} + Formats action items as a bullet list led by imperative verbs; keeps prose informational sentences as prose. + {:else if settings.llmPromptPreset === "code"} + Preserves technical terms, variable names, file paths, and symbols exactly as spoken. No translation of identifiers. + {:else} + No preset — the active profile's prompt governs tone alone. + {/if} +

+
+ + +
+

GPU concurrency

+ +

+ {#if settings.aiGpuConcurrency === "sequential"} + On tight-VRAM cards (≤6 GB), loading Whisper + LLM together OOMs. Sequential mode frees the other model before loading; adds a small reload pause between transcribe and cleanup. + {:else} + Both models stay resident in GPU memory. Faster transitions, but needs enough VRAM to hold both at once. + {/if} +

+
{/if} diff --git a/src/lib/stores/page.svelte.ts b/src/lib/stores/page.svelte.ts index dcde724..74f5560 100644 --- a/src/lib/stores/page.svelte.ts +++ b/src/lib/stores/page.svelte.ts @@ -61,6 +61,8 @@ const defaults: SettingsState = { fontSize: 14, aiTier: "cleanup", llmModelId: null, + llmPromptPreset: "default", + aiGpuConcurrency: "parallel", saveAudio: false, outputFolder: "", globalHotkey: "Ctrl+Shift+R", diff --git a/src/lib/types/app.ts b/src/lib/types/app.ts index 9665c10..415a2fc 100644 --- a/src/lib/types/app.ts +++ b/src/lib/types/app.ts @@ -12,6 +12,8 @@ export type WhisperModelSize = | "Distil-L"; export type AiTier = "off" | "cleanup" | "tasks"; export type LlmModelIdStr = "qwen3_1_7b" | "qwen3_4b_instruct_2507" | "qwen3_14b"; +export type LlmPromptPreset = "default" | "email" | "notes" | "code"; +export type AiGpuConcurrency = "parallel" | "sequential"; export type TaskBucket = "inbox" | "today" | "soon" | "later"; export type ToastSeverity = "info" | "success" | "warn" | "error"; @@ -47,6 +49,8 @@ export interface SettingsState { fontSize: number; aiTier: AiTier; llmModelId: LlmModelIdStr | null; + llmPromptPreset: LlmPromptPreset; + aiGpuConcurrency: AiGpuConcurrency; saveAudio: boolean; outputFolder: string; globalHotkey: string;