Two new Settings → AI knobs that compose cleanly with what already shipped (aiTier, LLM model, translator prompt framing). **B.1 #15 — Named cleanup presets.** LlmPromptPreset enum (Default / Email / Notes / Code) appends a short context hint onto the CLEANUP_PROMPT just before generation. Presets shape tone and structure ("email paragraph", "bulleted meeting notes", "preserve technical terms") without licensing the content-editing the translator-not-editor framing forbids. cleanup_transcript_text_cmd now takes `preset: Option<String>` which runs through the new LlmPromptPreset::parse (normalises aliases like "meeting-notes", collapses unknown values to Default). **A.1 #28 — Sequential-GPU guard.** New LocalEngine::unload drops the backend + model_id so a subsequent load actually reclaims VRAM. load_llm_model, load_model, and load_parakeet_model Tauri commands grow an optional `concurrent: bool` argument. When concurrent is Some(false), loading LLM first unloads whisper+parakeet, and vice versa — prevents VRAM OOM on tight-VRAM setups. Default is the previous parallel behaviour so nothing changes for multi-GB cards. Transcribe-in-progress paths (transcribe_pcm, transcribe_file, live) pass None, so mid-dictation model loads don't accidentally tear down the LLM. Settings UI (AI section): - Cleanup preset segmented button + descriptive copy for each option. - GPU concurrency segmented button with explicit trade-off text ("faster transitions vs fits in tight VRAM"). Frontend wiring: - settings.llmPromptPreset flows from DictationPage's cleanupTranscriptIfEnabled into the Tauri command. - settings.aiGpuConcurrency flows from both DictationPage (auto-load on record) and SettingsPage (manual load/unload buttons) as `concurrent: "parallel" === true` to the load commands. Tests: three new preset cases in crates/ai-formatting/src/llm_client.rs (parse aliases, suffix non-empty for non-default, default suffix empty). All 139 existing lib tests still pass. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -237,7 +237,9 @@ pub async fn start_live_transcription_session(
|
||||
"[live] starting session: engine={}, model={}, language={:?}, save_audio={}",
|
||||
config.engine, model_id, config.language, config.save_audio
|
||||
);
|
||||
ensure_model_loaded(&state, &config.engine, &model_id).await?;
|
||||
// None: live-transcription model loads don't enforce sequential-GPU
|
||||
// mode. The Settings-level load flow owns that guard.
|
||||
ensure_model_loaded(&state, &config.engine, &model_id, None).await?;
|
||||
|
||||
let session_id = live_state
|
||||
.next_session_id
|
||||
|
||||
@@ -2,7 +2,7 @@ use tauri::{Emitter, State};
|
||||
|
||||
use crate::commands::power::PowerAssertion;
|
||||
use crate::AppState;
|
||||
use kon_ai_formatting::llm_cleanup_text;
|
||||
use kon_ai_formatting::{llm_cleanup_text, LlmPromptPreset};
|
||||
use kon_core::hardware;
|
||||
use kon_llm::model_manager::{self, model_info};
|
||||
use kon_llm::LlmModelId;
|
||||
@@ -86,6 +86,7 @@ pub async fn load_llm_model(
|
||||
state: State<'_, AppState>,
|
||||
model_id: String,
|
||||
use_gpu: Option<bool>,
|
||||
concurrent: Option<bool>,
|
||||
) -> Result<(), String> {
|
||||
let id = parse_model_id(model_id)?;
|
||||
let path = model_manager::model_path(id);
|
||||
@@ -93,6 +94,15 @@ pub async fn load_llm_model(
|
||||
return Err("Model not downloaded — call download_llm_model first".to_string());
|
||||
}
|
||||
|
||||
// Sequential-GPU guard (brief item A.1 #28): when the user has opted
|
||||
// out of concurrent GPU residency, free the transcription engines
|
||||
// before loading the LLM. Prevents VRAM OOM on tight-GPU setups.
|
||||
// concurrent=None or Some(true) preserves legacy parallel behaviour.
|
||||
if concurrent == Some(false) {
|
||||
state.whisper_engine.unload();
|
||||
state.parakeet_engine.unload();
|
||||
}
|
||||
|
||||
let engine = state.llm_engine.clone();
|
||||
let use_gpu = use_gpu.unwrap_or(true);
|
||||
tokio::task::spawn_blocking(move || engine.load_model(id, &path, use_gpu))
|
||||
@@ -335,6 +345,7 @@ pub async fn cleanup_transcript_text_cmd(
|
||||
state: State<'_, AppState>,
|
||||
transcript: String,
|
||||
profile_id: Option<String>,
|
||||
preset: Option<String>,
|
||||
) -> Result<String, String> {
|
||||
let resolved_profile_id =
|
||||
profile_id.unwrap_or_else(|| kon_storage::DEFAULT_PROFILE_ID.to_string());
|
||||
@@ -346,13 +357,21 @@ pub async fn cleanup_transcript_text_cmd(
|
||||
.map(|term| term.term)
|
||||
.collect();
|
||||
|
||||
// Named preset (brief item B.1 #15): Email / Notes / Code shape the
|
||||
// output tone + structure without changing the translator-not-editor
|
||||
// contract. None or unknown → Default (no additional guidance).
|
||||
let resolved_preset = preset
|
||||
.as_deref()
|
||||
.map(LlmPromptPreset::parse)
|
||||
.unwrap_or(LlmPromptPreset::Default);
|
||||
|
||||
let engine = state.llm_engine.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
// macOS: pin a power assertion for the duration of the LLM
|
||||
// generation so App Nap can't decide to throttle us mid-token.
|
||||
// No-op on every other OS. Item #9.
|
||||
let _power_guard = PowerAssertion::begin("kon LLM cleanup");
|
||||
llm_cleanup_text(&engine, &transcript, &profile_terms)
|
||||
llm_cleanup_text(&engine, &transcript, &profile_terms, resolved_preset)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
|
||||
@@ -112,6 +112,7 @@ pub async fn ensure_model_loaded(
|
||||
state: &AppState,
|
||||
engine_name: &str,
|
||||
model_id: &str,
|
||||
concurrent: Option<bool>,
|
||||
) -> Result<(), String> {
|
||||
let model_id = ModelId::new(model_id);
|
||||
let entry = model_registry::find_model(&model_id)
|
||||
@@ -143,6 +144,15 @@ pub async fn ensure_model_loaded(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Sequential-GPU guard (brief item A.1 #28): if the user opts out
|
||||
// of concurrent GPU residency, free the LLM before bringing the
|
||||
// transcription engine on. None / Some(true) leaves the LLM
|
||||
// untouched (legacy parallel behaviour, safe on multi-GB VRAM
|
||||
// setups). Inverse guard lives in commands::llm::load_llm_model.
|
||||
if concurrent == Some(false) && state.llm_engine.is_loaded() {
|
||||
state.llm_engine.unload().map_err(|e| e.to_string())?;
|
||||
}
|
||||
|
||||
let engine_clone = engine.clone();
|
||||
let model_id_clone = model_id.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
@@ -510,9 +520,13 @@ pub fn list_models() -> Result<Vec<String>, String> {
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn load_model(state: tauri::State<'_, AppState>, size: String) -> Result<String, String> {
|
||||
pub async fn load_model(
|
||||
state: tauri::State<'_, AppState>,
|
||||
size: String,
|
||||
concurrent: Option<bool>,
|
||||
) -> Result<String, String> {
|
||||
let id = whisper_model_id(&size);
|
||||
ensure_model_loaded(&state, "whisper", id.as_str()).await?;
|
||||
ensure_model_loaded(&state, "whisper", id.as_str(), concurrent).await?;
|
||||
Ok(format!("Model {} loaded", size))
|
||||
}
|
||||
|
||||
@@ -561,9 +575,10 @@ pub fn list_parakeet_models() -> Result<Vec<String>, String> {
|
||||
pub async fn load_parakeet_model(
|
||||
state: tauri::State<'_, AppState>,
|
||||
name: String,
|
||||
concurrent: Option<bool>,
|
||||
) -> Result<String, String> {
|
||||
let id = parakeet_model_id(&name);
|
||||
ensure_model_loaded(&state, "parakeet", id.as_str()).await?;
|
||||
ensure_model_loaded(&state, "parakeet", id.as_str(), concurrent).await?;
|
||||
Ok(format!("Parakeet model {} loaded", name))
|
||||
}
|
||||
|
||||
|
||||
@@ -261,7 +261,9 @@ pub async fn transcribe_file(
|
||||
let engine_name = engine.unwrap_or_else(|| "whisper".to_string());
|
||||
let model_id =
|
||||
model_id.unwrap_or_else(|| default_model_id_for_engine(&engine_name).to_string());
|
||||
ensure_model_loaded(&state, &engine_name, &model_id).await?;
|
||||
// None: transcribe paths don't enforce sequential-GPU mode. That's
|
||||
// owned by the Settings-level load flows (see models.rs).
|
||||
ensure_model_loaded(&state, &engine_name, &model_id, None).await?;
|
||||
|
||||
let engine = pick_engine(&state, &engine_name)?;
|
||||
let options = TranscriptionOptions {
|
||||
|
||||
Reference in New Issue
Block a user