feat(llm): add kon-llm crate with llama-cpp-2 inference, model management, and Tauri commands

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

src-tauri/Cargo.toml

@@ -20,6 +20,7 @@ kon-transcription = { path = "../crates/transcription" }
 kon-ai-formatting = { path = "../crates/ai-formatting" }
 kon-storage = { path = "../crates/storage" }
 kon-cloud-providers = { path = "../crates/cloud-providers" }
+kon-llm = { path = "../crates/llm" }
 
 # Tauri
 tauri = { version = "2", features = ["tray-icon"] }

src-tauri/src/commands/llm.rs

@@ -0,0 +1,213 @@
+use std::sync::Arc;
+
+use serde::Serialize;
+use tauri::Emitter;
+
+use crate::AppState;
+use kon_llm::{
+    LlmModelEntry, LLM_MODELS,
+    is_llm_downloaded, download_llm_model,
+    model_manager::llm_model_path,
+};
+
+/// Serialisable LLM model info for the frontend.
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct LlmModelInfo {
+    pub id: String,
+    pub display_name: String,
+    pub disk_size_mb: u64,
+    pub ram_required_mb: u64,
+    pub description: String,
+    pub is_downloaded: bool,
+}
+
+fn model_info(entry: &LlmModelEntry) -> LlmModelInfo {
+    LlmModelInfo {
+        id: entry.id.to_string(),
+        display_name: entry.display_name.to_string(),
+        disk_size_mb: entry.disk_size.0,
+        ram_required_mb: entry.ram_required.0,
+        description: entry.description.to_string(),
+        is_downloaded: is_llm_downloaded(entry.id),
+    }
+}
+
+/// List available LLM models with download status.
+#[tauri::command]
+pub async fn list_llm_models() -> Vec<LlmModelInfo> {
+    LLM_MODELS.iter().map(model_info).collect()
+}
+
+/// Download an LLM model. Emits "llm-download-progress" events.
+#[tauri::command]
+pub async fn download_llm(
+    app: tauri::AppHandle,
+    id: String,
+) -> Result<(), String> {
+    let app_handle = app.clone();
+    download_llm_model(&id, move |progress| {
+        let _ = app_handle.emit("llm-download-progress", serde_json::json!({
+            "modelId": progress.model_id.as_str(),
+            "fileName": progress.file_name,
+            "bytesDownloaded": progress.bytes_downloaded,
+            "totalBytes": progress.total_bytes,
+            "percent": progress.percent,
+        }));
+    })
+    .await
+    .map_err(|e| e.to_string())?;
+
+    Ok(())
+}
+
+/// Load a downloaded LLM model into the engine.
+#[tauri::command]
+pub async fn load_llm(
+    state: tauri::State<'_, AppState>,
+    id: String,
+) -> Result<(), String> {
+    let path = llm_model_path(&id)
+        .ok_or_else(|| format!("Unknown model: {id}"))?;
+
+    if !path.exists() {
+        return Err(format!("Model not downloaded: {id}"));
+    }
+
+    let engine = state.llm_engine.clone();
+    tokio::task::spawn_blocking(move || {
+        engine.load(&path)
+    })
+    .await
+    .map_err(|e| e.to_string())?
+    .map_err(|e| e.to_string())
+}
+
+/// Check whether an LLM model is currently loaded.
+#[tauri::command]
+pub async fn check_llm_engine(
+    state: tauri::State<'_, AppState>,
+) -> Result<bool, String> {
+    Ok(state.llm_engine.is_loaded())
+}
+
+/// Run LLM inference with a system prompt and user prompt.
+#[tauri::command]
+pub async fn llm_generate(
+    state: tauri::State<'_, AppState>,
+    system_prompt: String,
+    user_prompt: String,
+    max_tokens: Option<u32>,
+) -> Result<String, String> {
+    let engine = state.llm_engine.clone();
+    let max = max_tokens.unwrap_or(512);
+
+    kon_llm::inference::run_llm_inference(engine, system_prompt, user_prompt, max)
+        .await
+        .map_err(|e| e.to_string())
+}
+
+/// Extract tasks from transcript text using LLM (falls back to empty if no model loaded).
+#[tauri::command]
+pub async fn extract_tasks_llm(
+    state: tauri::State<'_, AppState>,
+    transcript_text: String,
+) -> Result<Vec<TaskSuggestion>, String> {
+    if !state.llm_engine.is_loaded() {
+        // No LLM available — return empty (frontend falls back to rule-based JS extractor)
+        return Ok(vec![]);
+    }
+
+    let engine = state.llm_engine.clone();
+    let system_prompt = r#"Extract actionable tasks from the following voice transcription.
+Each task must start with a concrete verb.
+Return as JSON array of {"title": "...", "priority": "high|medium|low", "project": "..."}.
+Only extract genuine tasks — not observations or comments.
+If no tasks found, return empty array []."#.to_string();
+
+    let result = kon_llm::inference::run_llm_inference(
+        engine,
+        system_prompt,
+        transcript_text,
+        256,
+    )
+    .await
+    .map_err(|e| e.to_string())?;
+
+    // Parse JSON response — handle malformed LLM output gracefully
+    parse_task_suggestions(&result)
+}
+
+/// Decompose a task into 3-7 micro-steps using LLM.
+#[tauri::command]
+pub async fn decompose_task(
+    state: tauri::State<'_, AppState>,
+    task_text: String,
+) -> Result<Vec<String>, String> {
+    if !state.llm_engine.is_loaded() {
+        return Err("AI assistant not loaded — download a model in Settings to enable micro-stepping.".to_string());
+    }
+
+    let engine = state.llm_engine.clone();
+    let system_prompt = r#"Break this task into 3-7 micro-steps.
+Each step MUST start with a specific physical verb (e.g. 'Open', 'Type', 'Click', 'Pick up').
+Each step must be completable in under 5 minutes.
+Never use abstract verbs like 'organise', 'plan', 'consider'.
+Return as JSON array of strings."#.to_string();
+
+    let result = kon_llm::inference::run_llm_inference(
+        engine,
+        system_prompt,
+        task_text,
+        256,
+    )
+    .await
+    .map_err(|e| e.to_string())?;
+
+    parse_string_array(&result)
+}
+
+// --- Response types ---
+
+#[derive(Serialize, serde::Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TaskSuggestion {
+    pub title: String,
+    pub priority: String,
+    pub project: Option<String>,
+}
+
+/// Parse LLM output as a JSON array of TaskSuggestion.
+/// Handles common LLM output quirks: markdown code fences, trailing text.
+fn parse_task_suggestions(raw: &str) -> Result<Vec<TaskSuggestion>, String> {
+    let json = extract_json_array(raw);
+    serde_json::from_str::<Vec<TaskSuggestion>>(&json)
+        .map_err(|e| format!("Failed to parse LLM task output: {e}"))
+}
+
+/// Parse LLM output as a JSON array of strings.
+fn parse_string_array(raw: &str) -> Result<Vec<String>, String> {
+    let json = extract_json_array(raw);
+    serde_json::from_str::<Vec<String>>(&json)
+        .map_err(|e| format!("Failed to parse LLM micro-step output: {e}"))
+}
+
+/// Extract a JSON array from potentially messy LLM output.
+/// Strips markdown code fences, finds the first [ ... ] pair.
+fn extract_json_array(raw: &str) -> String {
+    let cleaned = raw
+        .replace("```json", "")
+        .replace("```", "")
+        .trim()
+        .to_string();
+
+    // Find the first [ ... ] pair
+    if let Some(start) = cleaned.find('[') {
+        if let Some(end) = cleaned.rfind(']') {
+            return cleaned[start..=end].to_string();
+        }
+    }
+
+    // Fallback: return empty array
+    "[]".to_string()
+}

src-tauri/src/commands/mod.rs

@@ -2,6 +2,7 @@ pub mod audio;
 pub mod clipboard;
 pub mod hardware;
 pub mod history;
+pub mod llm;
 pub mod models;
 pub mod tasks;
 pub mod timer;

src-tauri/src/lib.rs

@@ -7,13 +7,15 @@ use sqlx::SqlitePool;
 use tauri::Manager;
 
 use kon_core::types::EngineName;
+use kon_llm::LlmEngine;
 use kon_storage::{database_path, get_setting, init as init_db, set_setting};
 use kon_transcription::LocalEngine;
 
-/// Shared app state holding the transcription engines and database pool.
+/// Shared app state holding the transcription engines, LLM engine, and database pool.
 pub struct AppState {
     pub whisper_engine: Arc<LocalEngine>,
     pub parakeet_engine: Arc<LocalEngine>,
+    pub llm_engine: Arc<LlmEngine>,
     pub db: SqlitePool,
 }
 
@@ -108,6 +110,11 @@ pub fn run() {
             // Store init script for secondary windows (float, viewer)
             app.manage(PreferencesScript(init_script));
 
+            // Initialise LLM engine (model loaded on demand via settings)
+            let llm_engine = Arc::new(
+                LlmEngine::new().expect("LLM backend should initialise")
+            );
+
             app.manage(AppState {
                 whisper_engine: Arc::new(LocalEngine::new(
                     EngineName::new("whisper"),
@@ -115,6 +122,7 @@ pub fn run() {
                 parakeet_engine: Arc::new(LocalEngine::new(
                     EngineName::new("parakeet"),
                 )),
+                llm_engine,
                 db,
             });
 
@@ -155,6 +163,14 @@ pub fn run() {
             commands::tasks::reorder_tasks_cmd,
             commands::tasks::complete_task_cmd,
             commands::tasks::uncomplete_task_cmd,
+            // LLM
+            commands::llm::list_llm_models,
+            commands::llm::download_llm,
+            commands::llm::load_llm,
+            commands::llm::check_llm_engine,
+            commands::llm::llm_generate,
+            commands::llm::extract_tasks_llm,
+            commands::llm::decompose_task,
             // Timer
             commands::timer::save_timer,
             commands::timer::get_timer,