feat(llm): add kon-llm crate with llama-cpp-2 inference, model management, and Tauri commands

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/llm/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "kon-llm"
+version = "0.1.0"
+edition = "2021"
+description = "Local LLM inference via llama.cpp for Kon"
+
+[dependencies]
+kon-core = { path = "../core" }
+llama-cpp-2 = "0.1"
+tokio = { version = "1", features = ["rt", "sync"] }
+reqwest = { version = "0.12", features = ["stream"] }
+futures-util = "0.3"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+log = "0.4"

crates/llm/src/inference.rs

@@ -0,0 +1,144 @@
+use std::path::Path;
+use std::sync::Mutex;
+
+use llama_cpp_2::context::params::LlamaContextParams;
+use llama_cpp_2::llama_backend::LlamaBackend;
+use llama_cpp_2::llama_batch::LlamaBatch;
+use llama_cpp_2::model::params::LlamaModelParams;
+use llama_cpp_2::model::{AddBos, LlamaModel, Special};
+use llama_cpp_2::sampling::LlamaSampler;
+
+use kon_core::error::{KonError, Result};
+
+/// Thread-safe LLM inference engine wrapping llama.cpp.
+pub struct LlmEngine {
+    backend: LlamaBackend,
+    model: Mutex<Option<LlamaModel>>,
+    loaded_path: Mutex<Option<String>>,
+}
+
+// Safety: LlamaBackend and LlamaModel are thread-safe for read access.
+// The Mutex guards all mutation.
+unsafe impl Send for LlmEngine {}
+unsafe impl Sync for LlmEngine {}
+
+impl LlmEngine {
+    /// Create a new engine. Call `load()` before inference.
+    pub fn new() -> Result<Self> {
+        let backend = LlamaBackend::init()
+            .map_err(|e| KonError::Other(format!("LLM backend init failed: {e}")))?;
+        Ok(Self {
+            backend,
+            model: Mutex::new(None),
+            loaded_path: Mutex::new(None),
+        })
+    }
+
+    /// Load a GGUF model from disk.
+    pub fn load(&self, model_path: &Path) -> Result<()> {
+        let params = LlamaModelParams::default();
+        let model = LlamaModel::load_from_file(&self.backend, model_path, &params)
+            .map_err(|e| KonError::Other(format!("Model load failed: {e}")))?;
+
+        *self.model.lock().unwrap() = Some(model);
+        *self.loaded_path.lock().unwrap() = Some(model_path.to_string_lossy().to_string());
+
+        log::info!("LLM model loaded: {}", model_path.display());
+        Ok(())
+    }
+
+    /// Whether a model is currently loaded.
+    pub fn is_loaded(&self) -> bool {
+        self.model.lock().unwrap().is_some()
+    }
+
+    /// Generate text from a prompt. Blocking — call from spawn_blocking.
+    ///
+    /// Uses a system prompt + user prompt pattern. The system prompt sets
+    /// the behaviour (e.g. task extraction), the user prompt is the input.
+    pub fn generate(&self, system_prompt: &str, user_prompt: &str, max_tokens: u32) -> Result<String> {
+        let guard = self.model.lock().unwrap();
+        let model = guard.as_ref()
+            .ok_or(KonError::EngineNotLoaded)?;
+
+        // Format as chat-style prompt (works with most instruction-tuned models)
+        let full_prompt = format!(
+            "<|system|>\n{system_prompt}<|end|>\n<|user|>\n{user_prompt}<|end|>\n<|assistant|>\n"
+        );
+
+        let ctx_params = LlamaContextParams::default()
+            .with_n_ctx(std::num::NonZeroU32::new(2048));
+        let mut ctx = model.new_context(&self.backend, ctx_params)
+            .map_err(|e| KonError::Other(format!("Context creation failed: {e}")))?;
+
+        // Tokenise
+        let tokens = model.str_to_token(&full_prompt, AddBos::Always)
+            .map_err(|e| KonError::Other(format!("Tokenisation failed: {e}")))?;
+
+        // Create batch and add prompt tokens
+        let mut batch = LlamaBatch::new(2048, 1);
+        for (i, token) in tokens.iter().enumerate() {
+            let is_last = i == tokens.len() - 1;
+            batch.add(*token, i as i32, &[0], is_last)
+                .map_err(|e| KonError::Other(format!("Batch add failed: {e}")))?;
+        }
+
+        // Process prompt
+        ctx.decode(&mut batch)
+            .map_err(|e| KonError::Other(format!("Prompt decode failed: {e}")))?;
+
+        // Sample tokens
+        let mut sampler = LlamaSampler::greedy();
+        let mut output = String::new();
+        let mut n_decoded = tokens.len() as i32;
+
+        for _ in 0..max_tokens {
+            let new_token = sampler.sample(&ctx, batch.n_tokens() - 1);
+            sampler.accept(new_token);
+
+            if model.is_eog_token(new_token) {
+                break;
+            }
+
+            let token_str = model.token_to_str(new_token, Special::Tokenize)
+                .map_err(|e| KonError::Other(format!("Token decode failed: {e}")))?;
+            output.push_str(&token_str);
+
+            // Stop if we see end-of-assistant markers
+            if output.contains("<|end|>") || output.contains("<|user|>") {
+                // Trim the marker
+                if let Some(pos) = output.find("<|end|>") {
+                    output.truncate(pos);
+                }
+                if let Some(pos) = output.find("<|user|>") {
+                    output.truncate(pos);
+                }
+                break;
+            }
+
+            batch.clear();
+            batch.add(new_token, n_decoded, &[0], true)
+                .map_err(|e| KonError::Other(format!("Batch add failed: {e}")))?;
+            n_decoded += 1;
+
+            ctx.decode(&mut batch)
+                .map_err(|e| KonError::Other(format!("Token decode failed: {e}")))?;
+        }
+
+        Ok(output.trim().to_string())
+    }
+}
+
+/// Run LLM inference on a blocking thread.
+pub async fn run_llm_inference(
+    engine: std::sync::Arc<LlmEngine>,
+    system_prompt: String,
+    user_prompt: String,
+    max_tokens: u32,
+) -> Result<String> {
+    tokio::task::spawn_blocking(move || {
+        engine.generate(&system_prompt, &user_prompt, max_tokens)
+    })
+    .await
+    .map_err(|e| KonError::Other(format!("LLM inference thread failed: {e}")))?
+}

crates/llm/src/lib.rs

@@ -0,0 +1,5 @@
+pub mod inference;
+pub mod model_manager;
+
+pub use inference::LlmEngine;
+pub use model_manager::{LlmModelEntry, LLM_MODELS, llm_models_dir, is_llm_downloaded, download_llm_model};

crates/llm/src/model_manager.rs

@@ -0,0 +1,130 @@
+use std::path::PathBuf;
+
+use kon_core::error::{KonError, Result};
+use kon_core::types::{DownloadProgress, Megabytes, ModelId};
+
+/// Metadata for an LLM model in the catalogue.
+#[derive(Debug, Clone)]
+pub struct LlmModelEntry {
+    pub id: &'static str,
+    pub display_name: &'static str,
+    pub url: &'static str,
+    pub disk_size: Megabytes,
+    pub ram_required: Megabytes,
+    pub filename: &'static str,
+    pub description: &'static str,
+}
+
+/// LLM model catalogue — hardware-tiered options.
+pub const LLM_MODELS: &[LlmModelEntry] = &[
+    LlmModelEntry {
+        id: "phi-4-mini-q4",
+        display_name: "Phi-4 Mini (8GB RAM)",
+        url: "https://huggingface.co/bartowski/phi-4-mini-instruct-GGUF/resolve/main/phi-4-mini-instruct-Q4_K_M.gguf",
+        disk_size: Megabytes(2400),
+        ram_required: Megabytes(4000),
+        filename: "phi-4-mini-instruct-Q4_K_M.gguf",
+        description: "Compact and fast — ideal for 8GB systems",
+    },
+    LlmModelEntry {
+        id: "qwen3-7b-q4",
+        display_name: "Qwen 3 7B (16GB RAM)",
+        url: "https://huggingface.co/bartowski/Qwen3-8B-GGUF/resolve/main/Qwen3-8B-Q4_K_M.gguf",
+        disk_size: Megabytes(4900),
+        ram_required: Megabytes(8000),
+        filename: "Qwen3-8B-Q4_K_M.gguf",
+        description: "Higher quality — recommended for 16GB+ systems",
+    },
+];
+
+/// Directory for LLM GGUF models.
+pub fn llm_models_dir() -> PathBuf {
+    if cfg!(target_os = "windows") {
+        let local = std::env::var("LOCALAPPDATA").unwrap_or_else(|_| ".".to_string());
+        PathBuf::from(local).join("kon").join("llm-models")
+    } else {
+        let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
+        PathBuf::from(home).join(".kon").join("llm-models")
+    }
+}
+
+/// Check whether a model's GGUF file exists on disk.
+pub fn is_llm_downloaded(model_id: &str) -> bool {
+    if let Some(entry) = LLM_MODELS.iter().find(|m| m.id == model_id) {
+        llm_models_dir().join(entry.filename).exists()
+    } else {
+        false
+    }
+}
+
+/// Get the file path for a downloaded model.
+pub fn llm_model_path(model_id: &str) -> Option<PathBuf> {
+    LLM_MODELS
+        .iter()
+        .find(|m| m.id == model_id)
+        .map(|entry| llm_models_dir().join(entry.filename))
+}
+
+/// Download a GGUF model with progress callback.
+pub async fn download_llm_model(
+    model_id: &str,
+    on_progress: impl Fn(DownloadProgress) + Send + 'static,
+) -> Result<PathBuf> {
+    let entry = LLM_MODELS
+        .iter()
+        .find(|m| m.id == model_id)
+        .ok_or_else(|| KonError::ModelNotFound(ModelId::new(model_id)))?;
+
+    let dir = llm_models_dir();
+    std::fs::create_dir_all(&dir)?;
+
+    let dest = dir.join(entry.filename);
+    let part = dir.join(format!("{}.part", entry.filename));
+
+    // Stream download with progress
+    let response = reqwest::get(entry.url)
+        .await
+        .map_err(|e| KonError::DownloadFailed(format!("Request failed: {e}")))?;
+
+    let total = response.content_length().unwrap_or(0);
+
+    let mut stream = response.bytes_stream();
+    let mut file = tokio::fs::File::create(&part)
+        .await
+        .map_err(|e| KonError::DownloadFailed(format!("File create failed: {e}")))?;
+
+    let mut downloaded: u64 = 0;
+    let model_id_owned = ModelId::new(model_id);
+
+    use futures_util::StreamExt;
+    use tokio::io::AsyncWriteExt;
+
+    while let Some(chunk) = stream.next().await {
+        let chunk = chunk.map_err(|e| KonError::DownloadFailed(format!("Download chunk failed: {e}")))?;
+        file.write_all(&chunk)
+            .await
+            .map_err(|e| KonError::DownloadFailed(format!("Write failed: {e}")))?;
+
+        downloaded += chunk.len() as u64;
+        let percent = if total > 0 { (downloaded as f64 / total as f64 * 100.0) as u8 } else { 0 };
+
+        on_progress(DownloadProgress {
+            model_id: model_id_owned.clone(),
+            file_name: entry.filename.to_string(),
+            bytes_downloaded: downloaded,
+            total_bytes: total,
+            percent,
+        });
+    }
+
+    file.flush().await
+        .map_err(|e| KonError::DownloadFailed(format!("Flush failed: {e}")))?;
+    drop(file);
+
+    // Atomic rename
+    std::fs::rename(&part, &dest)
+        .map_err(|e| KonError::DownloadFailed(format!("Rename failed: {e}")))?;
+
+    log::info!("LLM model downloaded: {} → {}", model_id, dest.display());
+    Ok(dest)
+}

src-tauri/Cargo.toml

@@ -20,6 +20,7 @@ kon-transcription = { path = "../crates/transcription" }
 kon-ai-formatting = { path = "../crates/ai-formatting" }
 kon-storage = { path = "../crates/storage" }
 kon-cloud-providers = { path = "../crates/cloud-providers" }
+kon-llm = { path = "../crates/llm" }
 
 # Tauri
 tauri = { version = "2", features = ["tray-icon"] }

src-tauri/src/commands/llm.rs

@@ -0,0 +1,213 @@
+use std::sync::Arc;
+
+use serde::Serialize;
+use tauri::Emitter;
+
+use crate::AppState;
+use kon_llm::{
+    LlmModelEntry, LLM_MODELS,
+    is_llm_downloaded, download_llm_model,
+    model_manager::llm_model_path,
+};
+
+/// Serialisable LLM model info for the frontend.
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct LlmModelInfo {
+    pub id: String,
+    pub display_name: String,
+    pub disk_size_mb: u64,
+    pub ram_required_mb: u64,
+    pub description: String,
+    pub is_downloaded: bool,
+}
+
+fn model_info(entry: &LlmModelEntry) -> LlmModelInfo {
+    LlmModelInfo {
+        id: entry.id.to_string(),
+        display_name: entry.display_name.to_string(),
+        disk_size_mb: entry.disk_size.0,
+        ram_required_mb: entry.ram_required.0,
+        description: entry.description.to_string(),
+        is_downloaded: is_llm_downloaded(entry.id),
+    }
+}
+
+/// List available LLM models with download status.
+#[tauri::command]
+pub async fn list_llm_models() -> Vec<LlmModelInfo> {
+    LLM_MODELS.iter().map(model_info).collect()
+}
+
+/// Download an LLM model. Emits "llm-download-progress" events.
+#[tauri::command]
+pub async fn download_llm(
+    app: tauri::AppHandle,
+    id: String,
+) -> Result<(), String> {
+    let app_handle = app.clone();
+    download_llm_model(&id, move |progress| {
+        let _ = app_handle.emit("llm-download-progress", serde_json::json!({
+            "modelId": progress.model_id.as_str(),
+            "fileName": progress.file_name,
+            "bytesDownloaded": progress.bytes_downloaded,
+            "totalBytes": progress.total_bytes,
+            "percent": progress.percent,
+        }));
+    })
+    .await
+    .map_err(|e| e.to_string())?;
+
+    Ok(())
+}
+
+/// Load a downloaded LLM model into the engine.
+#[tauri::command]
+pub async fn load_llm(
+    state: tauri::State<'_, AppState>,
+    id: String,
+) -> Result<(), String> {
+    let path = llm_model_path(&id)
+        .ok_or_else(|| format!("Unknown model: {id}"))?;
+
+    if !path.exists() {
+        return Err(format!("Model not downloaded: {id}"));
+    }
+
+    let engine = state.llm_engine.clone();
+    tokio::task::spawn_blocking(move || {
+        engine.load(&path)
+    })
+    .await
+    .map_err(|e| e.to_string())?
+    .map_err(|e| e.to_string())
+}
+
+/// Check whether an LLM model is currently loaded.
+#[tauri::command]
+pub async fn check_llm_engine(
+    state: tauri::State<'_, AppState>,
+) -> Result<bool, String> {
+    Ok(state.llm_engine.is_loaded())
+}
+
+/// Run LLM inference with a system prompt and user prompt.
+#[tauri::command]
+pub async fn llm_generate(
+    state: tauri::State<'_, AppState>,
+    system_prompt: String,
+    user_prompt: String,
+    max_tokens: Option<u32>,
+) -> Result<String, String> {
+    let engine = state.llm_engine.clone();
+    let max = max_tokens.unwrap_or(512);
+
+    kon_llm::inference::run_llm_inference(engine, system_prompt, user_prompt, max)
+        .await
+        .map_err(|e| e.to_string())
+}
+
+/// Extract tasks from transcript text using LLM (falls back to empty if no model loaded).
+#[tauri::command]
+pub async fn extract_tasks_llm(
+    state: tauri::State<'_, AppState>,
+    transcript_text: String,
+) -> Result<Vec<TaskSuggestion>, String> {
+    if !state.llm_engine.is_loaded() {
+        // No LLM available — return empty (frontend falls back to rule-based JS extractor)
+        return Ok(vec![]);
+    }
+
+    let engine = state.llm_engine.clone();
+    let system_prompt = r#"Extract actionable tasks from the following voice transcription.
+Each task must start with a concrete verb.
+Return as JSON array of {"title": "...", "priority": "high|medium|low", "project": "..."}.
+Only extract genuine tasks — not observations or comments.
+If no tasks found, return empty array []."#.to_string();
+
+    let result = kon_llm::inference::run_llm_inference(
+        engine,
+        system_prompt,
+        transcript_text,
+        256,
+    )
+    .await
+    .map_err(|e| e.to_string())?;
+
+    // Parse JSON response — handle malformed LLM output gracefully
+    parse_task_suggestions(&result)
+}
+
+/// Decompose a task into 3-7 micro-steps using LLM.
+#[tauri::command]
+pub async fn decompose_task(
+    state: tauri::State<'_, AppState>,
+    task_text: String,
+) -> Result<Vec<String>, String> {
+    if !state.llm_engine.is_loaded() {
+        return Err("AI assistant not loaded — download a model in Settings to enable micro-stepping.".to_string());
+    }
+
+    let engine = state.llm_engine.clone();
+    let system_prompt = r#"Break this task into 3-7 micro-steps.
+Each step MUST start with a specific physical verb (e.g. 'Open', 'Type', 'Click', 'Pick up').
+Each step must be completable in under 5 minutes.
+Never use abstract verbs like 'organise', 'plan', 'consider'.
+Return as JSON array of strings."#.to_string();
+
+    let result = kon_llm::inference::run_llm_inference(
+        engine,
+        system_prompt,
+        task_text,
+        256,
+    )
+    .await
+    .map_err(|e| e.to_string())?;
+
+    parse_string_array(&result)
+}
+
+// --- Response types ---
+
+#[derive(Serialize, serde::Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TaskSuggestion {
+    pub title: String,
+    pub priority: String,
+    pub project: Option<String>,
+}
+
+/// Parse LLM output as a JSON array of TaskSuggestion.
+/// Handles common LLM output quirks: markdown code fences, trailing text.
+fn parse_task_suggestions(raw: &str) -> Result<Vec<TaskSuggestion>, String> {
+    let json = extract_json_array(raw);
+    serde_json::from_str::<Vec<TaskSuggestion>>(&json)
+        .map_err(|e| format!("Failed to parse LLM task output: {e}"))
+}
+
+/// Parse LLM output as a JSON array of strings.
+fn parse_string_array(raw: &str) -> Result<Vec<String>, String> {
+    let json = extract_json_array(raw);
+    serde_json::from_str::<Vec<String>>(&json)
+        .map_err(|e| format!("Failed to parse LLM micro-step output: {e}"))
+}
+
+/// Extract a JSON array from potentially messy LLM output.
+/// Strips markdown code fences, finds the first [ ... ] pair.
+fn extract_json_array(raw: &str) -> String {
+    let cleaned = raw
+        .replace("```json", "")
+        .replace("```", "")
+        .trim()
+        .to_string();
+
+    // Find the first [ ... ] pair
+    if let Some(start) = cleaned.find('[') {
+        if let Some(end) = cleaned.rfind(']') {
+            return cleaned[start..=end].to_string();
+        }
+    }
+
+    // Fallback: return empty array
+    "[]".to_string()
+}

src-tauri/src/commands/mod.rs

@@ -2,6 +2,7 @@ pub mod audio;
 pub mod clipboard;
 pub mod hardware;
 pub mod history;
+pub mod llm;
 pub mod models;
 pub mod tasks;
 pub mod timer;

src-tauri/src/lib.rs

@@ -7,13 +7,15 @@ use sqlx::SqlitePool;
 use tauri::Manager;
 
 use kon_core::types::EngineName;
+use kon_llm::LlmEngine;
 use kon_storage::{database_path, get_setting, init as init_db, set_setting};
 use kon_transcription::LocalEngine;
 
-/// Shared app state holding the transcription engines and database pool.
+/// Shared app state holding the transcription engines, LLM engine, and database pool.
 pub struct AppState {
     pub whisper_engine: Arc<LocalEngine>,
     pub parakeet_engine: Arc<LocalEngine>,
+    pub llm_engine: Arc<LlmEngine>,
     pub db: SqlitePool,
 }
 
@@ -108,6 +110,11 @@ pub fn run() {
             // Store init script for secondary windows (float, viewer)
             app.manage(PreferencesScript(init_script));
 
+            // Initialise LLM engine (model loaded on demand via settings)
+            let llm_engine = Arc::new(
+                LlmEngine::new().expect("LLM backend should initialise")
+            );
+
             app.manage(AppState {
                 whisper_engine: Arc::new(LocalEngine::new(
                     EngineName::new("whisper"),
@@ -115,6 +122,7 @@ pub fn run() {
                 parakeet_engine: Arc::new(LocalEngine::new(
                     EngineName::new("parakeet"),
                 )),
+                llm_engine,
                 db,
             });
 
@@ -155,6 +163,14 @@ pub fn run() {
             commands::tasks::reorder_tasks_cmd,
             commands::tasks::complete_task_cmd,
             commands::tasks::uncomplete_task_cmd,
+            // LLM
+            commands::llm::list_llm_models,
+            commands::llm::download_llm,
+            commands::llm::load_llm,
+            commands::llm::check_llm_engine,
+            commands::llm::llm_generate,
+            commands::llm::extract_tasks_llm,
+            commands::llm::decompose_task,
             // Timer
             commands::timer::save_timer,
             commands::timer::get_timer,