feat(llm): add kon-llm crate with llama-cpp-2 inference, model management, and Tauri commands

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/llm/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "kon-llm"
+version = "0.1.0"
+edition = "2021"
+description = "Local LLM inference via llama.cpp for Kon"
+
+[dependencies]
+kon-core = { path = "../core" }
+llama-cpp-2 = "0.1"
+tokio = { version = "1", features = ["rt", "sync"] }
+reqwest = { version = "0.12", features = ["stream"] }
+futures-util = "0.3"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+log = "0.4"

crates/llm/src/inference.rs

@@ -0,0 +1,144 @@
+use std::path::Path;
+use std::sync::Mutex;
+
+use llama_cpp_2::context::params::LlamaContextParams;
+use llama_cpp_2::llama_backend::LlamaBackend;
+use llama_cpp_2::llama_batch::LlamaBatch;
+use llama_cpp_2::model::params::LlamaModelParams;
+use llama_cpp_2::model::{AddBos, LlamaModel, Special};
+use llama_cpp_2::sampling::LlamaSampler;
+
+use kon_core::error::{KonError, Result};
+
+/// Thread-safe LLM inference engine wrapping llama.cpp.
+pub struct LlmEngine {
+    backend: LlamaBackend,
+    model: Mutex<Option<LlamaModel>>,
+    loaded_path: Mutex<Option<String>>,
+}
+
+// Safety: LlamaBackend and LlamaModel are thread-safe for read access.
+// The Mutex guards all mutation.
+unsafe impl Send for LlmEngine {}
+unsafe impl Sync for LlmEngine {}
+
+impl LlmEngine {
+    /// Create a new engine. Call `load()` before inference.
+    pub fn new() -> Result<Self> {
+        let backend = LlamaBackend::init()
+            .map_err(|e| KonError::Other(format!("LLM backend init failed: {e}")))?;
+        Ok(Self {
+            backend,
+            model: Mutex::new(None),
+            loaded_path: Mutex::new(None),
+        })
+    }
+
+    /// Load a GGUF model from disk.
+    pub fn load(&self, model_path: &Path) -> Result<()> {
+        let params = LlamaModelParams::default();
+        let model = LlamaModel::load_from_file(&self.backend, model_path, &params)
+            .map_err(|e| KonError::Other(format!("Model load failed: {e}")))?;
+
+        *self.model.lock().unwrap() = Some(model);
+        *self.loaded_path.lock().unwrap() = Some(model_path.to_string_lossy().to_string());
+
+        log::info!("LLM model loaded: {}", model_path.display());
+        Ok(())
+    }
+
+    /// Whether a model is currently loaded.
+    pub fn is_loaded(&self) -> bool {
+        self.model.lock().unwrap().is_some()
+    }
+
+    /// Generate text from a prompt. Blocking — call from spawn_blocking.
+    ///
+    /// Uses a system prompt + user prompt pattern. The system prompt sets
+    /// the behaviour (e.g. task extraction), the user prompt is the input.
+    pub fn generate(&self, system_prompt: &str, user_prompt: &str, max_tokens: u32) -> Result<String> {
+        let guard = self.model.lock().unwrap();
+        let model = guard.as_ref()
+            .ok_or(KonError::EngineNotLoaded)?;
+
+        // Format as chat-style prompt (works with most instruction-tuned models)
+        let full_prompt = format!(
+            "<|system|>\n{system_prompt}<|end|>\n<|user|>\n{user_prompt}<|end|>\n<|assistant|>\n"
+        );
+
+        let ctx_params = LlamaContextParams::default()
+            .with_n_ctx(std::num::NonZeroU32::new(2048));
+        let mut ctx = model.new_context(&self.backend, ctx_params)
+            .map_err(|e| KonError::Other(format!("Context creation failed: {e}")))?;
+
+        // Tokenise
+        let tokens = model.str_to_token(&full_prompt, AddBos::Always)
+            .map_err(|e| KonError::Other(format!("Tokenisation failed: {e}")))?;
+
+        // Create batch and add prompt tokens
+        let mut batch = LlamaBatch::new(2048, 1);
+        for (i, token) in tokens.iter().enumerate() {
+            let is_last = i == tokens.len() - 1;
+            batch.add(*token, i as i32, &[0], is_last)
+                .map_err(|e| KonError::Other(format!("Batch add failed: {e}")))?;
+        }
+
+        // Process prompt
+        ctx.decode(&mut batch)
+            .map_err(|e| KonError::Other(format!("Prompt decode failed: {e}")))?;
+
+        // Sample tokens
+        let mut sampler = LlamaSampler::greedy();
+        let mut output = String::new();
+        let mut n_decoded = tokens.len() as i32;
+
+        for _ in 0..max_tokens {
+            let new_token = sampler.sample(&ctx, batch.n_tokens() - 1);
+            sampler.accept(new_token);
+
+            if model.is_eog_token(new_token) {
+                break;
+            }
+
+            let token_str = model.token_to_str(new_token, Special::Tokenize)
+                .map_err(|e| KonError::Other(format!("Token decode failed: {e}")))?;
+            output.push_str(&token_str);
+
+            // Stop if we see end-of-assistant markers
+            if output.contains("<|end|>") || output.contains("<|user|>") {
+                // Trim the marker
+                if let Some(pos) = output.find("<|end|>") {
+                    output.truncate(pos);
+                }
+                if let Some(pos) = output.find("<|user|>") {
+                    output.truncate(pos);
+                }
+                break;
+            }
+
+            batch.clear();
+            batch.add(new_token, n_decoded, &[0], true)
+                .map_err(|e| KonError::Other(format!("Batch add failed: {e}")))?;
+            n_decoded += 1;
+
+            ctx.decode(&mut batch)
+                .map_err(|e| KonError::Other(format!("Token decode failed: {e}")))?;
+        }
+
+        Ok(output.trim().to_string())
+    }
+}
+
+/// Run LLM inference on a blocking thread.
+pub async fn run_llm_inference(
+    engine: std::sync::Arc<LlmEngine>,
+    system_prompt: String,
+    user_prompt: String,
+    max_tokens: u32,
+) -> Result<String> {
+    tokio::task::spawn_blocking(move || {
+        engine.generate(&system_prompt, &user_prompt, max_tokens)
+    })
+    .await
+    .map_err(|e| KonError::Other(format!("LLM inference thread failed: {e}")))?
+}

crates/llm/src/lib.rs

@@ -0,0 +1,5 @@
+pub mod inference;
+pub mod model_manager;
+
+pub use inference::LlmEngine;
+pub use model_manager::{LlmModelEntry, LLM_MODELS, llm_models_dir, is_llm_downloaded, download_llm_model};

crates/llm/src/model_manager.rs

@@ -0,0 +1,130 @@
+use std::path::PathBuf;
+
+use kon_core::error::{KonError, Result};
+use kon_core::types::{DownloadProgress, Megabytes, ModelId};
+
+/// Metadata for an LLM model in the catalogue.
+#[derive(Debug, Clone)]
+pub struct LlmModelEntry {
+    pub id: &'static str,
+    pub display_name: &'static str,
+    pub url: &'static str,
+    pub disk_size: Megabytes,
+    pub ram_required: Megabytes,
+    pub filename: &'static str,
+    pub description: &'static str,
+}
+
+/// LLM model catalogue — hardware-tiered options.
+pub const LLM_MODELS: &[LlmModelEntry] = &[
+    LlmModelEntry {
+        id: "phi-4-mini-q4",
+        display_name: "Phi-4 Mini (8GB RAM)",
+        url: "https://huggingface.co/bartowski/phi-4-mini-instruct-GGUF/resolve/main/phi-4-mini-instruct-Q4_K_M.gguf",
+        disk_size: Megabytes(2400),
+        ram_required: Megabytes(4000),
+        filename: "phi-4-mini-instruct-Q4_K_M.gguf",
+        description: "Compact and fast — ideal for 8GB systems",
+    },
+    LlmModelEntry {
+        id: "qwen3-7b-q4",
+        display_name: "Qwen 3 7B (16GB RAM)",
+        url: "https://huggingface.co/bartowski/Qwen3-8B-GGUF/resolve/main/Qwen3-8B-Q4_K_M.gguf",
+        disk_size: Megabytes(4900),
+        ram_required: Megabytes(8000),
+        filename: "Qwen3-8B-Q4_K_M.gguf",
+        description: "Higher quality — recommended for 16GB+ systems",
+    },
+];
+
+/// Directory for LLM GGUF models.
+pub fn llm_models_dir() -> PathBuf {
+    if cfg!(target_os = "windows") {
+        let local = std::env::var("LOCALAPPDATA").unwrap_or_else(|_| ".".to_string());
+        PathBuf::from(local).join("kon").join("llm-models")
+    } else {
+        let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
+        PathBuf::from(home).join(".kon").join("llm-models")
+    }
+}
+
+/// Check whether a model's GGUF file exists on disk.
+pub fn is_llm_downloaded(model_id: &str) -> bool {
+    if let Some(entry) = LLM_MODELS.iter().find(|m| m.id == model_id) {
+        llm_models_dir().join(entry.filename).exists()
+    } else {
+        false
+    }
+}
+
+/// Get the file path for a downloaded model.
+pub fn llm_model_path(model_id: &str) -> Option<PathBuf> {
+    LLM_MODELS
+        .iter()
+        .find(|m| m.id == model_id)
+        .map(|entry| llm_models_dir().join(entry.filename))
+}
+
+/// Download a GGUF model with progress callback.
+pub async fn download_llm_model(
+    model_id: &str,
+    on_progress: impl Fn(DownloadProgress) + Send + 'static,
+) -> Result<PathBuf> {
+    let entry = LLM_MODELS
+        .iter()
+        .find(|m| m.id == model_id)
+        .ok_or_else(|| KonError::ModelNotFound(ModelId::new(model_id)))?;
+
+    let dir = llm_models_dir();
+    std::fs::create_dir_all(&dir)?;
+
+    let dest = dir.join(entry.filename);
+    let part = dir.join(format!("{}.part", entry.filename));
+
+    // Stream download with progress
+    let response = reqwest::get(entry.url)
+        .await
+        .map_err(|e| KonError::DownloadFailed(format!("Request failed: {e}")))?;
+
+    let total = response.content_length().unwrap_or(0);
+
+    let mut stream = response.bytes_stream();
+    let mut file = tokio::fs::File::create(&part)
+        .await
+        .map_err(|e| KonError::DownloadFailed(format!("File create failed: {e}")))?;
+
+    let mut downloaded: u64 = 0;
+    let model_id_owned = ModelId::new(model_id);
+
+    use futures_util::StreamExt;
+    use tokio::io::AsyncWriteExt;
+
+    while let Some(chunk) = stream.next().await {
+        let chunk = chunk.map_err(|e| KonError::DownloadFailed(format!("Download chunk failed: {e}")))?;
+        file.write_all(&chunk)
+            .await
+            .map_err(|e| KonError::DownloadFailed(format!("Write failed: {e}")))?;
+
+        downloaded += chunk.len() as u64;
+        let percent = if total > 0 { (downloaded as f64 / total as f64 * 100.0) as u8 } else { 0 };
+
+        on_progress(DownloadProgress {
+            model_id: model_id_owned.clone(),
+            file_name: entry.filename.to_string(),
+            bytes_downloaded: downloaded,
+            total_bytes: total,
+            percent,
+        });
+    }
+
+    file.flush().await
+        .map_err(|e| KonError::DownloadFailed(format!("Flush failed: {e}")))?;
+    drop(file);
+
+    // Atomic rename
+    std::fs::rename(&part, &dest)
+        .map_err(|e| KonError::DownloadFailed(format!("Rename failed: {e}")))?;
+
+    log::info!("LLM model downloaded: {} → {}", model_id, dest.display());
+    Ok(dest)
+}