diff --git a/crates/llm/Cargo.toml b/crates/llm/Cargo.toml new file mode 100644 index 0000000..15c9458 --- /dev/null +++ b/crates/llm/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "kon-llm" +version = "0.1.0" +edition = "2021" +description = "Local LLM inference via llama.cpp for Kon" + +[dependencies] +kon-core = { path = "../core" } +llama-cpp-2 = "0.1" +tokio = { version = "1", features = ["rt", "sync"] } +reqwest = { version = "0.12", features = ["stream"] } +futures-util = "0.3" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +log = "0.4" diff --git a/crates/llm/src/inference.rs b/crates/llm/src/inference.rs new file mode 100644 index 0000000..91aee40 --- /dev/null +++ b/crates/llm/src/inference.rs @@ -0,0 +1,144 @@ +use std::path::Path; +use std::sync::Mutex; + +use llama_cpp_2::context::params::LlamaContextParams; +use llama_cpp_2::llama_backend::LlamaBackend; +use llama_cpp_2::llama_batch::LlamaBatch; +use llama_cpp_2::model::params::LlamaModelParams; +use llama_cpp_2::model::{AddBos, LlamaModel, Special}; +use llama_cpp_2::sampling::LlamaSampler; + +use kon_core::error::{KonError, Result}; + +/// Thread-safe LLM inference engine wrapping llama.cpp. +pub struct LlmEngine { + backend: LlamaBackend, + model: Mutex>, + loaded_path: Mutex>, +} + +// Safety: LlamaBackend and LlamaModel are thread-safe for read access. +// The Mutex guards all mutation. +unsafe impl Send for LlmEngine {} +unsafe impl Sync for LlmEngine {} + +impl LlmEngine { + /// Create a new engine. Call `load()` before inference. + pub fn new() -> Result { + let backend = LlamaBackend::init() + .map_err(|e| KonError::Other(format!("LLM backend init failed: {e}")))?; + Ok(Self { + backend, + model: Mutex::new(None), + loaded_path: Mutex::new(None), + }) + } + + /// Load a GGUF model from disk. + pub fn load(&self, model_path: &Path) -> Result<()> { + let params = LlamaModelParams::default(); + let model = LlamaModel::load_from_file(&self.backend, model_path, ¶ms) + .map_err(|e| KonError::Other(format!("Model load failed: {e}")))?; + + *self.model.lock().unwrap() = Some(model); + *self.loaded_path.lock().unwrap() = Some(model_path.to_string_lossy().to_string()); + + log::info!("LLM model loaded: {}", model_path.display()); + Ok(()) + } + + /// Whether a model is currently loaded. + pub fn is_loaded(&self) -> bool { + self.model.lock().unwrap().is_some() + } + + /// Generate text from a prompt. Blocking — call from spawn_blocking. + /// + /// Uses a system prompt + user prompt pattern. The system prompt sets + /// the behaviour (e.g. task extraction), the user prompt is the input. + pub fn generate(&self, system_prompt: &str, user_prompt: &str, max_tokens: u32) -> Result { + let guard = self.model.lock().unwrap(); + let model = guard.as_ref() + .ok_or(KonError::EngineNotLoaded)?; + + // Format as chat-style prompt (works with most instruction-tuned models) + let full_prompt = format!( + "<|system|>\n{system_prompt}<|end|>\n<|user|>\n{user_prompt}<|end|>\n<|assistant|>\n" + ); + + let ctx_params = LlamaContextParams::default() + .with_n_ctx(std::num::NonZeroU32::new(2048)); + let mut ctx = model.new_context(&self.backend, ctx_params) + .map_err(|e| KonError::Other(format!("Context creation failed: {e}")))?; + + // Tokenise + let tokens = model.str_to_token(&full_prompt, AddBos::Always) + .map_err(|e| KonError::Other(format!("Tokenisation failed: {e}")))?; + + // Create batch and add prompt tokens + let mut batch = LlamaBatch::new(2048, 1); + for (i, token) in tokens.iter().enumerate() { + let is_last = i == tokens.len() - 1; + batch.add(*token, i as i32, &[0], is_last) + .map_err(|e| KonError::Other(format!("Batch add failed: {e}")))?; + } + + // Process prompt + ctx.decode(&mut batch) + .map_err(|e| KonError::Other(format!("Prompt decode failed: {e}")))?; + + // Sample tokens + let mut sampler = LlamaSampler::greedy(); + let mut output = String::new(); + let mut n_decoded = tokens.len() as i32; + + for _ in 0..max_tokens { + let new_token = sampler.sample(&ctx, batch.n_tokens() - 1); + sampler.accept(new_token); + + if model.is_eog_token(new_token) { + break; + } + + let token_str = model.token_to_str(new_token, Special::Tokenize) + .map_err(|e| KonError::Other(format!("Token decode failed: {e}")))?; + output.push_str(&token_str); + + // Stop if we see end-of-assistant markers + if output.contains("<|end|>") || output.contains("<|user|>") { + // Trim the marker + if let Some(pos) = output.find("<|end|>") { + output.truncate(pos); + } + if let Some(pos) = output.find("<|user|>") { + output.truncate(pos); + } + break; + } + + batch.clear(); + batch.add(new_token, n_decoded, &[0], true) + .map_err(|e| KonError::Other(format!("Batch add failed: {e}")))?; + n_decoded += 1; + + ctx.decode(&mut batch) + .map_err(|e| KonError::Other(format!("Token decode failed: {e}")))?; + } + + Ok(output.trim().to_string()) + } +} + +/// Run LLM inference on a blocking thread. +pub async fn run_llm_inference( + engine: std::sync::Arc, + system_prompt: String, + user_prompt: String, + max_tokens: u32, +) -> Result { + tokio::task::spawn_blocking(move || { + engine.generate(&system_prompt, &user_prompt, max_tokens) + }) + .await + .map_err(|e| KonError::Other(format!("LLM inference thread failed: {e}")))? +} diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs new file mode 100644 index 0000000..3fa5d4c --- /dev/null +++ b/crates/llm/src/lib.rs @@ -0,0 +1,5 @@ +pub mod inference; +pub mod model_manager; + +pub use inference::LlmEngine; +pub use model_manager::{LlmModelEntry, LLM_MODELS, llm_models_dir, is_llm_downloaded, download_llm_model}; diff --git a/crates/llm/src/model_manager.rs b/crates/llm/src/model_manager.rs new file mode 100644 index 0000000..df66f32 --- /dev/null +++ b/crates/llm/src/model_manager.rs @@ -0,0 +1,130 @@ +use std::path::PathBuf; + +use kon_core::error::{KonError, Result}; +use kon_core::types::{DownloadProgress, Megabytes, ModelId}; + +/// Metadata for an LLM model in the catalogue. +#[derive(Debug, Clone)] +pub struct LlmModelEntry { + pub id: &'static str, + pub display_name: &'static str, + pub url: &'static str, + pub disk_size: Megabytes, + pub ram_required: Megabytes, + pub filename: &'static str, + pub description: &'static str, +} + +/// LLM model catalogue — hardware-tiered options. +pub const LLM_MODELS: &[LlmModelEntry] = &[ + LlmModelEntry { + id: "phi-4-mini-q4", + display_name: "Phi-4 Mini (8GB RAM)", + url: "https://huggingface.co/bartowski/phi-4-mini-instruct-GGUF/resolve/main/phi-4-mini-instruct-Q4_K_M.gguf", + disk_size: Megabytes(2400), + ram_required: Megabytes(4000), + filename: "phi-4-mini-instruct-Q4_K_M.gguf", + description: "Compact and fast — ideal for 8GB systems", + }, + LlmModelEntry { + id: "qwen3-7b-q4", + display_name: "Qwen 3 7B (16GB RAM)", + url: "https://huggingface.co/bartowski/Qwen3-8B-GGUF/resolve/main/Qwen3-8B-Q4_K_M.gguf", + disk_size: Megabytes(4900), + ram_required: Megabytes(8000), + filename: "Qwen3-8B-Q4_K_M.gguf", + description: "Higher quality — recommended for 16GB+ systems", + }, +]; + +/// Directory for LLM GGUF models. +pub fn llm_models_dir() -> PathBuf { + if cfg!(target_os = "windows") { + let local = std::env::var("LOCALAPPDATA").unwrap_or_else(|_| ".".to_string()); + PathBuf::from(local).join("kon").join("llm-models") + } else { + let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()); + PathBuf::from(home).join(".kon").join("llm-models") + } +} + +/// Check whether a model's GGUF file exists on disk. +pub fn is_llm_downloaded(model_id: &str) -> bool { + if let Some(entry) = LLM_MODELS.iter().find(|m| m.id == model_id) { + llm_models_dir().join(entry.filename).exists() + } else { + false + } +} + +/// Get the file path for a downloaded model. +pub fn llm_model_path(model_id: &str) -> Option { + LLM_MODELS + .iter() + .find(|m| m.id == model_id) + .map(|entry| llm_models_dir().join(entry.filename)) +} + +/// Download a GGUF model with progress callback. +pub async fn download_llm_model( + model_id: &str, + on_progress: impl Fn(DownloadProgress) + Send + 'static, +) -> Result { + let entry = LLM_MODELS + .iter() + .find(|m| m.id == model_id) + .ok_or_else(|| KonError::ModelNotFound(ModelId::new(model_id)))?; + + let dir = llm_models_dir(); + std::fs::create_dir_all(&dir)?; + + let dest = dir.join(entry.filename); + let part = dir.join(format!("{}.part", entry.filename)); + + // Stream download with progress + let response = reqwest::get(entry.url) + .await + .map_err(|e| KonError::DownloadFailed(format!("Request failed: {e}")))?; + + let total = response.content_length().unwrap_or(0); + + let mut stream = response.bytes_stream(); + let mut file = tokio::fs::File::create(&part) + .await + .map_err(|e| KonError::DownloadFailed(format!("File create failed: {e}")))?; + + let mut downloaded: u64 = 0; + let model_id_owned = ModelId::new(model_id); + + use futures_util::StreamExt; + use tokio::io::AsyncWriteExt; + + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| KonError::DownloadFailed(format!("Download chunk failed: {e}")))?; + file.write_all(&chunk) + .await + .map_err(|e| KonError::DownloadFailed(format!("Write failed: {e}")))?; + + downloaded += chunk.len() as u64; + let percent = if total > 0 { (downloaded as f64 / total as f64 * 100.0) as u8 } else { 0 }; + + on_progress(DownloadProgress { + model_id: model_id_owned.clone(), + file_name: entry.filename.to_string(), + bytes_downloaded: downloaded, + total_bytes: total, + percent, + }); + } + + file.flush().await + .map_err(|e| KonError::DownloadFailed(format!("Flush failed: {e}")))?; + drop(file); + + // Atomic rename + std::fs::rename(&part, &dest) + .map_err(|e| KonError::DownloadFailed(format!("Rename failed: {e}")))?; + + log::info!("LLM model downloaded: {} → {}", model_id, dest.display()); + Ok(dest) +} diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 55a49e6..3bcf18b 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -20,6 +20,7 @@ kon-transcription = { path = "../crates/transcription" } kon-ai-formatting = { path = "../crates/ai-formatting" } kon-storage = { path = "../crates/storage" } kon-cloud-providers = { path = "../crates/cloud-providers" } +kon-llm = { path = "../crates/llm" } # Tauri tauri = { version = "2", features = ["tray-icon"] } diff --git a/src-tauri/src/commands/llm.rs b/src-tauri/src/commands/llm.rs new file mode 100644 index 0000000..d730a0e --- /dev/null +++ b/src-tauri/src/commands/llm.rs @@ -0,0 +1,213 @@ +use std::sync::Arc; + +use serde::Serialize; +use tauri::Emitter; + +use crate::AppState; +use kon_llm::{ + LlmModelEntry, LLM_MODELS, + is_llm_downloaded, download_llm_model, + model_manager::llm_model_path, +}; + +/// Serialisable LLM model info for the frontend. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LlmModelInfo { + pub id: String, + pub display_name: String, + pub disk_size_mb: u64, + pub ram_required_mb: u64, + pub description: String, + pub is_downloaded: bool, +} + +fn model_info(entry: &LlmModelEntry) -> LlmModelInfo { + LlmModelInfo { + id: entry.id.to_string(), + display_name: entry.display_name.to_string(), + disk_size_mb: entry.disk_size.0, + ram_required_mb: entry.ram_required.0, + description: entry.description.to_string(), + is_downloaded: is_llm_downloaded(entry.id), + } +} + +/// List available LLM models with download status. +#[tauri::command] +pub async fn list_llm_models() -> Vec { + LLM_MODELS.iter().map(model_info).collect() +} + +/// Download an LLM model. Emits "llm-download-progress" events. +#[tauri::command] +pub async fn download_llm( + app: tauri::AppHandle, + id: String, +) -> Result<(), String> { + let app_handle = app.clone(); + download_llm_model(&id, move |progress| { + let _ = app_handle.emit("llm-download-progress", serde_json::json!({ + "modelId": progress.model_id.as_str(), + "fileName": progress.file_name, + "bytesDownloaded": progress.bytes_downloaded, + "totalBytes": progress.total_bytes, + "percent": progress.percent, + })); + }) + .await + .map_err(|e| e.to_string())?; + + Ok(()) +} + +/// Load a downloaded LLM model into the engine. +#[tauri::command] +pub async fn load_llm( + state: tauri::State<'_, AppState>, + id: String, +) -> Result<(), String> { + let path = llm_model_path(&id) + .ok_or_else(|| format!("Unknown model: {id}"))?; + + if !path.exists() { + return Err(format!("Model not downloaded: {id}")); + } + + let engine = state.llm_engine.clone(); + tokio::task::spawn_blocking(move || { + engine.load(&path) + }) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string()) +} + +/// Check whether an LLM model is currently loaded. +#[tauri::command] +pub async fn check_llm_engine( + state: tauri::State<'_, AppState>, +) -> Result { + Ok(state.llm_engine.is_loaded()) +} + +/// Run LLM inference with a system prompt and user prompt. +#[tauri::command] +pub async fn llm_generate( + state: tauri::State<'_, AppState>, + system_prompt: String, + user_prompt: String, + max_tokens: Option, +) -> Result { + let engine = state.llm_engine.clone(); + let max = max_tokens.unwrap_or(512); + + kon_llm::inference::run_llm_inference(engine, system_prompt, user_prompt, max) + .await + .map_err(|e| e.to_string()) +} + +/// Extract tasks from transcript text using LLM (falls back to empty if no model loaded). +#[tauri::command] +pub async fn extract_tasks_llm( + state: tauri::State<'_, AppState>, + transcript_text: String, +) -> Result, String> { + if !state.llm_engine.is_loaded() { + // No LLM available — return empty (frontend falls back to rule-based JS extractor) + return Ok(vec![]); + } + + let engine = state.llm_engine.clone(); + let system_prompt = r#"Extract actionable tasks from the following voice transcription. +Each task must start with a concrete verb. +Return as JSON array of {"title": "...", "priority": "high|medium|low", "project": "..."}. +Only extract genuine tasks — not observations or comments. +If no tasks found, return empty array []."#.to_string(); + + let result = kon_llm::inference::run_llm_inference( + engine, + system_prompt, + transcript_text, + 256, + ) + .await + .map_err(|e| e.to_string())?; + + // Parse JSON response — handle malformed LLM output gracefully + parse_task_suggestions(&result) +} + +/// Decompose a task into 3-7 micro-steps using LLM. +#[tauri::command] +pub async fn decompose_task( + state: tauri::State<'_, AppState>, + task_text: String, +) -> Result, String> { + if !state.llm_engine.is_loaded() { + return Err("AI assistant not loaded — download a model in Settings to enable micro-stepping.".to_string()); + } + + let engine = state.llm_engine.clone(); + let system_prompt = r#"Break this task into 3-7 micro-steps. +Each step MUST start with a specific physical verb (e.g. 'Open', 'Type', 'Click', 'Pick up'). +Each step must be completable in under 5 minutes. +Never use abstract verbs like 'organise', 'plan', 'consider'. +Return as JSON array of strings."#.to_string(); + + let result = kon_llm::inference::run_llm_inference( + engine, + system_prompt, + task_text, + 256, + ) + .await + .map_err(|e| e.to_string())?; + + parse_string_array(&result) +} + +// --- Response types --- + +#[derive(Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TaskSuggestion { + pub title: String, + pub priority: String, + pub project: Option, +} + +/// Parse LLM output as a JSON array of TaskSuggestion. +/// Handles common LLM output quirks: markdown code fences, trailing text. +fn parse_task_suggestions(raw: &str) -> Result, String> { + let json = extract_json_array(raw); + serde_json::from_str::>(&json) + .map_err(|e| format!("Failed to parse LLM task output: {e}")) +} + +/// Parse LLM output as a JSON array of strings. +fn parse_string_array(raw: &str) -> Result, String> { + let json = extract_json_array(raw); + serde_json::from_str::>(&json) + .map_err(|e| format!("Failed to parse LLM micro-step output: {e}")) +} + +/// Extract a JSON array from potentially messy LLM output. +/// Strips markdown code fences, finds the first [ ... ] pair. +fn extract_json_array(raw: &str) -> String { + let cleaned = raw + .replace("```json", "") + .replace("```", "") + .trim() + .to_string(); + + // Find the first [ ... ] pair + if let Some(start) = cleaned.find('[') { + if let Some(end) = cleaned.rfind(']') { + return cleaned[start..=end].to_string(); + } + } + + // Fallback: return empty array + "[]".to_string() +} diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 1761316..a2fd34b 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -2,6 +2,7 @@ pub mod audio; pub mod clipboard; pub mod hardware; pub mod history; +pub mod llm; pub mod models; pub mod tasks; pub mod timer; diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index ff4152d..30e9728 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -7,13 +7,15 @@ use sqlx::SqlitePool; use tauri::Manager; use kon_core::types::EngineName; +use kon_llm::LlmEngine; use kon_storage::{database_path, get_setting, init as init_db, set_setting}; use kon_transcription::LocalEngine; -/// Shared app state holding the transcription engines and database pool. +/// Shared app state holding the transcription engines, LLM engine, and database pool. pub struct AppState { pub whisper_engine: Arc, pub parakeet_engine: Arc, + pub llm_engine: Arc, pub db: SqlitePool, } @@ -108,6 +110,11 @@ pub fn run() { // Store init script for secondary windows (float, viewer) app.manage(PreferencesScript(init_script)); + // Initialise LLM engine (model loaded on demand via settings) + let llm_engine = Arc::new( + LlmEngine::new().expect("LLM backend should initialise") + ); + app.manage(AppState { whisper_engine: Arc::new(LocalEngine::new( EngineName::new("whisper"), @@ -115,6 +122,7 @@ pub fn run() { parakeet_engine: Arc::new(LocalEngine::new( EngineName::new("parakeet"), )), + llm_engine, db, }); @@ -155,6 +163,14 @@ pub fn run() { commands::tasks::reorder_tasks_cmd, commands::tasks::complete_task_cmd, commands::tasks::uncomplete_task_cmd, + // LLM + commands::llm::list_llm_models, + commands::llm::download_llm, + commands::llm::load_llm, + commands::llm::check_llm_engine, + commands::llm::llm_generate, + commands::llm::extract_tasks_llm, + commands::llm::decompose_task, // Timer commands::timer::save_timer, commands::timer::get_timer,