From 699cb7e08e21f5c50c42c63204613e3b1f435952 Mon Sep 17 00:00:00 2001 From: Jake Date: Fri, 1 May 2026 09:57:21 +0100 Subject: [PATCH] feat(llm): bump model registry to Qwen3.5 + Qwen3.6 family (4 tiers) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the three older Qwen3 variants with a four-tier ladder spanning a wider hardware range: - Qwen3_5_2B_Q4 (Minimal, 8 GB RAM, ~1.3 GB download) - Qwen3_5_4B_Q4 (Standard, 16 GB RAM / 6 GB VRAM, ~2.7 GB) — DEFAULT - Qwen3_5_9B_Q4 (High, 32 GB RAM / 12 GB VRAM, ~5.7 GB) - Qwen3_6_27B_Q4 (Maximum, 64 GB RAM / 24 GB VRAM, ~17 GB) All four GGUFs sourced from unsloth's HF org with pinned commit SHAs. Sizes and SHA256 hashes verified against the live X-Linked-Etag / X-Linked-Size headers on the LFS CDN. Q4_K_M quantisation throughout (common sweet-spot for cleanup + task extraction). recommend_tier rewritten to span four bands; default_tier moves from the old 4B-Instruct-2507 to Qwen3.5 4B. The 27B Maximum tier honestly needs 64 GB RAM to run without partial offload — surfaced in the description string so the Settings UI can warn realistically. In-tree smoke tests (smoke.rs, content_tags_smoke.rs) updated to reference the new smallest tier so a developer's MAGNOTIA_LLM_TEST_MODEL points at the cheapest GGUF to download. Crate description in crates/llm/Cargo.toml refreshed to mention the new family. NOTE (out of scope; not fixed): the size_bytes / sha256 / hf_url methods could collapse into a single LlmModelMetadata table to remove four parallel match arms. Layer 2 cleanup, separate session. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/llm/Cargo.toml | 2 +- crates/llm/src/model_manager.rs | 136 ++++++++++++++----------- crates/llm/tests/content_tags_smoke.rs | 2 +- crates/llm/tests/smoke.rs | 2 +- 4 files changed, 81 insertions(+), 61 deletions(-) diff --git a/crates/llm/Cargo.toml b/crates/llm/Cargo.toml index 3ca38e9..d33eeea 100644 --- a/crates/llm/Cargo.toml +++ b/crates/llm/Cargo.toml @@ -2,7 +2,7 @@ name = "magnotia-llm" version = "0.1.0" edition = "2021" -description = "Local LLM engine for Magnotia (Qwen3 via llama-cpp-2): transcript cleanup, task extraction, micro-step decomposition" +description = "Local LLM engine for Magnotia (Qwen3.5 / Qwen3.6 via llama-cpp-2): transcript cleanup, task extraction, micro-step decomposition" [features] # Default desktop build keeps the existing openmp + vulkan acceleration. diff --git a/crates/llm/src/model_manager.rs b/crates/llm/src/model_manager.rs index 1f8566d..b250d2c 100644 --- a/crates/llm/src/model_manager.rs +++ b/crates/llm/src/model_manager.rs @@ -12,100 +12,119 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; #[allow(non_camel_case_types)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum LlmModelId { - #[serde(rename = "qwen3_1_7b")] - Qwen3_1_7B_Q4, - #[serde(rename = "qwen3_4b_instruct_2507")] - Qwen3_4BInstruct2507Q4, - #[serde(rename = "qwen3_14b")] - Qwen3_14BQ5, + #[serde(rename = "qwen3_5_2b")] + Qwen3_5_2B_Q4, + #[serde(rename = "qwen3_5_4b")] + Qwen3_5_4B_Q4, + #[serde(rename = "qwen3_5_9b")] + Qwen3_5_9B_Q4, + #[serde(rename = "qwen3_6_27b")] + Qwen3_6_27B_Q4, } impl LlmModelId { pub fn default_tier() -> Self { - Self::Qwen3_4BInstruct2507Q4 + Self::Qwen3_5_4B_Q4 } pub fn as_str(&self) -> &'static str { match self { - Self::Qwen3_1_7B_Q4 => "qwen3_1_7b", - Self::Qwen3_4BInstruct2507Q4 => "qwen3_4b_instruct_2507", - Self::Qwen3_14BQ5 => "qwen3_14b", + Self::Qwen3_5_2B_Q4 => "qwen3_5_2b", + Self::Qwen3_5_4B_Q4 => "qwen3_5_4b", + Self::Qwen3_5_9B_Q4 => "qwen3_5_9b", + Self::Qwen3_6_27B_Q4 => "qwen3_6_27b", } } pub fn display_name(&self) -> &'static str { match self { - Self::Qwen3_1_7B_Q4 => "Qwen3 1.7B", - Self::Qwen3_4BInstruct2507Q4 => "Qwen3 4B Instruct 2507", - Self::Qwen3_14BQ5 => "Qwen3 14B", + Self::Qwen3_5_2B_Q4 => "Qwen3.5 2B", + Self::Qwen3_5_4B_Q4 => "Qwen3.5 4B", + Self::Qwen3_5_9B_Q4 => "Qwen3.5 9B", + Self::Qwen3_6_27B_Q4 => "Qwen3.6 27B", } } pub fn file_name(&self) -> &'static str { match self { - Self::Qwen3_1_7B_Q4 => "Qwen3-1.7B-Q4_K_M.gguf", - Self::Qwen3_4BInstruct2507Q4 => "Qwen3-4B-Instruct-2507-Q4_K_M.gguf", - Self::Qwen3_14BQ5 => "Qwen3-14B-Q5_K_M.gguf", + Self::Qwen3_5_2B_Q4 => "Qwen3.5-2B-Q4_K_M.gguf", + Self::Qwen3_5_4B_Q4 => "Qwen3.5-4B-Q4_K_M.gguf", + Self::Qwen3_5_9B_Q4 => "Qwen3.5-9B-Q4_K_M.gguf", + Self::Qwen3_6_27B_Q4 => "Qwen3.6-27B-Q4_K_M.gguf", } } pub fn size_bytes(&self) -> u64 { match self { - Self::Qwen3_1_7B_Q4 => 1_107_409_472, - Self::Qwen3_4BInstruct2507Q4 => 2_497_281_120, - Self::Qwen3_14BQ5 => 10_514_570_624, + Self::Qwen3_5_2B_Q4 => 1_280_835_840, + Self::Qwen3_5_4B_Q4 => 2_740_937_888, + Self::Qwen3_5_9B_Q4 => 5_680_522_464, + Self::Qwen3_6_27B_Q4 => 16_817_244_384, } } pub fn minimum_ram_bytes(&self) -> u64 { match self { - Self::Qwen3_1_7B_Q4 => 8 * 1024_u64.pow(3), - Self::Qwen3_4BInstruct2507Q4 => 16 * 1024_u64.pow(3), - Self::Qwen3_14BQ5 => 32 * 1024_u64.pow(3), + Self::Qwen3_5_2B_Q4 => 8 * 1024_u64.pow(3), + Self::Qwen3_5_4B_Q4 => 16 * 1024_u64.pow(3), + Self::Qwen3_5_9B_Q4 => 32 * 1024_u64.pow(3), + Self::Qwen3_6_27B_Q4 => 64 * 1024_u64.pow(3), } } pub fn recommended_vram_bytes(&self) -> Option { match self { - Self::Qwen3_1_7B_Q4 => None, - Self::Qwen3_4BInstruct2507Q4 => Some(8 * 1024_u64.pow(3)), - Self::Qwen3_14BQ5 => Some(16 * 1024_u64.pow(3)), + Self::Qwen3_5_2B_Q4 => None, + Self::Qwen3_5_4B_Q4 => Some(6 * 1024_u64.pow(3)), + Self::Qwen3_5_9B_Q4 => Some(12 * 1024_u64.pow(3)), + Self::Qwen3_6_27B_Q4 => Some(24 * 1024_u64.pow(3)), } } pub fn description(&self) -> &'static str { match self { - Self::Qwen3_1_7B_Q4 => "Low tier for 8 GB RAM and CPU-heavy machines.", - Self::Qwen3_4BInstruct2507Q4 => { - "Default tier for cleanup and task extraction on 16 GB systems." + Self::Qwen3_5_2B_Q4 => "Minimal tier for 8 GB RAM and CPU-heavy machines.", + Self::Qwen3_5_4B_Q4 => { + "Standard tier for cleanup and task extraction on 16 GB systems." + } + Self::Qwen3_5_9B_Q4 => "High tier for 32 GB RAM with a 12 GB+ GPU.", + Self::Qwen3_6_27B_Q4 => { + "Maximum tier for 64 GB RAM with a 24 GB GPU; partial CPU offload below that." } - Self::Qwen3_14BQ5 => "High tier for 32 GB+ RAM and larger GPUs.", } } pub fn hf_url(&self) -> &'static str { match self { - Self::Qwen3_1_7B_Q4 => { - "https://huggingface.co/unsloth/Qwen3-1.7B-GGUF/resolve/d7f544eead698dbd1f15126ef60b45a1e1933222/Qwen3-1.7B-Q4_K_M.gguf" + Self::Qwen3_5_2B_Q4 => { + "https://huggingface.co/unsloth/Qwen3.5-2B-GGUF/resolve/f6d5376be1edb4d416d56da11e5397a961aca8ae/Qwen3.5-2B-Q4_K_M.gguf" } - Self::Qwen3_4BInstruct2507Q4 => { - "https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/a06e946bb6b655725eafa393f4a9745d460374c9/Qwen3-4B-Instruct-2507-Q4_K_M.gguf" + Self::Qwen3_5_4B_Q4 => { + "https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/e87f176479d0855a907a41277aca2f8ee7a09523/Qwen3.5-4B-Q4_K_M.gguf" } - Self::Qwen3_14BQ5 => { - "https://huggingface.co/unsloth/Qwen3-14B-GGUF/resolve/a04a82c4739b3ef5fa6da7d10261db2c67dd1985/Qwen3-14B-Q5_K_M.gguf" + Self::Qwen3_5_9B_Q4 => { + "https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/3885219b6810b007914f3a7950a8d1b469d598a5/Qwen3.5-9B-Q4_K_M.gguf" + } + Self::Qwen3_6_27B_Q4 => { + "https://huggingface.co/unsloth/Qwen3.6-27B-GGUF/resolve/82d411acf4a06cfb8d9b073a5211bf410bfc29bf/Qwen3.6-27B-Q4_K_M.gguf" } } } pub fn sha256(&self) -> &'static str { match self { - Self::Qwen3_1_7B_Q4 => { - "de942b0819216caa3bfe487180dd1bb37398fa1c98cb42bb0bbac7ab7d6e8a12" + Self::Qwen3_5_2B_Q4 => { + "aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223" } - Self::Qwen3_4BInstruct2507Q4 => { - "bf52d44a54b81d44219833556849529ee96f09da673a38783dddc2e2eaf17881" + Self::Qwen3_5_4B_Q4 => { + "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4" + } + Self::Qwen3_5_9B_Q4 => { + "03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8" + } + Self::Qwen3_6_27B_Q4 => { + "5ed60d0af4650a854b1755bd392f9aef4872643dc25a254bc68043fa638392a0" } - Self::Qwen3_14BQ5 => "6f87abc471bd509ad46aca4284b3cfa926d8114bc491bb0a7a3a7f74c16ef95b", } } } @@ -121,9 +140,10 @@ impl FromStr for LlmModelId { fn from_str(value: &str) -> Result { match value { - "qwen3_1_7b" => Ok(Self::Qwen3_1_7B_Q4), - "qwen3_4b_instruct_2507" => Ok(Self::Qwen3_4BInstruct2507Q4), - "qwen3_14b" => Ok(Self::Qwen3_14BQ5), + "qwen3_5_2b" => Ok(Self::Qwen3_5_2B_Q4), + "qwen3_5_4b" => Ok(Self::Qwen3_5_4B_Q4), + "qwen3_5_9b" => Ok(Self::Qwen3_5_9B_Q4), + "qwen3_6_27b" => Ok(Self::Qwen3_6_27B_Q4), other => Err(format!("Unknown LLM model id: {other}")), } } @@ -154,9 +174,10 @@ pub enum DownloadError { } const ALL_MODELS: &[LlmModelId] = &[ - LlmModelId::Qwen3_1_7B_Q4, - LlmModelId::Qwen3_4BInstruct2507Q4, - LlmModelId::Qwen3_14BQ5, + LlmModelId::Qwen3_5_2B_Q4, + LlmModelId::Qwen3_5_4B_Q4, + LlmModelId::Qwen3_5_9B_Q4, + LlmModelId::Qwen3_6_27B_Q4, ]; static ACTIVE_DOWNLOADS: LazyLock>> = @@ -206,16 +227,15 @@ pub fn model_info(id: LlmModelId) -> LlmModelInfo { } pub fn recommend_tier(total_ram_bytes: u64, total_vram_bytes: Option) -> LlmModelId { - if total_vram_bytes.unwrap_or(0) >= 16 * 1024_u64.pow(3) - && total_ram_bytes >= 32 * 1024_u64.pow(3) - { - LlmModelId::Qwen3_14BQ5 - } else if total_vram_bytes.unwrap_or(0) >= 8 * 1024_u64.pow(3) - || total_ram_bytes >= 16 * 1024_u64.pow(3) - { - LlmModelId::Qwen3_4BInstruct2507Q4 + let vram = total_vram_bytes.unwrap_or(0); + if vram >= 24 * 1024_u64.pow(3) && total_ram_bytes >= 64 * 1024_u64.pow(3) { + LlmModelId::Qwen3_6_27B_Q4 + } else if vram >= 12 * 1024_u64.pow(3) && total_ram_bytes >= 32 * 1024_u64.pow(3) { + LlmModelId::Qwen3_5_9B_Q4 + } else if vram >= 6 * 1024_u64.pow(3) || total_ram_bytes >= 16 * 1024_u64.pow(3) { + LlmModelId::Qwen3_5_4B_Q4 } else { - LlmModelId::Qwen3_1_7B_Q4 + LlmModelId::Qwen3_5_2B_Q4 } } @@ -389,15 +409,15 @@ mod tests { #[test] fn model_path_contains_model_dir_and_filename() { - let path = model_path(LlmModelId::Qwen3_1_7B_Q4); - assert!(path.to_string_lossy().ends_with("Qwen3-1.7B-Q4_K_M.gguf")); + let path = model_path(LlmModelId::Qwen3_5_2B_Q4); + assert!(path.to_string_lossy().ends_with("Qwen3.5-2B-Q4_K_M.gguf")); assert!(path.starts_with(model_dir())); } #[test] fn recommend_tier_prefers_mid_by_default() { let tier = recommend_tier(16 * 1024_u64.pow(3), None); - assert_eq!(tier, LlmModelId::Qwen3_4BInstruct2507Q4); + assert_eq!(tier, LlmModelId::Qwen3_5_4B_Q4); } #[tokio::test] diff --git a/crates/llm/tests/content_tags_smoke.rs b/crates/llm/tests/content_tags_smoke.rs index 99671b9..3f90cb5 100644 --- a/crates/llm/tests/content_tags_smoke.rs +++ b/crates/llm/tests/content_tags_smoke.rs @@ -24,7 +24,7 @@ fn extract_content_tags_returns_valid_pair() { let engine = LlmEngine::new(); engine - .load_model(LlmModelId::Qwen3_1_7B_Q4, &model_path, true) + .load_model(LlmModelId::Qwen3_5_2B_Q4, &model_path, true) .expect("load model"); let transcript = "Tomorrow I need to run through the grant application one more time \ diff --git a/crates/llm/tests/smoke.rs b/crates/llm/tests/smoke.rs index 9b9f7b0..3e01a07 100644 --- a/crates/llm/tests/smoke.rs +++ b/crates/llm/tests/smoke.rs @@ -26,7 +26,7 @@ fn llama_cpp_2_smoke_generates_and_wraps() { let engine = LlmEngine::new(); engine - .load_model(LlmModelId::Qwen3_1_7B_Q4, &model_path, true) + .load_model(LlmModelId::Qwen3_5_2B_Q4, &model_path, true) .expect("load model"); let completion = engine