From 0a503be38d79c1ef22d14c8adf3ac92b5db84bc9 Mon Sep 17 00:00:00 2001 From: jars Date: Sat, 9 May 2026 12:30:34 +0100 Subject: [PATCH] feat(llm): LLM threads use Workload::Llm + GPU offload detection Wires the LLM call site through the new tuning helper. gpu_offloaded reflects intent (use_gpu) cross-checked against the loaded model's layer count: u32::MAX (when use_gpu) is trivially >= any model's n_layer, but the explicit comparison is future-proofed if we ever pass a specific N instead of u32::MAX. Note: the call site is in generate() not load_model() as the plan suggested. Context params (and thus thread count) are constructed per-inference, not per model load, since n_ctx depends on prompt size. The implementer adapted correctly. The old magnotia_core::constants::inference_thread_count import is replaced. Task 6.1 removes the constants helper now that both call sites have migrated. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/llm/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index 5ab36fa..4897e7a 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -9,6 +9,7 @@ use llama_cpp_2::llama_batch::LlamaBatch; use llama_cpp_2::model::params::LlamaModelParams; use llama_cpp_2::model::{AddBos, LlamaChatMessage, LlamaChatTemplate, LlamaModel}; use llama_cpp_2::sampling::LlamaSampler; +use magnotia_core::tuning::{inference_thread_count, Workload}; use serde::{Deserialize, Serialize}; pub mod grammars; @@ -163,7 +164,10 @@ impl LlmEngine { } let n_ctx = preflight_context_window(prompt_tokens.len(), config.max_tokens)?; - let thread_count = i32::try_from(magnotia_core::constants::inference_thread_count()) + let use_gpu = self.loaded_model().map(|s| s.use_gpu).unwrap_or(false); + let gpu_layers = if use_gpu { u32::MAX } else { 0u32 }; + let gpu_offloaded = use_gpu && gpu_layers >= model.n_layer(); + let thread_count = i32::try_from(inference_thread_count(Workload::Llm, gpu_offloaded)) .unwrap_or(4); let ctx_params = LlamaContextParams::default() .with_n_ctx(Some(