feat(core): tuning module with Workload + helper skeleton
Adds crates/core/src/tuning.rs with MIN/MAX_INFERENCE_THREADS consts, Workload enum (Llm/Whisper), and inference_thread_count() helper matching the existing constants::inference_thread_count clamp behaviour. Three unit tests pass. tracing dep added to Cargo.toml (used by future tasks). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,7 @@ thiserror = "2"
|
||||
sysinfo = "0.35"
|
||||
async-trait = "0.1"
|
||||
num_cpus = "1"
|
||||
tracing = "0.1"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
@@ -6,6 +6,7 @@ pub mod paths;
|
||||
pub mod process_watch;
|
||||
pub mod recommendation;
|
||||
pub mod power;
|
||||
pub mod tuning;
|
||||
pub mod types;
|
||||
|
||||
pub use error::{MagnotiaError, Result};
|
||||
|
||||
90
crates/core/src/tuning.rs
Normal file
90
crates/core/src/tuning.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
//! Inference thread-count tuning. Combines physical-core budget,
|
||||
//! battery awareness, and GPU-offload awareness into a single helper
|
||||
//! callable from both inference call sites.
|
||||
|
||||
use crate::power::{self, PowerState};
|
||||
|
||||
/// Lower bound for inference threads. Single-threaded inference is
|
||||
/// measurably worse than two on every multi-core part.
|
||||
pub const MIN_INFERENCE_THREADS: usize = 2;
|
||||
|
||||
/// Upper bound for inference threads. whisper.cpp + llama.cpp scaling
|
||||
/// flattens around physical core count; SMT siblings contend for FPU
|
||||
/// during F16/F32 matmul, so going past physical cores anti-scales.
|
||||
/// 8 is a conservative ceiling that leaves <5% on the table for
|
||||
/// big-iron desktops while keeping consumer 6c/12t laptops out of
|
||||
/// contention territory. Users can override at runtime via
|
||||
/// MAGNOTIA_INFERENCE_THREADS.
|
||||
pub const MAX_INFERENCE_THREADS: usize = 8;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Workload {
|
||||
/// Llama-style transformer. Near-zero CPU work when fully
|
||||
/// offloaded; CPU floor in that case is GPU_FLOOR_LLM.
|
||||
Llm,
|
||||
/// Whisper transcription. Keeps mel spectrogram, decoder
|
||||
/// bookkeeping, and beam search on the CPU even with full Vulkan
|
||||
/// offload; CPU floor is higher: GPU_FLOOR_WHISPER.
|
||||
Whisper,
|
||||
}
|
||||
|
||||
const GPU_FLOOR_LLM: usize = 2;
|
||||
const GPU_FLOOR_WHISPER: usize = 4;
|
||||
|
||||
/// Inference thread count, clamped to the physical-core budget plus
|
||||
/// the battery and GPU-offload heuristics.
|
||||
///
|
||||
/// Resolution order:
|
||||
/// 1. `MAGNOTIA_INFERENCE_THREADS=N` — absolute bypass, returns N.
|
||||
/// 2. base = num_cpus::get_physical() (fallback: available_parallelism).
|
||||
/// 3. on battery → base /= 2.
|
||||
/// 4. gpu_offloaded → base = min(base, gpu_floor(workload)).
|
||||
/// 5. clamp to [MIN_INFERENCE_THREADS, MAX_INFERENCE_THREADS].
|
||||
pub fn inference_thread_count(_workload: Workload, _gpu_offloaded: bool) -> usize {
|
||||
if let Ok(s) = std::env::var("MAGNOTIA_INFERENCE_THREADS") {
|
||||
if let Ok(n) = s.parse::<usize>() {
|
||||
if n > 0 {
|
||||
return n;
|
||||
}
|
||||
}
|
||||
}
|
||||
let physical = num_cpus::get_physical();
|
||||
let chosen = if physical > 0 {
|
||||
physical
|
||||
} else {
|
||||
std::thread::available_parallelism()
|
||||
.map(|p| p.get())
|
||||
.unwrap_or(MIN_INFERENCE_THREADS)
|
||||
};
|
||||
chosen.clamp(MIN_INFERENCE_THREADS, MAX_INFERENCE_THREADS)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn matches_existing_clamp_when_no_clamps_apply() {
|
||||
// With no env override, no battery, no gpu_offload, helper
|
||||
// should return physical-core count clamped to [2, 8].
|
||||
// We can't pin physical exactly without mocking num_cpus; just
|
||||
// assert the result is in range.
|
||||
std::env::remove_var("MAGNOTIA_INFERENCE_THREADS");
|
||||
let n = inference_thread_count(Workload::Llm, false);
|
||||
assert!(n >= MIN_INFERENCE_THREADS && n <= MAX_INFERENCE_THREADS,
|
||||
"got {n}, expected within [{MIN_INFERENCE_THREADS}, {MAX_INFERENCE_THREADS}]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn env_var_bypasses_clamps() {
|
||||
std::env::set_var("MAGNOTIA_INFERENCE_THREADS", "10");
|
||||
let n = inference_thread_count(Workload::Llm, true);
|
||||
assert_eq!(n, 10);
|
||||
std::env::remove_var("MAGNOTIA_INFERENCE_THREADS");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workload_variants_distinct() {
|
||||
assert_ne!(Workload::Llm, Workload::Whisper);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user