feat(core): tuning module with Workload + helper skeleton
Adds crates/core/src/tuning.rs with MIN/MAX_INFERENCE_THREADS consts, Workload enum (Llm/Whisper), and inference_thread_count() helper matching the existing constants::inference_thread_count clamp behaviour. Three unit tests pass. tracing dep added to Cargo.toml (used by future tasks). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,7 @@ thiserror = "2"
|
|||||||
sysinfo = "0.35"
|
sysinfo = "0.35"
|
||||||
async-trait = "0.1"
|
async-trait = "0.1"
|
||||||
num_cpus = "1"
|
num_cpus = "1"
|
||||||
|
tracing = "0.1"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ pub mod paths;
|
|||||||
pub mod process_watch;
|
pub mod process_watch;
|
||||||
pub mod recommendation;
|
pub mod recommendation;
|
||||||
pub mod power;
|
pub mod power;
|
||||||
|
pub mod tuning;
|
||||||
pub mod types;
|
pub mod types;
|
||||||
|
|
||||||
pub use error::{MagnotiaError, Result};
|
pub use error::{MagnotiaError, Result};
|
||||||
|
|||||||
90
crates/core/src/tuning.rs
Normal file
90
crates/core/src/tuning.rs
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
//! Inference thread-count tuning. Combines physical-core budget,
|
||||||
|
//! battery awareness, and GPU-offload awareness into a single helper
|
||||||
|
//! callable from both inference call sites.
|
||||||
|
|
||||||
|
use crate::power::{self, PowerState};
|
||||||
|
|
||||||
|
/// Lower bound for inference threads. Single-threaded inference is
|
||||||
|
/// measurably worse than two on every multi-core part.
|
||||||
|
pub const MIN_INFERENCE_THREADS: usize = 2;
|
||||||
|
|
||||||
|
/// Upper bound for inference threads. whisper.cpp + llama.cpp scaling
|
||||||
|
/// flattens around physical core count; SMT siblings contend for FPU
|
||||||
|
/// during F16/F32 matmul, so going past physical cores anti-scales.
|
||||||
|
/// 8 is a conservative ceiling that leaves <5% on the table for
|
||||||
|
/// big-iron desktops while keeping consumer 6c/12t laptops out of
|
||||||
|
/// contention territory. Users can override at runtime via
|
||||||
|
/// MAGNOTIA_INFERENCE_THREADS.
|
||||||
|
pub const MAX_INFERENCE_THREADS: usize = 8;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum Workload {
|
||||||
|
/// Llama-style transformer. Near-zero CPU work when fully
|
||||||
|
/// offloaded; CPU floor in that case is GPU_FLOOR_LLM.
|
||||||
|
Llm,
|
||||||
|
/// Whisper transcription. Keeps mel spectrogram, decoder
|
||||||
|
/// bookkeeping, and beam search on the CPU even with full Vulkan
|
||||||
|
/// offload; CPU floor is higher: GPU_FLOOR_WHISPER.
|
||||||
|
Whisper,
|
||||||
|
}
|
||||||
|
|
||||||
|
const GPU_FLOOR_LLM: usize = 2;
|
||||||
|
const GPU_FLOOR_WHISPER: usize = 4;
|
||||||
|
|
||||||
|
/// Inference thread count, clamped to the physical-core budget plus
|
||||||
|
/// the battery and GPU-offload heuristics.
|
||||||
|
///
|
||||||
|
/// Resolution order:
|
||||||
|
/// 1. `MAGNOTIA_INFERENCE_THREADS=N` — absolute bypass, returns N.
|
||||||
|
/// 2. base = num_cpus::get_physical() (fallback: available_parallelism).
|
||||||
|
/// 3. on battery → base /= 2.
|
||||||
|
/// 4. gpu_offloaded → base = min(base, gpu_floor(workload)).
|
||||||
|
/// 5. clamp to [MIN_INFERENCE_THREADS, MAX_INFERENCE_THREADS].
|
||||||
|
pub fn inference_thread_count(_workload: Workload, _gpu_offloaded: bool) -> usize {
|
||||||
|
if let Ok(s) = std::env::var("MAGNOTIA_INFERENCE_THREADS") {
|
||||||
|
if let Ok(n) = s.parse::<usize>() {
|
||||||
|
if n > 0 {
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let physical = num_cpus::get_physical();
|
||||||
|
let chosen = if physical > 0 {
|
||||||
|
physical
|
||||||
|
} else {
|
||||||
|
std::thread::available_parallelism()
|
||||||
|
.map(|p| p.get())
|
||||||
|
.unwrap_or(MIN_INFERENCE_THREADS)
|
||||||
|
};
|
||||||
|
chosen.clamp(MIN_INFERENCE_THREADS, MAX_INFERENCE_THREADS)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn matches_existing_clamp_when_no_clamps_apply() {
|
||||||
|
// With no env override, no battery, no gpu_offload, helper
|
||||||
|
// should return physical-core count clamped to [2, 8].
|
||||||
|
// We can't pin physical exactly without mocking num_cpus; just
|
||||||
|
// assert the result is in range.
|
||||||
|
std::env::remove_var("MAGNOTIA_INFERENCE_THREADS");
|
||||||
|
let n = inference_thread_count(Workload::Llm, false);
|
||||||
|
assert!(n >= MIN_INFERENCE_THREADS && n <= MAX_INFERENCE_THREADS,
|
||||||
|
"got {n}, expected within [{MIN_INFERENCE_THREADS}, {MAX_INFERENCE_THREADS}]");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn env_var_bypasses_clamps() {
|
||||||
|
std::env::set_var("MAGNOTIA_INFERENCE_THREADS", "10");
|
||||||
|
let n = inference_thread_count(Workload::Llm, true);
|
||||||
|
assert_eq!(n, 10);
|
||||||
|
std::env::remove_var("MAGNOTIA_INFERENCE_THREADS");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn workload_variants_distinct() {
|
||||||
|
assert_ne!(Workload::Llm, Workload::Whisper);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user