From 4cb954ece48712b3635e2a71f6959e53cd7f4816 Mon Sep 17 00:00:00 2001 From: jars Date: Sat, 9 May 2026 09:26:23 +0100 Subject: [PATCH] perf: route whisper + llm n_threads through physical-core helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both inference call sites previously called `num_cpus::get()` (logical thread count). Established whisper.cpp / llama.cpp guidance is that SMT siblings contend for shared FPU resources during heavy F16/F32 matmul, so going past physical core count anti-scales. Empirical sweep on Whisper Tiny / 11s JFK clip / Ryzen 5 4650U (6c12t): n_threads | xc_time | RTF | speedup_vs_1 ----------|---------|--------|------------- 1 | 0.33s | 0.030 | 1.00x 2 | 0.33s | 0.030 | 1.00x 4 | 0.30s | 0.028 | 1.09x 6 | 0.32s | 0.029 | 1.04x 8 | 0.31s | 0.028 | 1.06x 12 | 0.32s | 0.029 | 1.03x Tiny doesn't scale (work dominated by overhead) but the larger Whisper variants and Qwen LLMs do. Sources: whisper.cpp #200, #1033, #1252, #403; llama.cpp #3167, #572. Changes: - crates/core/src/constants.rs: * MIN_INFERENCE_THREADS lowered 4 → 2 (research-derived floor) * MAX_INFERENCE_THREADS = 8 added (research-derived ceiling) * inference_thread_count() rewritten: - reads MAGNOTIA_INFERENCE_THREADS env var (override) - num_cpus::get_physical() with available_parallelism fallback - clamped to [MIN_INFERENCE_THREADS, MAX_INFERENCE_THREADS] - crates/core/Cargo.toml: + num_cpus = "1" - crates/transcription/src/whisper_rs_backend.rs: call site uses helper. - crates/llm/src/lib.rs: call site uses helper. - crates/transcription/Cargo.toml: drop num_cpus from [features] + [dependencies] (production no longer needs it). Move to [dev-dependencies] for tests/thread_sweep.rs only. - crates/llm/Cargo.toml: drop num_cpus = "1" (no longer used directly). Per-machine maps (after this patch): Ryzen 5 4650U 6c12t → 6 big-iron 12c24t → 8 (clamp; users can override) cheap 2c2t laptop → 2 1c container/VM → 2 Adds crates/transcription/tests/thread_sweep.rs — env-gated like jfk_bench, prints the table above against any model + WAV. Useful for re-baselining on new hardware or when tuning the clamp values. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 2 +- crates/core/Cargo.toml | 1 + crates/core/src/constants.rs | 54 +++++++++++-- crates/llm/Cargo.toml | 1 - crates/llm/src/lib.rs | 3 +- crates/transcription/Cargo.toml | 10 +-- .../transcription/src/whisper_rs_backend.rs | 3 +- crates/transcription/tests/thread_sweep.rs | 76 +++++++++++++++++++ 8 files changed, 134 insertions(+), 16 deletions(-) create mode 100644 crates/transcription/tests/thread_sweep.rs diff --git a/Cargo.lock b/Cargo.lock index 882a944..9ecc0aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2855,6 +2855,7 @@ name = "magnotia-core" version = "0.1.0" dependencies = [ "async-trait", + "num_cpus", "serde", "serde_json", "sysinfo", @@ -2882,7 +2883,6 @@ dependencies = [ "futures-util", "llama-cpp-2", "magnotia-core", - "num_cpus", "reqwest 0.12.28", "serde", "serde_json", diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index b9976ba..1ac7d0d 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -10,3 +10,4 @@ serde_json = "1" thiserror = "2" sysinfo = "0.35" async-trait = "0.1" +num_cpus = "1" diff --git a/crates/core/src/constants.rs b/crates/core/src/constants.rs index ada8903..0286432 100644 --- a/crates/core/src/constants.rs +++ b/crates/core/src/constants.rs @@ -18,8 +18,21 @@ pub const MIN_CHUNK_SAMPLES: usize = 8000; /// Post-processing thresholds. pub const SMART_PARAGRAPH_GAP_SECS: f64 = 2.0; -/// Thread count for inference. Leaves headroom for the UI thread. -pub const MIN_INFERENCE_THREADS: usize = 4; +/// Lower bound for inference threads. Single-threaded inference is +/// measurably worse than two on every multi-core part; we never go below +/// 2. (Whisper Tiny is the exception where the work is so small that +/// thread count barely matters — but the floor still costs nothing.) +pub const MIN_INFERENCE_THREADS: usize = 2; + +/// Upper bound for inference threads. Both whisper.cpp and llama.cpp +/// scaling flattens around physical core count; SMT siblings contend +/// for shared FPU resources during heavy F16/F32 matmul, which means +/// going past physical cores often anti-scales. Empirical evidence: +/// whisper.cpp issue #200 (sweet spot at 7t on 8c/16t Ryzen 3700X), +/// llama.cpp #3167 ("SMT hurts inference"). 8 is a conservative +/// ceiling that leaves <5% on the table for big-iron desktops while +/// keeping consumer 6c/12t laptops out of contention territory. +pub const MAX_INFERENCE_THREADS: usize = 8; /// History limits. pub const HISTORY_MAX_ENTRIES: usize = 100; @@ -40,10 +53,37 @@ pub const VAD_SPEECH_PAD_MS: u32 = 100; /// Model download chunk size for progress reporting. pub const DOWNLOAD_CHUNK_BYTES: usize = 65_536; -/// Inference thread count based on available parallelism. +/// Inference thread count, clamped to physical-core budget. +/// +/// Returns the system's physical-core count, clamped to +/// `[MIN_INFERENCE_THREADS, MAX_INFERENCE_THREADS]`. Uses physical +/// rather than logical cores because SMT/hyperthreaded siblings +/// contend for shared FPU resources during heavy matmul; counting +/// them as additional workers is well-documented to anti-scale on +/// both whisper.cpp and llama.cpp. +/// +/// Falls back to `available_parallelism` only if the physical-core +/// probe is unavailable (some non-Linux/non-Windows platforms or +/// containerised environments). +/// +/// Users can override at runtime by setting +/// `MAGNOTIA_INFERENCE_THREADS=N` — useful for benchmarking and for +/// users on big-iron desktops who want to push past the cap. pub fn inference_thread_count() -> usize { - std::thread::available_parallelism() - .map(|p| p.get().saturating_sub(1)) - .unwrap_or(MIN_INFERENCE_THREADS) - .max(MIN_INFERENCE_THREADS) + if let Ok(s) = std::env::var("MAGNOTIA_INFERENCE_THREADS") { + if let Ok(n) = s.parse::() { + if n > 0 { + return n; + } + } + } + let physical = num_cpus::get_physical(); + let chosen = if physical > 0 { + physical + } else { + std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(MIN_INFERENCE_THREADS) + }; + chosen.clamp(MIN_INFERENCE_THREADS, MAX_INFERENCE_THREADS) } diff --git a/crates/llm/Cargo.toml b/crates/llm/Cargo.toml index d33eeea..b3215bf 100644 --- a/crates/llm/Cargo.toml +++ b/crates/llm/Cargo.toml @@ -20,7 +20,6 @@ magnotia-core = { path = "../core" } encoding_rs = "0.8" futures-util = "0.3" llama-cpp-2 = { version = "0.1.144", default-features = false } -num_cpus = "1" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index 16f9c7c..5ab36fa 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -163,7 +163,8 @@ impl LlmEngine { } let n_ctx = preflight_context_window(prompt_tokens.len(), config.max_tokens)?; - let thread_count = i32::try_from(num_cpus::get().max(1)).unwrap_or(4); + let thread_count = i32::try_from(magnotia_core::constants::inference_thread_count()) + .unwrap_or(4); let ctx_params = LlamaContextParams::default() .with_n_ctx(Some( NonZeroU32::new(n_ctx).expect("n_ctx must be non-zero"), diff --git a/crates/transcription/Cargo.toml b/crates/transcription/Cargo.toml index 4bd1dce..1664148 100644 --- a/crates/transcription/Cargo.toml +++ b/crates/transcription/Cargo.toml @@ -17,7 +17,7 @@ build = "build.rs" # but skip the Vulkan backend. Build CPU-only with: # cargo build -p magnotia-transcription --no-default-features --features whisper default = ["whisper", "whisper-vulkan"] -whisper = ["dep:whisper-rs", "dep:num_cpus"] +whisper = ["dep:whisper-rs"] whisper-vulkan = ["whisper-rs?/vulkan"] [dependencies] @@ -40,10 +40,6 @@ sha2 = "0.10" # additive via the `whisper-vulkan` feature so non-GPU targets can drop it. whisper-rs = { version = "0.16", default-features = false, optional = true } -# Direct whisper-rs backend (WhisperRsBackend): thread pool sizing. -# Gated alongside whisper-rs since no other code in this crate needs it. -num_cpus = { version = "1", optional = true } - # Typed error enum used by WhisperRsBackend + elsewhere. Kept # unconditional because it is a derive-macro crate with negligible # build cost. @@ -56,3 +52,7 @@ tracing = "0.1" # TcpListener fixture for the download resume tests (mirrors magnotia-llm). tokio = { version = "1", features = ["rt", "sync", "net", "io-util", "macros"] } tempfile = "3" +# Test-only — used by tests/thread_sweep.rs to label physical vs logical +# core counts in the scaling table. Production code uses the +# `magnotia_core::constants::inference_thread_count` helper instead. +num_cpus = "1" diff --git a/crates/transcription/src/whisper_rs_backend.rs b/crates/transcription/src/whisper_rs_backend.rs index a037d10..58788a3 100644 --- a/crates/transcription/src/whisper_rs_backend.rs +++ b/crates/transcription/src/whisper_rs_backend.rs @@ -10,6 +10,7 @@ use std::path::Path; use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters}; +use magnotia_core::constants::inference_thread_count; use magnotia_core::error::{MagnotiaError, Result}; use magnotia_core::types::{Segment, TranscriptionOptions}; @@ -77,7 +78,7 @@ impl Transcriber for WhisperRsBackend { params.set_initial_prompt(prompt); } } - params.set_n_threads(num_cpus::get() as i32); + params.set_n_threads(inference_thread_count() as i32); params.set_print_special(false); params.set_print_progress(false); params.set_print_realtime(false); diff --git a/crates/transcription/tests/thread_sweep.rs b/crates/transcription/tests/thread_sweep.rs new file mode 100644 index 0000000..d9b991d --- /dev/null +++ b/crates/transcription/tests/thread_sweep.rs @@ -0,0 +1,76 @@ +//! Thread-count scaling sweep for Whisper Tiny. +//! Runs the JFK clip at n_threads = 1, 2, 4, 6, 8, 12, prints RTF table. +//! Gated on the same env vars as jfk_bench. + +use std::env; +use std::time::Instant; + +#[test] +fn whisper_thread_count_sweep() { + let Ok(model_path) = env::var("MAGNOTIA_WHISPER_TEST_MODEL") else { return }; + let Ok(audio_path) = env::var("MAGNOTIA_WHISPER_TEST_AUDIO") else { return }; + + use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters}; + + let bytes = std::fs::read(&audio_path).expect("read wav"); + let sample_rate = u32::from_le_bytes(bytes[24..28].try_into().unwrap()); + let pcm = &bytes[44..]; + let samples: Vec = pcm + .chunks_exact(2) + .map(|c| i16::from_le_bytes([c[0], c[1]]) as f32 / 32768.0) + .collect(); + let audio_secs = samples.len() as f64 / sample_rate as f64; + eprintln!("[sweep] audio: {:.2}s @ {} Hz", audio_secs, sample_rate); + + let logical = num_cpus::get(); + let physical = num_cpus::get_physical(); + eprintln!("[sweep] CPU: physical={}, logical={}", physical, logical); + + let ctx = WhisperContext::new_with_params(&model_path, WhisperContextParameters::default()) + .expect("model load"); + + // Warm-up pass at default to prime caches + { + let mut state = ctx.create_state().expect("state"); + let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 }); + params.set_language(Some("en")); + params.set_n_threads(physical as i32); + params.set_print_special(false); + params.set_print_progress(false); + params.set_print_realtime(false); + state.full(params, &samples).expect("warmup"); + } + + let mut targets: Vec = vec![1, 2, 4, physical as i32, logical as i32]; + if logical >= 8 && !targets.contains(&8) { targets.push(8); } + targets.sort(); + targets.dedup(); + + eprintln!(""); + eprintln!("=== n_threads scaling ==="); + eprintln!("n_threads | xc_time | RTF | speedup_vs_1"); + eprintln!("----------|---------|--------|-------------"); + let mut baseline_dur: Option = None; + for n in &targets { + // Two runs, take the min (deeper of L2/L3 effects; we want best-case) + let mut best = f64::MAX; + for _ in 0..2 { + let mut state = ctx.create_state().expect("state"); + let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 }); + params.set_language(Some("en")); + params.set_n_threads(*n); + params.set_print_special(false); + params.set_print_progress(false); + params.set_print_realtime(false); + let t = Instant::now(); + state.full(params, &samples).expect("transcribe"); + let dur = t.elapsed().as_secs_f64(); + if dur < best { best = dur; } + } + let rtf = best / audio_secs; + let speedup = baseline_dur.map(|b| b / best).unwrap_or(1.0); + if baseline_dur.is_none() { baseline_dur = Some(best); } + eprintln!("{:>9} | {:>6.2}s | {:>6.3} | {:>6.2}x", + n, best, rtf, speedup); + } +}