perf: route whisper + llm n_threads through physical-core helper
Both inference call sites previously called `num_cpus::get()` (logical
thread count). Established whisper.cpp / llama.cpp guidance is that
SMT siblings contend for shared FPU resources during heavy F16/F32
matmul, so going past physical core count anti-scales. Empirical
sweep on Whisper Tiny / 11s JFK clip / Ryzen 5 4650U (6c12t):
n_threads | xc_time | RTF | speedup_vs_1
----------|---------|--------|-------------
1 | 0.33s | 0.030 | 1.00x
2 | 0.33s | 0.030 | 1.00x
4 | 0.30s | 0.028 | 1.09x
6 | 0.32s | 0.029 | 1.04x
8 | 0.31s | 0.028 | 1.06x
12 | 0.32s | 0.029 | 1.03x
Tiny doesn't scale (work dominated by overhead) but the larger
Whisper variants and Qwen LLMs do. Sources: whisper.cpp #200, #1033,
#1252, #403; llama.cpp #3167, #572.
Changes:
- crates/core/src/constants.rs:
* MIN_INFERENCE_THREADS lowered 4 → 2 (research-derived floor)
* MAX_INFERENCE_THREADS = 8 added (research-derived ceiling)
* inference_thread_count() rewritten:
- reads MAGNOTIA_INFERENCE_THREADS env var (override)
- num_cpus::get_physical() with available_parallelism fallback
- clamped to [MIN_INFERENCE_THREADS, MAX_INFERENCE_THREADS]
- crates/core/Cargo.toml: + num_cpus = "1"
- crates/transcription/src/whisper_rs_backend.rs: call site uses helper.
- crates/llm/src/lib.rs: call site uses helper.
- crates/transcription/Cargo.toml: drop num_cpus from [features] +
[dependencies] (production no longer needs it). Move to
[dev-dependencies] for tests/thread_sweep.rs only.
- crates/llm/Cargo.toml: drop num_cpus = "1" (no longer used directly).
Per-machine maps (after this patch):
Ryzen 5 4650U 6c12t → 6
big-iron 12c24t → 8 (clamp; users can override)
cheap 2c2t laptop → 2
1c container/VM → 2
Adds crates/transcription/tests/thread_sweep.rs — env-gated like
jfk_bench, prints the table above against any model + WAV. Useful for
re-baselining on new hardware or when tuning the clamp values.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
76
crates/transcription/tests/thread_sweep.rs
Normal file
76
crates/transcription/tests/thread_sweep.rs
Normal file
@@ -0,0 +1,76 @@
|
||||
//! Thread-count scaling sweep for Whisper Tiny.
|
||||
//! Runs the JFK clip at n_threads = 1, 2, 4, 6, 8, 12, prints RTF table.
|
||||
//! Gated on the same env vars as jfk_bench.
|
||||
|
||||
use std::env;
|
||||
use std::time::Instant;
|
||||
|
||||
#[test]
|
||||
fn whisper_thread_count_sweep() {
|
||||
let Ok(model_path) = env::var("MAGNOTIA_WHISPER_TEST_MODEL") else { return };
|
||||
let Ok(audio_path) = env::var("MAGNOTIA_WHISPER_TEST_AUDIO") else { return };
|
||||
|
||||
use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters};
|
||||
|
||||
let bytes = std::fs::read(&audio_path).expect("read wav");
|
||||
let sample_rate = u32::from_le_bytes(bytes[24..28].try_into().unwrap());
|
||||
let pcm = &bytes[44..];
|
||||
let samples: Vec<f32> = pcm
|
||||
.chunks_exact(2)
|
||||
.map(|c| i16::from_le_bytes([c[0], c[1]]) as f32 / 32768.0)
|
||||
.collect();
|
||||
let audio_secs = samples.len() as f64 / sample_rate as f64;
|
||||
eprintln!("[sweep] audio: {:.2}s @ {} Hz", audio_secs, sample_rate);
|
||||
|
||||
let logical = num_cpus::get();
|
||||
let physical = num_cpus::get_physical();
|
||||
eprintln!("[sweep] CPU: physical={}, logical={}", physical, logical);
|
||||
|
||||
let ctx = WhisperContext::new_with_params(&model_path, WhisperContextParameters::default())
|
||||
.expect("model load");
|
||||
|
||||
// Warm-up pass at default to prime caches
|
||||
{
|
||||
let mut state = ctx.create_state().expect("state");
|
||||
let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 });
|
||||
params.set_language(Some("en"));
|
||||
params.set_n_threads(physical as i32);
|
||||
params.set_print_special(false);
|
||||
params.set_print_progress(false);
|
||||
params.set_print_realtime(false);
|
||||
state.full(params, &samples).expect("warmup");
|
||||
}
|
||||
|
||||
let mut targets: Vec<i32> = vec![1, 2, 4, physical as i32, logical as i32];
|
||||
if logical >= 8 && !targets.contains(&8) { targets.push(8); }
|
||||
targets.sort();
|
||||
targets.dedup();
|
||||
|
||||
eprintln!("");
|
||||
eprintln!("=== n_threads scaling ===");
|
||||
eprintln!("n_threads | xc_time | RTF | speedup_vs_1");
|
||||
eprintln!("----------|---------|--------|-------------");
|
||||
let mut baseline_dur: Option<f64> = None;
|
||||
for n in &targets {
|
||||
// Two runs, take the min (deeper of L2/L3 effects; we want best-case)
|
||||
let mut best = f64::MAX;
|
||||
for _ in 0..2 {
|
||||
let mut state = ctx.create_state().expect("state");
|
||||
let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 });
|
||||
params.set_language(Some("en"));
|
||||
params.set_n_threads(*n);
|
||||
params.set_print_special(false);
|
||||
params.set_print_progress(false);
|
||||
params.set_print_realtime(false);
|
||||
let t = Instant::now();
|
||||
state.full(params, &samples).expect("transcribe");
|
||||
let dur = t.elapsed().as_secs_f64();
|
||||
if dur < best { best = dur; }
|
||||
}
|
||||
let rtf = best / audio_secs;
|
||||
let speedup = baseline_dur.map(|b| b / best).unwrap_or(1.0);
|
||||
if baseline_dur.is_none() { baseline_dur = Some(best); }
|
||||
eprintln!("{:>9} | {:>6.2}s | {:>6.3} | {:>6.2}x",
|
||||
n, best, rtf, speedup);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user