perf: route whisper + llm n_threads through physical-core helper
Some checks failed
check / cargo check (macos-latest) (push) Has been cancelled
check / cargo check (ubuntu-22.04) (push) Has been cancelled
check / cargo check (windows-latest) (push) Has been cancelled
check / svelte build + lint (push) Has been cancelled

Both inference call sites previously called `num_cpus::get()` (logical
thread count). Established whisper.cpp / llama.cpp guidance is that
SMT siblings contend for shared FPU resources during heavy F16/F32
matmul, so going past physical core count anti-scales. Empirical
sweep on Whisper Tiny / 11s JFK clip / Ryzen 5 4650U (6c12t):

  n_threads | xc_time | RTF    | speedup_vs_1
  ----------|---------|--------|-------------
          1 |   0.33s |  0.030 |   1.00x
          2 |   0.33s |  0.030 |   1.00x
          4 |   0.30s |  0.028 |   1.09x
          6 |   0.32s |  0.029 |   1.04x
          8 |   0.31s |  0.028 |   1.06x
         12 |   0.32s |  0.029 |   1.03x

Tiny doesn't scale (work dominated by overhead) but the larger
Whisper variants and Qwen LLMs do. Sources: whisper.cpp #200, #1033,
#1252, #403; llama.cpp #3167, #572.

Changes:
- crates/core/src/constants.rs:
  * MIN_INFERENCE_THREADS lowered 4 → 2 (research-derived floor)
  * MAX_INFERENCE_THREADS = 8 added (research-derived ceiling)
  * inference_thread_count() rewritten:
      - reads MAGNOTIA_INFERENCE_THREADS env var (override)
      - num_cpus::get_physical() with available_parallelism fallback
      - clamped to [MIN_INFERENCE_THREADS, MAX_INFERENCE_THREADS]
- crates/core/Cargo.toml: + num_cpus = "1"
- crates/transcription/src/whisper_rs_backend.rs: call site uses helper.
- crates/llm/src/lib.rs: call site uses helper.
- crates/transcription/Cargo.toml: drop num_cpus from [features] +
  [dependencies] (production no longer needs it). Move to
  [dev-dependencies] for tests/thread_sweep.rs only.
- crates/llm/Cargo.toml: drop num_cpus = "1" (no longer used directly).

Per-machine maps (after this patch):
  Ryzen 5 4650U  6c12t → 6
  big-iron 12c24t       → 8 (clamp; users can override)
  cheap 2c2t laptop     → 2
  1c container/VM       → 2

Adds crates/transcription/tests/thread_sweep.rs — env-gated like
jfk_bench, prints the table above against any model + WAV. Useful for
re-baselining on new hardware or when tuning the clamp values.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
jars
2026-05-09 09:26:23 +01:00
parent fdf27db0a1
commit 4cb954ece4
8 changed files with 134 additions and 16 deletions

View File

@@ -17,7 +17,7 @@ build = "build.rs"
# but skip the Vulkan backend. Build CPU-only with:
# cargo build -p magnotia-transcription --no-default-features --features whisper
default = ["whisper", "whisper-vulkan"]
whisper = ["dep:whisper-rs", "dep:num_cpus"]
whisper = ["dep:whisper-rs"]
whisper-vulkan = ["whisper-rs?/vulkan"]
[dependencies]
@@ -40,10 +40,6 @@ sha2 = "0.10"
# additive via the `whisper-vulkan` feature so non-GPU targets can drop it.
whisper-rs = { version = "0.16", default-features = false, optional = true }
# Direct whisper-rs backend (WhisperRsBackend): thread pool sizing.
# Gated alongside whisper-rs since no other code in this crate needs it.
num_cpus = { version = "1", optional = true }
# Typed error enum used by WhisperRsBackend + elsewhere. Kept
# unconditional because it is a derive-macro crate with negligible
# build cost.
@@ -56,3 +52,7 @@ tracing = "0.1"
# TcpListener fixture for the download resume tests (mirrors magnotia-llm).
tokio = { version = "1", features = ["rt", "sync", "net", "io-util", "macros"] }
tempfile = "3"
# Test-only — used by tests/thread_sweep.rs to label physical vs logical
# core counts in the scaling table. Production code uses the
# `magnotia_core::constants::inference_thread_count` helper instead.
num_cpus = "1"

View File

@@ -10,6 +10,7 @@ use std::path::Path;
use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters};
use magnotia_core::constants::inference_thread_count;
use magnotia_core::error::{MagnotiaError, Result};
use magnotia_core::types::{Segment, TranscriptionOptions};
@@ -77,7 +78,7 @@ impl Transcriber for WhisperRsBackend {
params.set_initial_prompt(prompt);
}
}
params.set_n_threads(num_cpus::get() as i32);
params.set_n_threads(inference_thread_count() as i32);
params.set_print_special(false);
params.set_print_progress(false);
params.set_print_realtime(false);

View File

@@ -0,0 +1,76 @@
//! Thread-count scaling sweep for Whisper Tiny.
//! Runs the JFK clip at n_threads = 1, 2, 4, 6, 8, 12, prints RTF table.
//! Gated on the same env vars as jfk_bench.
use std::env;
use std::time::Instant;
#[test]
fn whisper_thread_count_sweep() {
let Ok(model_path) = env::var("MAGNOTIA_WHISPER_TEST_MODEL") else { return };
let Ok(audio_path) = env::var("MAGNOTIA_WHISPER_TEST_AUDIO") else { return };
use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters};
let bytes = std::fs::read(&audio_path).expect("read wav");
let sample_rate = u32::from_le_bytes(bytes[24..28].try_into().unwrap());
let pcm = &bytes[44..];
let samples: Vec<f32> = pcm
.chunks_exact(2)
.map(|c| i16::from_le_bytes([c[0], c[1]]) as f32 / 32768.0)
.collect();
let audio_secs = samples.len() as f64 / sample_rate as f64;
eprintln!("[sweep] audio: {:.2}s @ {} Hz", audio_secs, sample_rate);
let logical = num_cpus::get();
let physical = num_cpus::get_physical();
eprintln!("[sweep] CPU: physical={}, logical={}", physical, logical);
let ctx = WhisperContext::new_with_params(&model_path, WhisperContextParameters::default())
.expect("model load");
// Warm-up pass at default to prime caches
{
let mut state = ctx.create_state().expect("state");
let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 });
params.set_language(Some("en"));
params.set_n_threads(physical as i32);
params.set_print_special(false);
params.set_print_progress(false);
params.set_print_realtime(false);
state.full(params, &samples).expect("warmup");
}
let mut targets: Vec<i32> = vec![1, 2, 4, physical as i32, logical as i32];
if logical >= 8 && !targets.contains(&8) { targets.push(8); }
targets.sort();
targets.dedup();
eprintln!("");
eprintln!("=== n_threads scaling ===");
eprintln!("n_threads | xc_time | RTF | speedup_vs_1");
eprintln!("----------|---------|--------|-------------");
let mut baseline_dur: Option<f64> = None;
for n in &targets {
// Two runs, take the min (deeper of L2/L3 effects; we want best-case)
let mut best = f64::MAX;
for _ in 0..2 {
let mut state = ctx.create_state().expect("state");
let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 });
params.set_language(Some("en"));
params.set_n_threads(*n);
params.set_print_special(false);
params.set_print_progress(false);
params.set_print_realtime(false);
let t = Instant::now();
state.full(params, &samples).expect("transcribe");
let dur = t.elapsed().as_secs_f64();
if dur < best { best = dur; }
}
let rtf = best / audio_secs;
let speedup = baseline_dur.map(|b| b / best).unwrap_or(1.0);
if baseline_dur.is_none() { baseline_dur = Some(best); }
eprintln!("{:>9} | {:>6.2}s | {:>6.3} | {:>6.2}x",
n, best, rtf, speedup);
}
}