test(transcription): thread_sweep prints power-aware 4-panel table

Extends the JFK thread-count sweep to print four labelled panels
(AC+CPU, AC+GPU, battery+CPU, battery+GPU) driven by
MAGNOTIA_POWER_STATE_OVERRIDE. Each panel shows the helper's
predicted thread count for its (power, gpu) combination alongside
the empirical RTF table for n_threads = [1, 2, 4, physical, logical,
maybe 8].

The actual whisper context is initialised once (Vulkan if compiled
in and resolvable, CPU otherwise), so the RTF rows themselves are
produced by the same backend across panels. The CPU vs GPU axis
controls only the helper-pick column, which is the empirical
question we want answered: is the LLM=2 / Whisper=4 GPU floor right?

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
jars
2026-05-09 12:43:04 +01:00
parent 7eb69e6251
commit a58c2f0994

View File

@@ -1,16 +1,26 @@
//! Thread-count scaling sweep for Whisper Tiny. //! Thread-count scaling sweep for Whisper Tiny.
//! Runs the JFK clip at n_threads = 1, 2, 4, 6, 8, 12, prints RTF table. //! Runs the JFK clip at n_threads = 1, 2, 4, 6, 8, 12, prints RTF tables.
//! Gated on the same env vars as jfk_bench. //! Env-gated by `MAGNOTIA_WHISPER_TEST_MODEL` + `MAGNOTIA_WHISPER_TEST_AUDIO`.
//!
//! Now prints multiple panels driven by `MAGNOTIA_POWER_STATE_OVERRIDE` so
//! the helper's predicted thread count for each (power, GPU) combination
//! can be compared against the empirical RTF data.
use std::env; use std::env;
use std::time::Instant; use std::time::Instant;
use magnotia_core::hardware::vulkan_loader_available;
use magnotia_core::tuning::{inference_thread_count, Workload};
use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters};
#[test] #[test]
fn whisper_thread_count_sweep() { fn whisper_thread_count_sweep() {
let Ok(model_path) = env::var("MAGNOTIA_WHISPER_TEST_MODEL") else { return }; let Ok(model_path) = env::var("MAGNOTIA_WHISPER_TEST_MODEL") else {
let Ok(audio_path) = env::var("MAGNOTIA_WHISPER_TEST_AUDIO") else { return }; return;
};
use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters}; let Ok(audio_path) = env::var("MAGNOTIA_WHISPER_TEST_AUDIO") else {
return;
};
let bytes = std::fs::read(&audio_path).expect("read wav"); let bytes = std::fs::read(&audio_path).expect("read wav");
let sample_rate = u32::from_le_bytes(bytes[24..28].try_into().unwrap()); let sample_rate = u32::from_le_bytes(bytes[24..28].try_into().unwrap());
@@ -29,7 +39,7 @@ fn whisper_thread_count_sweep() {
let ctx = WhisperContext::new_with_params(&model_path, WhisperContextParameters::default()) let ctx = WhisperContext::new_with_params(&model_path, WhisperContextParameters::default())
.expect("model load"); .expect("model load");
// Warm-up pass at default to prime caches // Warm-up pass to prime caches.
{ {
let mut state = ctx.create_state().expect("state"); let mut state = ctx.create_state().expect("state");
let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 }); let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 });
@@ -42,17 +52,65 @@ fn whisper_thread_count_sweep() {
} }
let mut targets: Vec<i32> = vec![1, 2, 4, physical as i32, logical as i32]; let mut targets: Vec<i32> = vec![1, 2, 4, physical as i32, logical as i32];
if logical >= 8 && !targets.contains(&8) { targets.push(8); } if logical >= 8 && !targets.contains(&8) {
targets.push(8);
}
targets.sort(); targets.sort();
targets.dedup(); targets.dedup();
// Snapshot the runtime Vulkan loader status once. The actual whisper
// context above already initialised whichever backend it could; the
// GPU panels below differ only in label and predicted-helper-pick.
// The runtime RTF rows are produced by the same backend the warm-up
// used.
let vulkan_runtime_ok = cfg!(feature = "whisper-vulkan") && vulkan_loader_available();
eprintln!(
"[sweep] whisper-vulkan feature: {}, libvulkan resolvable at runtime: {}",
cfg!(feature = "whisper-vulkan"),
vulkan_runtime_ok
);
// Four panels: CPU and GPU axes for the predicted-helper-pick column,
// crossed with AC and battery via MAGNOTIA_POWER_STATE_OVERRIDE.
let panels = [
("AC, CPU", "ac", false),
("AC, GPU (Vulkan)", "ac", true),
("battery, CPU", "battery", false),
("battery, GPU (Vulkan)", "battery", true),
];
for (label, power, gpu_offloaded_for_helper) in panels {
env::set_var("MAGNOTIA_POWER_STATE_OVERRIDE", power);
let helper_pick = inference_thread_count(Workload::Whisper, gpu_offloaded_for_helper);
run_sweep_panel(
label,
helper_pick,
&ctx,
&samples,
audio_secs,
&targets,
);
}
env::remove_var("MAGNOTIA_POWER_STATE_OVERRIDE");
}
fn run_sweep_panel(
label: &str,
helper_pick: usize,
ctx: &WhisperContext,
samples: &[f32],
audio_secs: f64,
targets: &[i32],
) {
eprintln!(""); eprintln!("");
eprintln!("=== n_threads scaling ==="); eprintln!(
"=== n_threads scaling: {label} (helper picks: {helper_pick}) ==="
);
eprintln!("n_threads | xc_time | RTF | speedup_vs_1"); eprintln!("n_threads | xc_time | RTF | speedup_vs_1");
eprintln!("----------|---------|--------|-------------"); eprintln!("----------|---------|--------|-------------");
let mut baseline_dur: Option<f64> = None; let mut baseline_dur: Option<f64> = None;
for n in &targets { for n in targets {
// Two runs, take the min (deeper of L2/L3 effects; we want best-case) // Two runs, take the min — best-case after L2/L3 warm.
let mut best = f64::MAX; let mut best = f64::MAX;
for _ in 0..2 { for _ in 0..2 {
let mut state = ctx.create_state().expect("state"); let mut state = ctx.create_state().expect("state");
@@ -63,14 +121,20 @@ fn whisper_thread_count_sweep() {
params.set_print_progress(false); params.set_print_progress(false);
params.set_print_realtime(false); params.set_print_realtime(false);
let t = Instant::now(); let t = Instant::now();
state.full(params, &samples).expect("transcribe"); state.full(params, samples).expect("transcribe");
let dur = t.elapsed().as_secs_f64(); let dur = t.elapsed().as_secs_f64();
if dur < best { best = dur; } if dur < best {
best = dur;
}
} }
let rtf = best / audio_secs; let rtf = best / audio_secs;
let speedup = baseline_dur.map(|b| b / best).unwrap_or(1.0); let speedup = baseline_dur.map(|b| b / best).unwrap_or(1.0);
if baseline_dur.is_none() { baseline_dur = Some(best); } if baseline_dur.is_none() {
eprintln!("{:>9} | {:>6.2}s | {:>6.3} | {:>6.2}x", baseline_dur = Some(best);
n, best, rtf, speedup); }
eprintln!(
"{:>9} | {:>6.2}s | {:>6.3} | {:>6.2}x",
n, best, rtf, speedup
);
} }
} }