diff --git a/crates/audio/Cargo.toml b/crates/audio/Cargo.toml index 662a829..0584a65 100644 --- a/crates/audio/Cargo.toml +++ b/crates/audio/Cargo.toml @@ -6,3 +6,21 @@ description = "Audio capture (cpal), VAD, resampling (rubato), file decoding (sy [dependencies] kon-core = { path = "../core" } + +# Microphone capture +cpal = "0.17" + +# Voice activity detection (Silero VAD V5) +voice_activity_detector = "0.2" + +# High-quality resampling (sinc interpolation) +rubato = "0.15" + +# WAV file I/O +hound = "3.5" + +# Audio file decoding (mp3, aac, flac, wav, ogg, etc.) +symphonia = { version = "0.5", features = ["mp3", "aac", "flac", "pcm", "vorbis", "wav", "ogg", "isomp4"] } + +# Async runtime for threading +tokio = { version = "1", features = ["rt", "sync"] } diff --git a/crates/audio/src/capture.rs b/crates/audio/src/capture.rs new file mode 100644 index 0000000..94abcc8 --- /dev/null +++ b/crates/audio/src/capture.rs @@ -0,0 +1,75 @@ +use std::sync::mpsc; + +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; + +use kon_core::error::{KonError, Result}; + +/// A chunk of captured audio from the microphone. +pub struct AudioChunk { + pub samples: Vec, + pub sample_rate: u32, + pub channels: u16, +} + +/// Manages microphone capture via cpal. +/// Call `start()` to begin capturing, which returns a receiver for audio chunks. +/// Call `stop()` to end the stream. +pub struct MicrophoneCapture { + stream: Option, +} + +impl MicrophoneCapture { + /// Start capturing audio from the default input device. + /// Returns a receiver that yields AudioChunks as they arrive. + pub fn start() -> Result<(Self, mpsc::Receiver)> { + let host = cpal::default_host(); + let device = host.default_input_device().ok_or_else(|| { + KonError::AudioCaptureFailed("No input device found".into()) + })?; + + let config = device.default_input_config().map_err(|e| { + KonError::AudioCaptureFailed(format!("No input config: {e}")) + })?; + + let sample_rate = config.sample_rate(); + let channels = config.channels() as u16; + + let (tx, rx) = mpsc::channel::(); + + let stream = device + .build_input_stream( + &config.into(), + move |data: &[f32], _info: &cpal::InputCallbackInfo| { + let _ = tx.send(AudioChunk { + samples: data.to_vec(), + sample_rate, + channels, + }); + }, + |err| eprintln!("audio capture error: {err}"), + None, + ) + .map_err(|e| { + KonError::AudioCaptureFailed(format!("Build stream failed: {e}")) + })?; + + stream.play().map_err(|e| { + KonError::AudioCaptureFailed(format!("Stream play failed: {e}")) + })?; + + Ok((Self { stream: Some(stream) }, rx)) + } + + /// Stop capturing audio. + pub fn stop(&mut self) { + if let Some(stream) = self.stream.take() { + let _ = stream.pause(); + } + } +} + +impl Drop for MicrophoneCapture { + fn drop(&mut self) { + self.stop(); + } +} diff --git a/crates/audio/src/concurrency.rs b/crates/audio/src/concurrency.rs new file mode 100644 index 0000000..ce3d4e2 --- /dev/null +++ b/crates/audio/src/concurrency.rs @@ -0,0 +1,19 @@ +use std::path::Path; + +use kon_core::error::Result; +use kon_core::types::AudioSamples; + +use crate::decode::decode_audio_file; +use crate::resample::resample_to_16khz; + +/// Decode and resample an audio file on a blocking thread. +/// Returns 16kHz mono AudioSamples ready for transcription. +pub async fn decode_and_resample(path: &Path) -> Result { + let path = path.to_path_buf(); + tokio::task::spawn_blocking(move || { + let audio = decode_audio_file(&path)?; + resample_to_16khz(&audio) + }) + .await + .map_err(|e| kon_core::error::KonError::AudioDecodeFailed(format!("Task join error: {e}")))? +} diff --git a/crates/audio/src/decode.rs b/crates/audio/src/decode.rs new file mode 100644 index 0000000..5d08285 --- /dev/null +++ b/crates/audio/src/decode.rs @@ -0,0 +1,104 @@ +use std::fs::File; +use std::path::Path; + +use symphonia::core::audio::SampleBuffer; +use symphonia::core::codecs::DecoderOptions; +use symphonia::core::formats::FormatOptions; +use symphonia::core::io::MediaSourceStream; +use symphonia::core::meta::MetadataOptions; +use symphonia::core::probe::Hint; + +use kon_core::error::{KonError, Result}; +use kon_core::types::AudioSamples; + +/// Decode an audio file to mono f32 PCM samples. +/// Supports all formats symphonia handles: mp3, aac, flac, wav, ogg, etc. +pub fn decode_audio_file(path: &Path) -> Result { + let file = File::open(path) + .map_err(|e| KonError::AudioDecodeFailed(format!("Cannot open file: {e}")))?; + let mss = MediaSourceStream::new(Box::new(file), Default::default()); + + let mut hint = Hint::new(); + if let Some(ext) = path.extension().and_then(|e| e.to_str()) { + hint.with_extension(ext); + } + + let probed = symphonia::default::get_probe() + .format(&hint, mss, &FormatOptions::default(), &MetadataOptions::default()) + .map_err(|e| KonError::AudioDecodeFailed(format!("Unsupported format: {e}")))?; + + let mut format = probed.format; + + let track = format + .default_track() + .ok_or_else(|| KonError::AudioDecodeFailed("No audio track found".into()))?; + let sample_rate = track + .codec_params + .sample_rate + .ok_or_else(|| KonError::AudioDecodeFailed("Unknown sample rate".into()))?; + + if sample_rate == 0 { + return Err(KonError::AudioDecodeFailed("Invalid sample rate: 0".into())); + } + + let track_id = track.id; + + let mut decoder = symphonia::default::get_codecs() + .make(&track.codec_params, &DecoderOptions::default()) + .map_err(|e| KonError::AudioDecodeFailed(format!("Codec error: {e}")))?; + + let mut samples: Vec = Vec::new(); + let mut decode_errors = 0u32; + + loop { + let packet = match format.next_packet() { + Ok(p) => p, + Err(symphonia::core::errors::Error::IoError(ref e)) + if e.kind() == std::io::ErrorKind::UnexpectedEof => + { + break; + } + Err(symphonia::core::errors::Error::ResetRequired) => break, + Err(_) => break, + }; + + if packet.track_id() != track_id { + continue; + } + + let decoded = match decoder.decode(&packet) { + Ok(d) => d, + Err(_) => { + decode_errors += 1; + continue; + } + }; + + let spec = *decoded.spec(); + let channels = spec.channels.count(); + let mut sample_buf = + SampleBuffer::::new(decoded.capacity() as u64, spec); + sample_buf.copy_interleaved_ref(decoded); + + let buf = sample_buf.samples(); + if channels == 1 { + samples.extend_from_slice(buf); + } else { + for chunk in buf.chunks(channels) { + let sum: f32 = chunk.iter().sum(); + samples.push(sum / channels as f32); + } + } + } + + if samples.is_empty() { + if decode_errors > 0 { + return Err(KonError::AudioDecodeFailed(format!( + "No audio decoded ({decode_errors} packets failed — file may be corrupt)" + ))); + } + return Err(KonError::AudioDecodeFailed("No audio data decoded".into())); + } + + Ok(AudioSamples::new(samples, sample_rate, 1)) +} diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index b0c13f3..385d7b4 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -1,2 +1,13 @@ -// kon-audio: Microphone capture, voice activity detection, resampling, -// file decoding, and WAV I/O. +pub mod capture; +pub mod concurrency; +pub mod decode; +pub mod resample; +pub mod vad; +pub mod wav; + +pub use capture::{AudioChunk, MicrophoneCapture}; +pub use concurrency::decode_and_resample; +pub use decode::decode_audio_file; +pub use resample::resample_to_16khz; +pub use vad::SpeechDetector; +pub use wav::{read_wav, write_wav}; diff --git a/crates/audio/src/resample.rs b/crates/audio/src/resample.rs new file mode 100644 index 0000000..97a0d22 --- /dev/null +++ b/crates/audio/src/resample.rs @@ -0,0 +1,105 @@ +use rubato::{SincFixedIn, SincInterpolationParameters, SincInterpolationType, Resampler, WindowFunction}; + +use kon_core::constants::WHISPER_SAMPLE_RATE; +use kon_core::error::{KonError, Result}; +use kon_core::types::AudioSamples; + +/// Resample audio to 16kHz mono using sinc interpolation (rubato). +/// Returns a new AudioSamples at the target sample rate. +pub fn resample_to_16khz(audio: &AudioSamples) -> Result { + let from_rate = audio.sample_rate(); + let target_rate = WHISPER_SAMPLE_RATE; + + if from_rate == target_rate { + return Ok(AudioSamples::mono_16khz(audio.samples().to_vec())); + } + + if from_rate == 0 { + return Err(KonError::AudioDecodeFailed( + "Cannot resample: source rate is 0".into(), + )); + } + + let ratio = target_rate as f64 / from_rate as f64; + let chunk_size = 1024; + + let params = SincInterpolationParameters { + sinc_len: 256, + f_cutoff: 0.95, + oversampling_factor: 128, + interpolation: SincInterpolationType::Cubic, + window: WindowFunction::Blackman, + }; + + let mut resampler = SincFixedIn::::new( + ratio, + 1.1, + params, + chunk_size, + 1, // mono + ) + .map_err(|e| { + KonError::AudioDecodeFailed(format!("Resampler init failed: {e}")) + })?; + + let samples = audio.samples(); + let mut output_samples: Vec = Vec::new(); + + let mut offset = 0; + while offset < samples.len() { + let end = (offset + chunk_size).min(samples.len()); + let mut chunk = samples[offset..end].to_vec(); + + if chunk.len() < chunk_size { + chunk.resize(chunk_size, 0.0); + } + + let input = vec![chunk]; + let result = resampler.process(&input, None).map_err(|e| { + KonError::AudioDecodeFailed(format!("Resample failed: {e}")) + })?; + + if !result.is_empty() && !result[0].is_empty() { + output_samples.extend_from_slice(&result[0]); + } + + offset += chunk_size; + } + + // Trim to expected length (padding may have added extra samples) + let expected_len = (samples.len() as f64 * ratio) as usize; + output_samples.truncate(expected_len); + + Ok(AudioSamples::mono_16khz(output_samples)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resample_passthrough_at_16khz() { + let input = AudioSamples::mono_16khz(vec![0.1, 0.2, 0.3]); + let output = resample_to_16khz(&input).unwrap(); + assert_eq!(output.sample_rate(), 16000); + assert_eq!(output.samples().len(), 3); + } + + #[test] + fn resample_preserves_approximate_duration() { + let rate = 48000; + let duration_secs = 1.0; + let num_samples = (rate as f64 * duration_secs) as usize; + let samples: Vec = + (0..num_samples).map(|i| (i as f32 * 0.001).sin()).collect(); + + let input = AudioSamples::new(samples, rate, 1); + let output = resample_to_16khz(&input).unwrap(); + + let output_duration = output.samples().len() as f64 / 16000.0; + assert!( + (output_duration - duration_secs).abs() < 0.1, + "Duration mismatch: expected ~{duration_secs}s, got {output_duration}s" + ); + } +} diff --git a/crates/audio/src/vad.rs b/crates/audio/src/vad.rs new file mode 100644 index 0000000..437e05d --- /dev/null +++ b/crates/audio/src/vad.rs @@ -0,0 +1,50 @@ +use voice_activity_detector::VoiceActivityDetector; + +use kon_core::constants::{VAD_SPEECH_THRESHOLD, WHISPER_SAMPLE_RATE}; +use kon_core::error::{KonError, Result}; + +/// VAD chunk size for 16kHz audio (required by Silero VAD V5). +const VAD_CHUNK_SIZE: usize = 512; + +/// Wraps Silero VAD V5 for speech detection. +pub struct SpeechDetector { + vad: VoiceActivityDetector, + threshold: f64, +} + +impl SpeechDetector { + /// Create a new speech detector with the default threshold. + pub fn new() -> Result { + let vad = VoiceActivityDetector::builder() + .sample_rate(WHISPER_SAMPLE_RATE as i64) + .chunk_size(VAD_CHUNK_SIZE) + .build() + .map_err(|e| KonError::AudioCaptureFailed(format!("VAD init failed: {e}")))?; + + Ok(Self { + vad, + threshold: VAD_SPEECH_THRESHOLD, + }) + } + + /// Predict whether a chunk of 16kHz f32 audio contains speech. + /// Returns the speech probability (0.0 to 1.0). + pub fn predict(&mut self, samples: &[f32]) -> f32 { + self.vad.predict(samples.iter().copied()) + } + + /// Returns true if the chunk contains speech above the threshold. + pub fn is_speech(&mut self, samples: &[f32]) -> bool { + self.predict(samples) > self.threshold as f32 + } + + /// Reset the internal VAD state (call between recording sessions). + pub fn reset(&mut self) { + self.vad.reset(); + } + + /// The required chunk size for this VAD instance. + pub fn chunk_size(&self) -> usize { + VAD_CHUNK_SIZE + } +} diff --git a/crates/audio/src/wav.rs b/crates/audio/src/wav.rs new file mode 100644 index 0000000..eed82a7 --- /dev/null +++ b/crates/audio/src/wav.rs @@ -0,0 +1,83 @@ +use std::path::Path; + +use kon_core::error::{KonError, Result}; +use kon_core::types::AudioSamples; + +/// Write f32 PCM samples to a 16-bit WAV file. +pub fn write_wav(path: &Path, audio: &AudioSamples) -> Result<()> { + let spec = hound::WavSpec { + channels: audio.channels(), + sample_rate: audio.sample_rate(), + bits_per_sample: 16, + sample_format: hound::SampleFormat::Int, + }; + + let mut writer = hound::WavWriter::create(path, spec) + .map_err(|e| KonError::Io(std::io::Error::other(format!("WAV create failed: {e}"))))?; + + for &sample in audio.samples() { + let clamped = sample.clamp(-1.0, 1.0); + let int_sample = (clamped * i16::MAX as f32) as i16; + writer + .write_sample(int_sample) + .map_err(|e| KonError::Io(std::io::Error::other(format!("WAV write failed: {e}"))))?; + } + + writer + .finalize() + .map_err(|e| KonError::Io(std::io::Error::other(format!("WAV finalize failed: {e}"))))?; + + Ok(()) +} + +/// Read a WAV file to f32 PCM AudioSamples. +pub fn read_wav(path: &Path) -> Result { + let reader = hound::WavReader::open(path) + .map_err(|e| KonError::AudioDecodeFailed(format!("WAV open failed: {e}")))?; + + let spec = reader.spec(); + let sample_rate = spec.sample_rate; + let channels = spec.channels; + + let samples: Vec = match spec.sample_format { + hound::SampleFormat::Int => reader + .into_samples::() + .filter_map(|s| s.ok()) + .map(|s| s as f32 / (1 << (spec.bits_per_sample - 1)) as f32) + .collect(), + hound::SampleFormat::Float => reader + .into_samples::() + .filter_map(|s| s.ok()) + .collect(), + }; + + Ok(AudioSamples::new(samples, sample_rate, channels)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wav_roundtrip() { + let temp_dir = std::env::temp_dir(); + let path = temp_dir.join("kon_test_roundtrip.wav"); + + let original = AudioSamples::mono_16khz(vec![0.0, 0.5, -0.5, 0.25, -0.25]); + write_wav(&path, &original).unwrap(); + + let loaded = read_wav(&path).unwrap(); + assert_eq!(loaded.sample_rate(), 16000); + assert_eq!(loaded.samples().len(), 5); + + // 16-bit quantisation introduces small error + for (a, b) in original.samples().iter().zip(loaded.samples().iter()) { + assert!( + (a - b).abs() < 0.001, + "Sample mismatch: original={a}, loaded={b}" + ); + } + + std::fs::remove_file(&path).ok(); + } +}