feat(whisper): feed profile_terms into initial_prompt at decode time
Previously profile_terms only reached the LLM cleanup stage as the
dictionary_terms suffix. Whisper decoded without any vocabulary hint, so
domain names ('Wren', 'CORBEL') were misspelled on the first pass and the
LLM had to guess at the correction.
build_initial_prompt (src-tauri/src/commands/mod.rs) collapses caller /
profile / terms into a single Whisper prompt:
caller_prompt > profile_prompt + "Vocabulary: <terms>." > None
transcribe_pcm, transcribe_file, and start_live_transcription_session all
route through the helper, so the three paths stay in lockstep.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -12,6 +12,7 @@ use serde::{Deserialize, Serialize};
|
|||||||
use tauri::ipc::Channel;
|
use tauri::ipc::Channel;
|
||||||
|
|
||||||
use crate::commands::audio::persist_audio_samples;
|
use crate::commands::audio::persist_audio_samples;
|
||||||
|
use crate::commands::build_initial_prompt;
|
||||||
use crate::commands::models::{default_model_id_for_engine, ensure_model_loaded};
|
use crate::commands::models::{default_model_id_for_engine, ensure_model_loaded};
|
||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
use kon_ai_formatting::{post_process_segments, FormatMode, PostProcessOptions};
|
use kon_ai_formatting::{post_process_segments, FormatMode, PostProcessOptions};
|
||||||
@@ -214,16 +215,13 @@ pub async fn start_live_transcription_session(
|
|||||||
|
|
||||||
// Collapse the effective initial_prompt on the struct so downstream
|
// Collapse the effective initial_prompt on the struct so downstream
|
||||||
// `TranscriptionOptions` construction (see `maybe_dispatch_chunk`) picks
|
// `TranscriptionOptions` construction (see `maybe_dispatch_chunk`) picks
|
||||||
// up profile fallback without further plumbing.
|
// up profile fallback + vocabulary injection without further plumbing.
|
||||||
let effective_prompt = match config.initial_prompt.as_deref() {
|
let request_prompt = config.initial_prompt.clone().unwrap_or_default();
|
||||||
Some(p) if !p.is_empty() => p.to_string(),
|
config.initial_prompt = build_initial_prompt(
|
||||||
_ => profile.initial_prompt.clone(),
|
&request_prompt,
|
||||||
};
|
&profile.initial_prompt,
|
||||||
config.initial_prompt = if effective_prompt.is_empty() {
|
&profile_terms,
|
||||||
None
|
);
|
||||||
} else {
|
|
||||||
Some(effective_prompt)
|
|
||||||
};
|
|
||||||
|
|
||||||
let model_id = config
|
let model_id = config
|
||||||
.model_id
|
.model_id
|
||||||
|
|||||||
@@ -12,3 +12,93 @@ pub mod transcription;
|
|||||||
pub mod transcripts;
|
pub mod transcripts;
|
||||||
pub mod update;
|
pub mod update;
|
||||||
pub mod windows;
|
pub mod windows;
|
||||||
|
|
||||||
|
/// Build the Whisper `initial_prompt` for a transcription request.
|
||||||
|
///
|
||||||
|
/// Precedence:
|
||||||
|
/// 1. Caller-supplied `request_prompt` (non-empty wins outright — the caller
|
||||||
|
/// has already made the decision).
|
||||||
|
/// 2. Profile's stored prompt + profile terms (joined: the prompt frames the
|
||||||
|
/// task, the vocabulary biases recognition toward domain terms).
|
||||||
|
/// 3. Profile prompt alone, or vocabulary alone.
|
||||||
|
/// 4. `None` if nothing is set.
|
||||||
|
///
|
||||||
|
/// Feeding `profile_terms` into `initial_prompt` (the OpenWhispr pattern) lets
|
||||||
|
/// whisper.cpp bias its decoder toward the correct spelling of user-specific
|
||||||
|
/// vocabulary at decode time, before any LLM cleanup pass.
|
||||||
|
pub fn build_initial_prompt(
|
||||||
|
request_prompt: &str,
|
||||||
|
profile_prompt: &str,
|
||||||
|
profile_terms: &[String],
|
||||||
|
) -> Option<String> {
|
||||||
|
let trimmed_request = request_prompt.trim();
|
||||||
|
if !trimmed_request.is_empty() {
|
||||||
|
return Some(trimmed_request.to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
let trimmed_profile = profile_prompt.trim();
|
||||||
|
let terms_list = profile_terms
|
||||||
|
.iter()
|
||||||
|
.map(|term| term.trim())
|
||||||
|
.filter(|term| !term.is_empty())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ");
|
||||||
|
|
||||||
|
match (trimmed_profile.is_empty(), terms_list.is_empty()) {
|
||||||
|
(true, true) => None,
|
||||||
|
(false, true) => Some(trimmed_profile.to_string()),
|
||||||
|
(true, false) => Some(format!("Vocabulary: {terms_list}.")),
|
||||||
|
(false, false) => Some(format!("{trimmed_profile} Vocabulary: {terms_list}.")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::build_initial_prompt;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn caller_prompt_overrides_everything() {
|
||||||
|
let got = build_initial_prompt(
|
||||||
|
"caller wins",
|
||||||
|
"profile prompt",
|
||||||
|
&["Wren".into(), "CORBEL".into()],
|
||||||
|
);
|
||||||
|
assert_eq!(got.as_deref(), Some("caller wins"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn profile_prompt_and_terms_are_joined() {
|
||||||
|
let got = build_initial_prompt(
|
||||||
|
"",
|
||||||
|
"You are a meeting notes assistant.",
|
||||||
|
&["Wren".into(), "CORBEL".into()],
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
got.as_deref(),
|
||||||
|
Some("You are a meeting notes assistant. Vocabulary: Wren, CORBEL."),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn terms_only_produces_vocabulary_sentence() {
|
||||||
|
let got = build_initial_prompt("", "", &["Wren".into(), "CORBEL".into()]);
|
||||||
|
assert_eq!(got.as_deref(), Some("Vocabulary: Wren, CORBEL."));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn profile_prompt_alone_is_passed_through() {
|
||||||
|
let got = build_initial_prompt("", "Be concise.", &[]);
|
||||||
|
assert_eq!(got.as_deref(), Some("Be concise."));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn all_empty_returns_none() {
|
||||||
|
assert_eq!(build_initial_prompt("", "", &[]), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn whitespace_only_terms_are_skipped() {
|
||||||
|
let got = build_initial_prompt("", "", &[" ".into(), "Wren".into(), "".into()]);
|
||||||
|
assert_eq!(got.as_deref(), Some("Vocabulary: Wren."));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use tauri::Emitter;
|
use tauri::Emitter;
|
||||||
|
|
||||||
|
use crate::commands::build_initial_prompt;
|
||||||
use crate::commands::models::{default_model_id_for_engine, ensure_model_loaded};
|
use crate::commands::models::{default_model_id_for_engine, ensure_model_loaded};
|
||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
use kon_ai_formatting::{post_process_segments, FormatMode, PostProcessOptions};
|
use kon_ai_formatting::{post_process_segments, FormatMode, PostProcessOptions};
|
||||||
@@ -165,20 +166,14 @@ pub async fn transcribe_pcm(
|
|||||||
.map(|t| t.term)
|
.map(|t| t.term)
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let effective_prompt = if !initial_prompt.is_empty() {
|
|
||||||
initial_prompt
|
|
||||||
} else {
|
|
||||||
profile.initial_prompt.clone()
|
|
||||||
};
|
|
||||||
|
|
||||||
let engine = state.whisper_engine.clone();
|
let engine = state.whisper_engine.clone();
|
||||||
let options = TranscriptionOptions {
|
let options = TranscriptionOptions {
|
||||||
language: Some(language),
|
language: Some(language),
|
||||||
initial_prompt: if effective_prompt.is_empty() {
|
initial_prompt: build_initial_prompt(
|
||||||
None
|
&initial_prompt,
|
||||||
} else {
|
&profile.initial_prompt,
|
||||||
Some(effective_prompt)
|
&profile_terms,
|
||||||
},
|
),
|
||||||
};
|
};
|
||||||
|
|
||||||
let timed = tokio::task::spawn_blocking(move || {
|
let timed = tokio::task::spawn_blocking(move || {
|
||||||
@@ -252,12 +247,6 @@ pub async fn transcribe_file(
|
|||||||
.map(|t| t.term)
|
.map(|t| t.term)
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let effective_prompt = if !initial_prompt.is_empty() {
|
|
||||||
initial_prompt
|
|
||||||
} else {
|
|
||||||
profile.initial_prompt.clone()
|
|
||||||
};
|
|
||||||
|
|
||||||
let engine_name = engine.unwrap_or_else(|| "whisper".to_string());
|
let engine_name = engine.unwrap_or_else(|| "whisper".to_string());
|
||||||
let model_id =
|
let model_id =
|
||||||
model_id.unwrap_or_else(|| default_model_id_for_engine(&engine_name).to_string());
|
model_id.unwrap_or_else(|| default_model_id_for_engine(&engine_name).to_string());
|
||||||
@@ -266,11 +255,11 @@ pub async fn transcribe_file(
|
|||||||
let engine = pick_engine(&state, &engine_name)?;
|
let engine = pick_engine(&state, &engine_name)?;
|
||||||
let options = TranscriptionOptions {
|
let options = TranscriptionOptions {
|
||||||
language: Some(language),
|
language: Some(language),
|
||||||
initial_prompt: if effective_prompt.is_empty() {
|
initial_prompt: build_initial_prompt(
|
||||||
None
|
&initial_prompt,
|
||||||
} else {
|
&profile.initial_prompt,
|
||||||
Some(effective_prompt)
|
&profile_terms,
|
||||||
},
|
),
|
||||||
};
|
};
|
||||||
let engine_name_for_worker = engine_name.clone();
|
let engine_name_for_worker = engine_name.clone();
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user