feat(llm): wire Phase 3 local LLM runtime via llama-cpp-2
kon-llm now owns a real LlamaBackend + LlamaModel, with three Qwen3 tiers (1.7B Q4, 4B-Instruct-2507 Q4, 14B Q5) selectable per hardware. Downloads are resumable with SHA-256 verification and stored under ~/.kon/models/llm. Engine exposes three high-level surfaces — all greedy/temp-0, GBNF-constrained where output shape matters: - cleanup_text (prompt-injection-hardened system prompt; profile terms appended as "preserve these spellings" suffix) - decompose_task (3–7 micro-steps, constrained JSON array) - extract_tasks (optional-array; empty when no explicit commitments) post_process_segments now takes an Option<&LlmEngine> and, when loaded and format_mode != Raw, joins segments → cleanup → replaces segments with the cleaned text (first segment span). Rule-based path still runs first; LLM errors log and keep rule-based output. Tauri commands: recommend_llm_tier, check_llm_model, download_llm_model, load_llm_model, unload_llm_model, delete_llm_model, get_llm_status, cleanup_transcript_text_cmd, extract_tasks_from_transcript_cmd, decompose_and_store (LLM-backed subtasks). Settings: AI tier toggle (off / cleanup / tasks), model picker with downloaded/loaded status, download progress events via kon:llm-download-progress. Dictation: ensureLlmModelLoaded on mount, cleanupTranscriptIfEnabled after stop when tier != off and format_mode != Raw, LLM task extraction when tier=tasks (regex fallback on failure). Interim: both llama-cpp-sys-2 and whisper-rs-sys statically link their own ggml, so src-tauri/build.rs emits -Wl,--allow-multiple-definition on Linux. Replace with a system-ggml shared-lib setup as a follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -68,6 +68,7 @@
|
||||
}
|
||||
|
||||
await checkModelState();
|
||||
await ensureLlmModelLoaded();
|
||||
});
|
||||
|
||||
onDestroy(() => {
|
||||
@@ -223,6 +224,18 @@
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureLlmModelLoaded() {
|
||||
if (!tauriRuntimeAvailable || settings.aiTier === "off" || !settings.llmModelId) return;
|
||||
try {
|
||||
const status = await invoke("check_llm_model", { modelId: settings.llmModelId });
|
||||
if (status?.downloaded && !status.loaded) {
|
||||
await invoke("load_llm_model", { modelId: settings.llmModelId });
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn("ensureLlmModelLoaded failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadModel() {
|
||||
if (!tauriRuntimeAvailable) {
|
||||
error = browserPreviewMessage;
|
||||
@@ -386,6 +399,56 @@
|
||||
statusChannel = null;
|
||||
}
|
||||
|
||||
function replaceSegmentsWithCleanedText(cleanedText) {
|
||||
if (!cleanedText.trim()) return;
|
||||
if (segments.length === 0) {
|
||||
segments = [{
|
||||
start: 0,
|
||||
end: Math.max((Date.now() - startTime) / 1000, 0),
|
||||
text: cleanedText,
|
||||
}];
|
||||
return;
|
||||
}
|
||||
segments = [{
|
||||
start: segments[0].start,
|
||||
end: segments[segments.length - 1].end,
|
||||
text: cleanedText,
|
||||
}];
|
||||
}
|
||||
|
||||
async function cleanupTranscriptIfEnabled(text) {
|
||||
if (!text.trim()) return text;
|
||||
if (settings.aiTier === "off" || settings.formatMode === "Raw") return text;
|
||||
|
||||
const llmLoaded = await invoke("get_llm_status").catch(() => false);
|
||||
if (!llmLoaded) return text;
|
||||
|
||||
try {
|
||||
const cleaned = await invoke("cleanup_transcript_text_cmd", {
|
||||
transcript: text,
|
||||
profileId: profilesStore.activeProfileId,
|
||||
});
|
||||
return cleaned?.trim() ? cleaned.trim() : text;
|
||||
} catch (err) {
|
||||
console.warn("LLM cleanup failed, keeping existing transcript", err);
|
||||
return text;
|
||||
}
|
||||
}
|
||||
|
||||
async function extractTasksForTranscript(text) {
|
||||
const llmLoaded = await invoke("get_llm_status").catch(() => false);
|
||||
if (settings.aiTier === "tasks" && llmLoaded) {
|
||||
try {
|
||||
const items = await invoke("extract_tasks_from_transcript_cmd", { transcript: text });
|
||||
return items.map((taskText) => ({ text: taskText }));
|
||||
} catch (err) {
|
||||
console.warn("LLM extract_tasks failed, falling back to regex", err);
|
||||
}
|
||||
}
|
||||
|
||||
return extractTasks(text);
|
||||
}
|
||||
|
||||
async function waitForResultDrain(previousActivityAt = 0) {
|
||||
const firstMessageDeadline = Date.now() + 400;
|
||||
while (Date.now() < firstMessageDeadline) {
|
||||
@@ -410,6 +473,12 @@
|
||||
|
||||
async function finaliseTranscription(audioPath = null) {
|
||||
if (transcript.trim()) {
|
||||
const cleanedTranscript = await cleanupTranscriptIfEnabled(transcript);
|
||||
if (cleanedTranscript !== transcript) {
|
||||
transcript = cleanedTranscript;
|
||||
replaceSegmentsWithCleanedText(cleanedTranscript);
|
||||
}
|
||||
|
||||
if (settings.autoCopy) {
|
||||
navigator.clipboard.writeText(transcript).catch(() => {
|
||||
invoke("copy_to_clipboard", { text: transcript }).catch(() => {});
|
||||
@@ -434,7 +503,7 @@
|
||||
});
|
||||
|
||||
// Extract tasks from transcript
|
||||
const extracted = extractTasks(transcript);
|
||||
const extracted = await extractTasksForTranscript(transcript);
|
||||
extractedCount = extracted.length;
|
||||
for (const item of extracted) {
|
||||
addTask({
|
||||
@@ -516,12 +585,12 @@
|
||||
});
|
||||
}
|
||||
|
||||
function manualExtractTasks() {
|
||||
async function manualExtractTasks() {
|
||||
if (!transcript.trim() || aiProcessing) return;
|
||||
aiProcessing = true;
|
||||
aiStatus = "Extracting tasks...";
|
||||
try {
|
||||
const extracted = extractTasks(transcript);
|
||||
const extracted = await extractTasksForTranscript(transcript);
|
||||
for (const item of extracted) {
|
||||
addTask({ text: item.text, bucket: "inbox" });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user