feat(llm): wire Phase 3 local LLM runtime via llama-cpp-2

kon-llm now owns a real LlamaBackend + LlamaModel, with three Qwen3 tiers
(1.7B Q4, 4B-Instruct-2507 Q4, 14B Q5) selectable per hardware. Downloads
are resumable with SHA-256 verification and stored under ~/.kon/models/llm.

Engine exposes three high-level surfaces — all greedy/temp-0, GBNF-constrained
where output shape matters:
- cleanup_text (prompt-injection-hardened system prompt; profile terms
  appended as "preserve these spellings" suffix)
- decompose_task (3–7 micro-steps, constrained JSON array)
- extract_tasks (optional-array; empty when no explicit commitments)

post_process_segments now takes an Option<&LlmEngine> and, when loaded and
format_mode != Raw, joins segments → cleanup → replaces segments with the
cleaned text (first segment span). Rule-based path still runs first; LLM
errors log and keep rule-based output.

Tauri commands: recommend_llm_tier, check_llm_model, download_llm_model,
load_llm_model, unload_llm_model, delete_llm_model, get_llm_status,
cleanup_transcript_text_cmd, extract_tasks_from_transcript_cmd,
decompose_and_store (LLM-backed subtasks).

Settings: AI tier toggle (off / cleanup / tasks), model picker with
downloaded/loaded status, download progress events via
kon:llm-download-progress.

Dictation: ensureLlmModelLoaded on mount, cleanupTranscriptIfEnabled after
stop when tier != off and format_mode != Raw, LLM task extraction when
tier=tasks (regex fallback on failure).

Interim: both llama-cpp-sys-2 and whisper-rs-sys statically link their own
ggml, so src-tauri/build.rs emits -Wl,--allow-multiple-definition on Linux.
Replace with a system-ggml shared-lib setup as a follow-up.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/lib/pages/DictationPage.svelte

@@ -68,6 +68,7 @@
     }
 
     await checkModelState();
+    await ensureLlmModelLoaded();
   });
 
   onDestroy(() => {
@@ -223,6 +224,18 @@
     }
   }
 
+  async function ensureLlmModelLoaded() {
+    if (!tauriRuntimeAvailable || settings.aiTier === "off" || !settings.llmModelId) return;
+    try {
+      const status = await invoke("check_llm_model", { modelId: settings.llmModelId });
+      if (status?.downloaded && !status.loaded) {
+        await invoke("load_llm_model", { modelId: settings.llmModelId });
+      }
+    } catch (err) {
+      console.warn("ensureLlmModelLoaded failed", err);
+    }
+  }
+
   async function loadModel() {
     if (!tauriRuntimeAvailable) {
       error = browserPreviewMessage;
@@ -386,6 +399,56 @@
     statusChannel = null;
   }
 
+  function replaceSegmentsWithCleanedText(cleanedText) {
+    if (!cleanedText.trim()) return;
+    if (segments.length === 0) {
+      segments = [{
+        start: 0,
+        end: Math.max((Date.now() - startTime) / 1000, 0),
+        text: cleanedText,
+      }];
+      return;
+    }
+    segments = [{
+      start: segments[0].start,
+      end: segments[segments.length - 1].end,
+      text: cleanedText,
+    }];
+  }
+
+  async function cleanupTranscriptIfEnabled(text) {
+    if (!text.trim()) return text;
+    if (settings.aiTier === "off" || settings.formatMode === "Raw") return text;
+
+    const llmLoaded = await invoke("get_llm_status").catch(() => false);
+    if (!llmLoaded) return text;
+
+    try {
+      const cleaned = await invoke("cleanup_transcript_text_cmd", {
+        transcript: text,
+        profileId: profilesStore.activeProfileId,
+      });
+      return cleaned?.trim() ? cleaned.trim() : text;
+    } catch (err) {
+      console.warn("LLM cleanup failed, keeping existing transcript", err);
+      return text;
+    }
+  }
+
+  async function extractTasksForTranscript(text) {
+    const llmLoaded = await invoke("get_llm_status").catch(() => false);
+    if (settings.aiTier === "tasks" && llmLoaded) {
+      try {
+        const items = await invoke("extract_tasks_from_transcript_cmd", { transcript: text });
+        return items.map((taskText) => ({ text: taskText }));
+      } catch (err) {
+        console.warn("LLM extract_tasks failed, falling back to regex", err);
+      }
+    }
+
+    return extractTasks(text);
+  }
+
   async function waitForResultDrain(previousActivityAt = 0) {
     const firstMessageDeadline = Date.now() + 400;
     while (Date.now() < firstMessageDeadline) {
@@ -410,6 +473,12 @@
 
   async function finaliseTranscription(audioPath = null) {
     if (transcript.trim()) {
+      const cleanedTranscript = await cleanupTranscriptIfEnabled(transcript);
+      if (cleanedTranscript !== transcript) {
+        transcript = cleanedTranscript;
+        replaceSegmentsWithCleanedText(cleanedTranscript);
+      }
+
       if (settings.autoCopy) {
         navigator.clipboard.writeText(transcript).catch(() => {
           invoke("copy_to_clipboard", { text: transcript }).catch(() => {});
@@ -434,7 +503,7 @@
       });
 
       // Extract tasks from transcript
-      const extracted = extractTasks(transcript);
+      const extracted = await extractTasksForTranscript(transcript);
       extractedCount = extracted.length;
       for (const item of extracted) {
         addTask({
@@ -516,12 +585,12 @@
     });
   }
 
-  function manualExtractTasks() {
+  async function manualExtractTasks() {
     if (!transcript.trim() || aiProcessing) return;
     aiProcessing = true;
     aiStatus = "Extracting tasks...";
     try {
-      const extracted = extractTasks(transcript);
+      const extracted = await extractTasksForTranscript(transcript);
       for (const item of extracted) {
         addTask({ text: item.text, bucket: "inbox" });
       }

src/lib/pages/SettingsPage.svelte

@@ -25,8 +25,39 @@
   let downloadingModel = $state("");
   let downloadProgress = $state(0);
   let unlisten = null;
+  let unlistenLlm = null;
   let outputFolderEl = $state(null);
   let outputFolderWidth = $state(0);
+  let llmStatuses = $state({});
+  let llmStatus = $state("Checking...");
+  let llmDownloadingModel = $state("");
+  let llmDownloadProgress = $state(0);
+  let llmLoaded = $state(false);
+  let systemInfo = $state(null);
+
+  const LLM_MODELS = [
+    {
+      id: "qwen3_1_7b",
+      label: "Low",
+      subtitle: "Qwen3 1.7B",
+      fit: "8 GB RAM, CPU-heavy machines",
+      size: "~1.1 GB",
+    },
+    {
+      id: "qwen3_4b_instruct_2507",
+      label: "Default",
+      subtitle: "Qwen3 4B Instruct 2507",
+      fit: "16 GB RAM or 8 GB+ VRAM",
+      size: "~2.5 GB",
+    },
+    {
+      id: "qwen3_14b",
+      label: "High",
+      subtitle: "Qwen3 14B",
+      fit: "32 GB RAM or 16 GB+ VRAM",
+      size: "~10.5 GB",
+    },
+  ];
 
   // Parakeet state
   let parakeetStatus = $state("Checking...");
@@ -341,9 +372,154 @@
     runtimeCapabilities = await invoke("get_runtime_capabilities");
   }
 
+  function selectedLlmModelId() {
+    return settings.llmModelId || "qwen3_4b_instruct_2507";
+  }
+
+  function llmModelStatus(modelId) {
+    return llmStatuses[modelId] || null;
+  }
+
+  function llmModelDownloaded(modelId) {
+    return !!llmModelStatus(modelId)?.downloaded;
+  }
+
+  function llmModelLoaded(modelId) {
+    return !!llmModelStatus(modelId)?.loaded;
+  }
+
+  function llmHardwareWarning(modelId) {
+    const ramMb = systemInfo?.ram_mb || 0;
+    if (modelId === "qwen3_14b" && ramMb < 32768) {
+      return "High tier will swap heavily on this machine. Expect slow responses.";
+    }
+    if (modelId === "qwen3_4b_instruct_2507" && ramMb < 16384 && !hasGpuAcceleration()) {
+      return "Default tier is best with 16 GB RAM or a GPU-backed build.";
+    }
+    return "";
+  }
+
+  function llmTierAvailable(modelId) {
+    const ramMb = systemInfo?.ram_mb || 0;
+    if (modelId === "qwen3_14b") return ramMb >= 32768;
+    if (modelId === "qwen3_4b_instruct_2507") return ramMb >= 16384 || hasGpuAcceleration();
+    return true;
+  }
+
+  async function ensureRecommendedLlmTier() {
+    if (settings.llmModelId) return;
+    try {
+      settings.llmModelId = await invoke("recommend_llm_tier");
+    } catch {
+      settings.llmModelId = "qwen3_4b_instruct_2507";
+    }
+  }
+
+  async function refreshLlmStatus() {
+    const statuses = {};
+    for (const model of LLM_MODELS) {
+      try {
+        statuses[model.id] = await invoke("check_llm_model", { modelId: model.id });
+      } catch {}
+    }
+    llmStatuses = statuses;
+    llmLoaded = await invoke("get_llm_status").catch(() => false);
+    const selected = llmModelStatus(selectedLlmModelId());
+    llmStatus = selected?.loaded
+      ? `${selected.displayName} loaded`
+      : selected?.downloaded
+        ? `${selected.displayName} downloaded`
+        : "No LLM model downloaded";
+  }
+
+  async function downloadSelectedLlmModel() {
+    const modelId = selectedLlmModelId();
+    llmDownloadingModel = modelId;
+    llmDownloadProgress = 0;
+    llmStatus = "Downloading...";
+    try {
+      await invoke("download_llm_model", { modelId });
+      llmDownloadingModel = "";
+      await refreshLlmStatus();
+      llmStatus = "Download complete";
+    } catch (err) {
+      llmDownloadingModel = "";
+      llmStatus = typeof err === "string" ? err : "LLM download failed";
+    }
+  }
+
+  async function loadSelectedLlmModel() {
+    const modelId = selectedLlmModelId();
+    llmStatus = "Loading...";
+    try {
+      await invoke("load_llm_model", { modelId });
+      await refreshLlmStatus();
+    } catch (err) {
+      llmStatus = typeof err === "string" ? err : "LLM load failed";
+    }
+  }
+
+  async function unloadLlmModel() {
+    try {
+      await invoke("unload_llm_model");
+      await refreshLlmStatus();
+      llmStatus = "Model unloaded";
+    } catch (err) {
+      llmStatus = typeof err === "string" ? err : "LLM unload failed";
+    }
+  }
+
+  async function deleteSelectedLlmModel() {
+    const modelId = selectedLlmModelId();
+    try {
+      await invoke("delete_llm_model", { modelId });
+      await refreshLlmStatus();
+      llmStatus = "Downloaded model removed";
+    } catch (err) {
+      llmStatus = typeof err === "string" ? err : "Delete failed";
+    }
+  }
+
+  async function setAiTier(nextTier) {
+    settings.aiTier = nextTier;
+    if (nextTier === "off") {
+      await unloadLlmModel();
+      return;
+    }
+
+    if (llmModelDownloaded(selectedLlmModelId())) {
+      await loadSelectedLlmModel();
+    } else {
+      llmStatus = "Download a model to enable AI features.";
+    }
+  }
+
+  async function selectLlmModel(modelId) {
+    settings.llmModelId = modelId;
+    if (llmLoaded) {
+      await unloadLlmModel();
+    } else {
+      await refreshLlmStatus();
+    }
+    llmStatus = llmModelDownloaded(modelId)
+      ? "Selected model changed. Load it to enable AI features."
+      : "Selected model changed. Download it to enable AI features.";
+  }
+
+  async function toggleAiSection() {
+    openSection = openSection === 'ai' ? null : 'ai';
+    if (openSection === 'ai') {
+      await ensureRecommendedLlmTier();
+      await refreshLlmStatus();
+    }
+  }
+
   onMount(async () => {
     try {
       await refreshRuntimeCapabilities();
+      systemInfo = await invoke("probe_system").catch(() => null);
+      await ensureRecommendedLlmTier();
+      await refreshLlmStatus();
       const loaded = await invoke("check_engine");
       engineOk = loaded;
       engineStatus = loaded ? "Model loaded" : "No model loaded";
@@ -376,6 +552,11 @@
       downloadProgress = event.payload.percent || event.payload.progress || 0;
     });
 
+    unlistenLlm = await listen("kon:llm-download-progress", (event) => {
+      llmDownloadProgress = event.payload.percent || 0;
+      llmDownloadingModel = event.payload.modelId || llmDownloadingModel;
+    });
+
     unlistenParakeet = await listen("parakeet-download-progress", (event) => {
       parakeetProgress = event.payload.percent || event.payload.progress || 0;
     });
@@ -383,6 +564,7 @@
 
   onDestroy(() => {
     if (unlisten) unlisten();
+    if (unlistenLlm) unlistenLlm();
     if (unlistenParakeet) unlistenParakeet();
   });
 
@@ -1006,17 +1188,146 @@
       <div class="border-b border-border-subtle">
         <button
           class="flex items-center justify-between w-full py-4 px-5 text-left"
-          onclick={() => openSection = openSection === 'ai' ? null : 'ai'}
+          onclick={toggleAiSection}
         >
           <h3 class="font-display text-[18px] italic text-text">AI Assistant</h3>
           <span class="text-text-tertiary text-[16px] leading-none">{openSection === 'ai' ? '−' : '+'}</span>
         </button>
         {#if openSection === 'ai'}
           <div class="px-5 pb-5 animate-fade-in">
-            <p class="text-[11px] text-text-tertiary mb-4">Local LLM for smart task extraction, transcript cleanup, and formatting. Runs 100% offline.</p>
-            <div class="bg-bg-input rounded-lg px-3 py-2.5 border border-border-subtle">
-              <p class="text-[12px] text-text-secondary font-medium mb-1">Coming soon</p>
-              <p class="text-[11px] text-text-tertiary">AI-powered cleanup and smart extraction are being rebuilt with a faster engine. Task extraction currently uses rule-based matching, which runs automatically after each recording.</p>
+            <p class="text-[11px] text-text-tertiary mb-4">
+              Local LLM for transcript cleanup, smart task extraction, and task breakdown. Runs fully offline after the model is downloaded.
+            </p>
+
+            <div class="mb-5">
+              <p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Feature Tier</p>
+              <div class="flex flex-wrap gap-2">
+                <button
+                  type="button"
+                  class="px-3 py-2 rounded-lg border text-[12px] transition-colors
+                    {settings.aiTier === 'off'
+                      ? 'bg-bg-elevated border-accent text-text'
+                      : 'bg-bg-input border-border text-text-tertiary hover:text-text'}"
+                  onclick={() => setAiTier("off")}
+                >Off</button>
+                <button
+                  type="button"
+                  class="px-3 py-2 rounded-lg border text-[12px] transition-colors
+                    {settings.aiTier === 'cleanup'
+                      ? 'bg-bg-elevated border-accent text-text'
+                      : 'bg-bg-input border-border text-text-tertiary hover:text-text'}"
+                  onclick={() => setAiTier("cleanup")}
+                >Cleanup only</button>
+                <button
+                  type="button"
+                  class="px-3 py-2 rounded-lg border text-[12px] transition-colors
+                    {settings.aiTier === 'tasks'
+                      ? 'bg-bg-elevated border-accent text-text'
+                      : 'bg-bg-input border-border text-text-tertiary hover:text-text'}"
+                  onclick={() => setAiTier("tasks")}
+                >Cleanup + Tasks</button>
+              </div>
+              <p class="text-[11px] text-text-tertiary mt-2">
+                {settings.aiTier === "off"
+                  ? "No local LLM calls. Kon falls back to the existing rule-based path."
+                  : settings.aiTier === "cleanup"
+                    ? "Use the local model for transcript cleanup and formatting."
+                    : "Use the local model for cleanup, task extraction, and task breakdown."}
+              </p>
+            </div>
+
+            <div class="mb-5">
+              <p class="text-[10px] font-medium text-text-tertiary uppercase tracking-wider mb-2">Model Tier</p>
+              <div class="space-y-2">
+                {#each LLM_MODELS as model}
+                  <button
+                    type="button"
+                    class="w-full text-left rounded-lg border px-3 py-3 transition-colors
+                      {selectedLlmModelId() === model.id
+                        ? 'border-accent bg-bg-elevated'
+                        : 'border-border bg-bg-input hover:border-accent/50'}
+                      {llmTierAvailable(model.id) ? '' : 'opacity-70'}"
+                    onclick={() => selectLlmModel(model.id)}
+                  >
+                    <div class="flex items-start justify-between gap-3">
+                      <div>
+                        <div class="flex items-center gap-2">
+                          <span class="text-[12px] font-medium text-text">{model.label}</span>
+                          <span class="text-[11px] text-text-secondary">{model.subtitle}</span>
+                        </div>
+                        <p class="text-[11px] text-text-tertiary mt-1">{model.size} · {model.fit}</p>
+                        {#if llmHardwareWarning(model.id)}
+                          <p class="text-[11px] text-warning mt-2">{llmHardwareWarning(model.id)}</p>
+                        {/if}
+                      </div>
+                      <div class="text-right text-[11px]">
+                        {#if llmModelLoaded(model.id)}
+                          <span class="text-success">Loaded</span>
+                        {:else if llmModelDownloaded(model.id)}
+                          <span class="text-text-secondary">Downloaded</span>
+                        {:else}
+                          <span class="text-text-tertiary">Not downloaded</span>
+                        {/if}
+                      </div>
+                    </div>
+                  </button>
+                {/each}
+              </div>
+            </div>
+
+            <div class="bg-bg-input rounded-lg px-3 py-3 border border-border-subtle">
+              <div class="flex items-center justify-between gap-3 flex-wrap">
+                <div>
+                  <p class="text-[12px] text-text-secondary font-medium">
+                    {LLM_MODELS.find((model) => model.id === selectedLlmModelId())?.subtitle || "Local model"}
+                  </p>
+                  <p class="text-[11px] text-text-tertiary mt-1">{llmStatus}</p>
+                </div>
+
+                <div class="flex items-center gap-2 flex-wrap">
+                  {#if llmDownloadingModel === selectedLlmModelId()}
+                    <span class="text-[11px] text-warning">{llmDownloadProgress}% downloading…</span>
+                  {:else if !llmModelDownloaded(selectedLlmModelId())}
+                    <button
+                      type="button"
+                      class="px-3 py-2 rounded-lg bg-accent text-bg text-[12px] hover:bg-accent-hover"
+                      onclick={downloadSelectedLlmModel}
+                    >Download</button>
+                  {:else if !llmModelLoaded(selectedLlmModelId())}
+                    <button
+                      type="button"
+                      class="px-3 py-2 rounded-lg bg-accent text-bg text-[12px] hover:bg-accent-hover"
+                      onclick={loadSelectedLlmModel}
+                    >Load</button>
+                    <button
+                      type="button"
+                      class="px-3 py-2 rounded-lg border border-border text-[12px] text-text-secondary hover:text-text"
+                      onclick={deleteSelectedLlmModel}
+                    >Delete</button>
+                  {:else}
+                    <button
+                      type="button"
+                      class="px-3 py-2 rounded-lg border border-border text-[12px] text-text-secondary hover:text-text"
+                      onclick={unloadLlmModel}
+                    >Unload</button>
+                    <button
+                      type="button"
+                      class="px-3 py-2 rounded-lg border border-border text-[12px] text-text-secondary hover:text-text"
+                      onclick={deleteSelectedLlmModel}
+                    >Delete</button>
+                  {/if}
+                </div>
+              </div>
+
+              <p class="text-[11px] text-text-tertiary mt-3">
+                Recommended for this machine:
+                <span class="text-text">
+                  {LLM_MODELS.find((model) => model.id === (settings.llmModelId || "qwen3_4b_instruct_2507"))?.subtitle || "Qwen3 4B Instruct 2507"}
+                </span>
+                {#if systemInfo}
+                  · {Math.round((systemInfo.ram_mb || 0) / 1024)} GB RAM detected
+                {/if}
+              </p>
             </div>
           </div>
         {/if}

src/lib/stores/page.svelte.ts

@@ -49,8 +49,8 @@ const defaults: SettingsState = {
   includeTimestamps: true,
   theme: "Dark",
   fontSize: 14,
-  llmModelSize: "small",
-  llmEnabled: false,
+  aiTier: "cleanup",
+  llmModelId: null,
   saveAudio: false,
   outputFolder: "",
   globalHotkey: "Ctrl+Shift+R",

src/lib/types/app.ts

@@ -4,6 +4,8 @@ export type ReduceMotion = "system" | "on" | "off";
 export type RecordingEngine = "whisper" | "parakeet";
 export type FormatMode = "Raw" | "Clean" | "Smart";
 export type WhisperModelSize = "Tiny" | "Base" | "Small" | "Medium";
+export type AiTier = "off" | "cleanup" | "tasks";
+export type LlmModelIdStr = "qwen3_1_7b" | "qwen3_4b_instruct_2507" | "qwen3_14b";
 export type TaskBucket = "inbox" | "today" | "soon" | "later";
 export type ToastSeverity = "info" | "success" | "warn" | "error";
 
@@ -31,8 +33,8 @@ export interface SettingsState {
   includeTimestamps: boolean;
   theme: "Dark" | "Light" | "System";
   fontSize: number;
-  llmModelSize: string;
-  llmEnabled: boolean;
+  aiTier: AiTier;
+  llmModelId: LlmModelIdStr | null;
   saveAudio: boolean;
   outputFolder: string;
   globalHotkey: string;