From 401b6c36546c12a5cb37ca65a7e70860dca70f86 Mon Sep 17 00:00:00 2001 From: Jake Date: Thu, 14 May 2026 20:07:42 +0100 Subject: [PATCH] =?UTF-8?q?agent:=20lumotia=20=E2=80=94=20Phase=20B.9=20st?= =?UTF-8?q?rip=20Qwen=20=E2=80=A6=20reasoning=20before=20JS?= =?UTF-8?q?ON-envelope=20scan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase B.9 audit of commit 1d71e8e (replace GBNF grammar with manual brace-counting JSON-envelope extractor). Existing coverage: * parse_string_array_trims_and_dedupes * json_envelope_complete_detects_finished_{object,array} * json_envelope_complete_ignores_braces_inside_strings * json_envelope_complete_rejects_prefixes_and_trailing_text * extract_json_envelope_skips_qwen_thinking_prefix (EMPTY think block) * extract_json_envelope_handles_arrays_and_trailing_stop_text Solid for the cases tested. One real residual. The `_skips_qwen_thinking_prefix` regression uses an EMPTY block: `"\n\n\n\n{...}"`. Qwen3.5's reasoning mode emits non-empty reasoning when enabled (and reasoning is a documented Qwen feature, surfaced in the model name family the engine targets). The naive "find the first '{' or '[' in the whole text" extractor breaks in two ways once the reasoning is non-empty: 1. **JSON-looking text in thinking.** The model thinks out loud about the schema: "the answer should look like {\"topic\":\"x\",\"intent\":\"y\"}". The extractor sees the FIRST '{' (inside the reasoning), scans for its matching '}', and returns the reasoning literal as the envelope. The actual answer after is dropped. 2. **Unbalanced braces in thinking.** The model writes "I wonder about {something unfinished" inside . The extractor starts its brace-stack on that unbalanced '{', never finds a matching '}', scans past picking up the real answer's '{' (stack now has TWO '}' targets), eventually finds one '}' which pops the thinking's, then end of input — returns None. The actual answer is lost entirely. Fix: split on the FIRST `` and scan only the substring after. Anything before `` is reasoning, anything after is the answer proper. Falls back to the whole text when no `` is present (covers non-reasoning models AND the empty-thinking case the existing test pins). Backwards-compatible: * Empty thinking — split_once returns ("", "\n\n{...}"); scan finds the '{' and returns the answer. Existing test passes. * No thinking tags at all — split_once returns None; fall back to full text. Existing tests pass. * Trailing stop tokens (`<|im_end|>` etc.) — unchanged behaviour; they sit after the envelope and don't affect the scan. New regression tests: * extract_json_envelope_skips_thinking_block_with_json_looking_content — thinking with a JSON literal followed by the real answer. Pre-fix would return the thinking's literal; post-fix returns the answer. * extract_json_envelope_survives_unbalanced_braces_in_thinking — the unbalanced-brace-in-thinking case. Pre-fix returns None; post-fix returns the real answer. Verification: * cargo test -p lumotia-llm --lib → 28/28 pass including the two new tests. * cargo fmt --check → clean. * cargo clippy -p lumotia-llm --all-targets -- -D warnings → clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/llm/src/lib.rs | 62 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index aec5c9d..8111ffc 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -631,10 +631,27 @@ fn json_envelope_complete(text: &str) -> bool { } fn extract_json_envelope(text: &str) -> Option<&str> { - let start = text + // Phase B.9 audit residual (2026-05-14): strip the leading + // `…` reasoning block before scanning. Qwen-style + // models emit non-empty reasoning when thinking mode is on, and + // the reasoning can contain JSON-looking literals (e.g. + // "the answer should be {\"x\":1}") or unbalanced braces ("I wonder + // about {..."). The naive "find the first '{' or '['" extractor + // would then either return the wrong envelope or pollute the + // brace-stack and return None. We split on the FIRST `` — + // anything before it is reasoning, anything after is the answer + // proper. Falls back to the whole text when no `` is + // present (covers non-reasoning models and the empty-thinking + // case already covered by `extract_json_envelope_skips_qwen_thinking_prefix`). + let scan_region = text + .split_once("") + .map(|(_, rest)| rest) + .unwrap_or(text); + + let start = scan_region .char_indices() .find_map(|(idx, ch)| (ch == '{' || ch == '[').then_some(idx))?; - let mut chars = text[start..].char_indices(); + let mut chars = scan_region[start..].char_indices(); let (_, first) = chars.next()?; let mut stack = vec![match first { @@ -667,7 +684,7 @@ fn extract_json_envelope(text: &str) -> Option<&str> { } if stack.is_empty() { let end = start + offset + ch.len_utf8(); - return Some(&text[start..end]); + return Some(&scan_region[start..end]); } } _ => {} @@ -812,6 +829,45 @@ mod tests { ); } + /// Phase B.9 audit regression (2026-05-14). The original + /// `extract_json_envelope_skips_qwen_thinking_prefix` test only + /// covered an EMPTY `` block. Qwen-style reasoning + /// is typically non-empty and can contain JSON-looking literals + /// (the model thinking out loud about what shape it should emit). + /// The naive "first '{' wins" extractor mis-identified the + /// reasoning's literal as the answer envelope and returned it, + /// skipping the actual answer that followed ``. + /// + /// Post-fix the extractor strips the leading `…` + /// block before scanning, so the reasoning's literal cannot + /// poison the result. + #[test] + fn extract_json_envelope_skips_thinking_block_with_json_looking_content() { + let raw = "The answer should look like {\"topic\":\"reasoning-example\",\"intent\":\"capture\"} \ + based on the schema.{\"topic\":\"real-answer\",\"intent\":\"planning\"}"; + assert_eq!( + extract_json_envelope(raw), + Some("{\"topic\":\"real-answer\",\"intent\":\"planning\"}"), + ); + } + + /// Phase B.9 audit regression (2026-05-14). If the reasoning block + /// contains UNBALANCED braces (e.g. the model writes "I wonder + /// about {..." inside `…`), the pre-strip extractor + /// would start its stack on that unbalanced `{`, never find a + /// matching `}`, and continue past `` polluting the stack + /// with the real answer's braces — ultimately returning None and + /// losing the answer entirely. Stripping the reasoning block first + /// makes both cases moot. + #[test] + fn extract_json_envelope_survives_unbalanced_braces_in_thinking() { + let raw = "I wonder about {something unfinished here{\"topic\":\"recovery\",\"intent\":\"capture\"}"; + assert_eq!( + extract_json_envelope(raw), + Some("{\"topic\":\"recovery\",\"intent\":\"capture\"}"), + ); + } + #[test] fn prompt_preflight_rejects_oversized_prompt_tokens() { let err = preflight_context_window(7_105, 1_024).unwrap_err();