fix(tool_parsing): require a pipe on the Qwen bare end marker

The `end` branch of _QWEN_BARE_MARKER_RE had both pipes optional
(`\|?end\|?`), so it also matched a bare `end` between whitespace and
replaced it with a space. Messages containing Ruby, Lua or shell code that
closes a block with a lone `end` had those lines deleted, and ordinary prose
lost the word too.

Require at least one pipe so only real turn markers match; `|end`, `end|`,
`|end|` and `/|end|` strip exactly as before. Applied to the duplicated
pattern in static/js/chatRenderer.js as well.

Fixes #5547

src/tool_parsing.py

@@ -187,8 +187,12 @@ _FUNCTION_MODEL_NAME_RE = re.compile(
 _FUNCTION_MODEL_PARAMS_OPEN_RE = re.compile(r"<parameters>\s*", re.IGNORECASE)
 _FUNCTION_MODEL_PARAMS_CLOSE_RE = re.compile(r"</parameters>", re.IGNORECASE)
 _QWEN_ROLE_MARKER_RE = re.compile(r"</?\|(?:assistant|assistan|user|system|tool)\|>?|</\|end\|>?", re.IGNORECASE)
+# At least one pipe is required around `end`. Both pipes used to be optional
+# (`\|?end\|?`), which also matched a bare `end` on its own line and deleted it
+# from ordinary prose and from Ruby/Lua/shell snippets that close blocks with
+# one; see #5547. `|end`, `end|`, `|end|` and `/|end|` still strip as before.
 _QWEN_BARE_MARKER_RE = re.compile(
-    r"(?:^|[\t\r\n ])(?:\|?end\|?|/?\|end\|)(?=[\t\r\n ]|$)|"
+    r"(?:^|[\t\r\n ])(?:/?\|end\||\|end|end\|)(?=[\t\r\n ]|$)|"
     r"(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)",
     re.IGNORECASE,
 )

static/js/chatRenderer.js

@@ -478,7 +478,10 @@ const DSML_STRAY_RE = /<\s*\/?\s*[｜|]+\s*DSML\s*[｜|]+[^>]*>/gi;
 const DSML_INVOKE_RE = /<\s*[｜|]+\s*DSML\s*[｜|]+\s*invoke\b[^>]*>[\s\S]*?(?:<\s*\/\s*[｜|]+\s*DSML\s*[｜|]+\s*invoke\s*>|$)/gi;
 const RAW_OPENAI_TOOL_JSON_RE = /(?:\[\s*)?\{\s*"function"\s*:\s*\{[\s\S]*?\}\s*,\s*"id"\s*:\s*"[^"]*"\s*,\s*"type"\s*:\s*"function"\s*\}\s*\]?/gi;
 const QWEN_ROLE_MARKER_RE = /<\/?\|(?:assistant|assistan|user|system|tool)\|>?|<\/\|end\|>?/gi;
-const QWEN_BARE_MARKER_RE = /(?:^|[\t\r\n ])(?:\|?end\|?|\/?\|end\|)(?=[\t\r\n ]|$)|(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)/gi;
+// Keep in sync with _QWEN_BARE_MARKER_RE in src/tool_parsing.py. At least one
+// pipe is required around `end`: with both optional (`\|?end\|?`) this also ate
+// a bare `end` on its own line, breaking Ruby/Lua/shell snippets (#5547).
+const QWEN_BARE_MARKER_RE = /(?:^|[\t\r\n ])(?:\/?\|end\||\|end|end\|)(?=[\t\r\n ]|$)|(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)/gi;
 // Self-narration about tool results (model echoing stdout/exit_code)
 const TOOL_NARRATION_RE = /(?:The (?:result|output) shows?:?\s*)?-?\s*(?:stdout|stderr|exit_code):\s*.+/gi;
 

tests/test_tool_parsing_bare_end_marker.py

@@ -0,0 +1,96 @@
+"""Regression: the Qwen bare-marker scrub must not eat a lone `end` (#5547).
+
+`_QWEN_BARE_MARKER_RE` cleans Qwen turn markers that leak into content. Its
+`end` branch was `\\|?end\\|?` — both pipes optional — so it also matched a bare
+`end` surrounded by whitespace and replaced it with a space. Any message
+containing Ruby, Lua or shell code that closes a block with a lone `end` had
+those lines silently deleted, in the stored text and in the rendered message.
+
+Requiring at least one pipe keeps every real marker (`|end`, `end|`, `|end|`,
+`/|end|`) stripping as before. The same pattern is duplicated in
+static/js/chatRenderer.js, so the JS copy is checked here too — the two must
+not drift.
+"""
+import json
+import re
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+import src.agent_tools  # noqa: F401  (break agent_tools<->tool_parsing import cycle)
+from src.tool_parsing import strip_tool_blocks
+
+_REPO = Path(__file__).resolve().parent.parent
+_CHAT_RENDERER = _REPO / "static" / "js" / "chatRenderer.js"
+
+# Inputs that must survive untouched, and the substring that proves they did.
+KEPT = [
+    ("loop do\n    puts \"yo\"\nend\n", "\nend"),          # the reported Ruby case
+    ("if x then\nend", "\nend"),
+    ("function f()\nend\n", "\nend"),
+    ("a end b", "a end b"),
+    ("append end", "append end"),
+    ("END", "END"),
+    ("\nEnd\n", "End"),
+]
+
+# Real markers — at least one pipe, plus the role word — with the exact output
+# they must still produce. Asserted as equality rather than "marker not in out"
+# so narrowing the pattern can't pass by deleting more than it should.
+STRIPPED = [
+    ("a |end| b", "a  b"),
+    ("a /|end| b", "a  b"),
+    ("a |end b", "a  b"),
+    ("a end| b", "a  b"),
+    ("x assistant y", "x  y"),
+]
+
+
+@pytest.mark.parametrize("text,kept", KEPT)
+def test_bare_end_survives_stripping(text, kept):
+    assert kept in strip_tool_blocks(text)
+
+
+@pytest.mark.parametrize("text,expected", STRIPPED)
+def test_piped_end_markers_are_still_stripped(text, expected):
+    assert strip_tool_blocks(text) == expected
+
+
+def test_bare_end_inside_a_fenced_block_survives():
+    """The scrub runs over the whole message, fenced regions included."""
+    out = strip_tool_blocks("Here:\n```ruby\nloop do\n  puts 1\nend\n```\nDone.")
+    assert "\nend\n" in out
+
+
+def _js_bare_marker_regex_source():
+    src = _CHAT_RENDERER.read_text(encoding="utf-8")
+    m = re.search(r"^const QWEN_BARE_MARKER_RE = (/.*/[gimsuy]*);$", src, re.MULTILINE)
+    assert m, "QWEN_BARE_MARKER_RE literal not found in chatRenderer.js"
+    return m.group(1)
+
+
+def test_js_copy_of_the_pattern_matches_the_python_one():
+    """Guard the duplication: the JS branch must require a pipe too."""
+    if shutil.which("node") is None:
+        pytest.skip("node binary not on PATH")
+
+    cases = [text for text, _ in KEPT] + [text for text, _ in STRIPPED]
+    script = (
+        "const RE = %s;\n"
+        "const cases = JSON.parse(process.argv[1]);\n"
+        "console.log(JSON.stringify(cases.map(c => c.replace(RE, ' '))));"
+        % _js_bare_marker_regex_source()
+    )
+    result = subprocess.run(
+        ["node", "--input-type=module", "-e", script, json.dumps(cases)],
+        cwd=_REPO, capture_output=True, timeout=15, text=True,
+    )
+    assert result.returncode == 0, f"node failed:\n{result.stderr}"
+    got = json.loads(result.stdout.splitlines()[-1])
+
+    for (text, kept), out in zip(KEPT, got):
+        assert kept in out, f"JS regex dropped {kept!r} from {text!r}"
+    for (text, expected), out in zip(STRIPPED, got[len(KEPT):]):
+        assert out == expected, f"JS regex: {text!r} -> {out!r}, expected {expected!r}"