diff --git a/src/tool_parsing.py b/src/tool_parsing.py index 2885cc00f..98dc1b5f6 100644 --- a/src/tool_parsing.py +++ b/src/tool_parsing.py @@ -187,8 +187,12 @@ _FUNCTION_MODEL_NAME_RE = re.compile( _FUNCTION_MODEL_PARAMS_OPEN_RE = re.compile(r"\s*", re.IGNORECASE) _FUNCTION_MODEL_PARAMS_CLOSE_RE = re.compile(r"", re.IGNORECASE) _QWEN_ROLE_MARKER_RE = re.compile(r"?|?", re.IGNORECASE) +# At least one pipe is required around `end`. Both pipes used to be optional +# (`\|?end\|?`), which also matched a bare `end` on its own line and deleted it +# from ordinary prose and from Ruby/Lua/shell snippets that close blocks with +# one; see #5547. `|end`, `end|`, `|end|` and `/|end|` still strip as before. _QWEN_BARE_MARKER_RE = re.compile( - r"(?:^|[\t\r\n ])(?:\|?end\|?|/?\|end\|)(?=[\t\r\n ]|$)|" + r"(?:^|[\t\r\n ])(?:/?\|end\||\|end|end\|)(?=[\t\r\n ]|$)|" r"(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)", re.IGNORECASE, ) diff --git a/static/js/chatRenderer.js b/static/js/chatRenderer.js index 10709679d..1d6e2e4a9 100644 --- a/static/js/chatRenderer.js +++ b/static/js/chatRenderer.js @@ -478,7 +478,10 @@ const DSML_STRAY_RE = /<\s*\/?\s*[｜|]+\s*DSML\s*[｜|]+[^>]*>/gi; const DSML_INVOKE_RE = /<\s*[｜|]+\s*DSML\s*[｜|]+\s*invoke\b[^>]*>[\s\S]*?(?:<\s*\/\s*[｜|]+\s*DSML\s*[｜|]+\s*invoke\s*>|$)/gi; const RAW_OPENAI_TOOL_JSON_RE = /(?:\[\s*)?\{\s*"function"\s*:\s*\{[\s\S]*?\}\s*,\s*"id"\s*:\s*"[^"]*"\s*,\s*"type"\s*:\s*"function"\s*\}\s*\]?/gi; const QWEN_ROLE_MARKER_RE = /<\/?\|(?:assistant|assistan|user|system|tool)\|>?|<\/\|end\|>?/gi; -const QWEN_BARE_MARKER_RE = /(?:^|[\t\r\n ])(?:\|?end\|?|\/?\|end\|)(?=[\t\r\n ]|$)|(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)/gi; +// Keep in sync with _QWEN_BARE_MARKER_RE in src/tool_parsing.py. At least one +// pipe is required around `end`: with both optional (`\|?end\|?`) this also ate +// a bare `end` on its own line, breaking Ruby/Lua/shell snippets (#5547). +const QWEN_BARE_MARKER_RE = /(?:^|[\t\r\n ])(?:\/?\|end\||\|end|end\|)(?=[\t\r\n ]|$)|(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)/gi; // Self-narration about tool results (model echoing stdout/exit_code) const TOOL_NARRATION_RE = /(?:The (?:result|output) shows?:?\s*)?-?\s*(?:stdout|stderr|exit_code):\s*.+/gi; diff --git a/tests/test_tool_parsing_bare_end_marker.py b/tests/test_tool_parsing_bare_end_marker.py new file mode 100644 index 000000000..6167c8dde --- /dev/null +++ b/tests/test_tool_parsing_bare_end_marker.py @@ -0,0 +1,96 @@ +"""Regression: the Qwen bare-marker scrub must not eat a lone `end` (#5547). + +`_QWEN_BARE_MARKER_RE` cleans Qwen turn markers that leak into content. Its +`end` branch was `\\|?end\\|?` — both pipes optional — so it also matched a bare +`end` surrounded by whitespace and replaced it with a space. Any message +containing Ruby, Lua or shell code that closes a block with a lone `end` had +those lines silently deleted, in the stored text and in the rendered message. + +Requiring at least one pipe keeps every real marker (`|end`, `end|`, `|end|`, +`/|end|`) stripping as before. The same pattern is duplicated in +static/js/chatRenderer.js, so the JS copy is checked here too — the two must +not drift. +""" +import json +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +import src.agent_tools # noqa: F401 (break agent_tools<->tool_parsing import cycle) +from src.tool_parsing import strip_tool_blocks + +_REPO = Path(__file__).resolve().parent.parent +_CHAT_RENDERER = _REPO / "static" / "js" / "chatRenderer.js" + +# Inputs that must survive untouched, and the substring that proves they did. +KEPT = [ + ("loop do\n puts \"yo\"\nend\n", "\nend"), # the reported Ruby case + ("if x then\nend", "\nend"), + ("function f()\nend\n", "\nend"), + ("a end b", "a end b"), + ("append end", "append end"), + ("END", "END"), + ("\nEnd\n", "End"), +] + +# Real markers — at least one pipe, plus the role word — with the exact output +# they must still produce. Asserted as equality rather than "marker not in out" +# so narrowing the pattern can't pass by deleting more than it should. +STRIPPED = [ + ("a |end| b", "a b"), + ("a /|end| b", "a b"), + ("a |end b", "a b"), + ("a end| b", "a b"), + ("x assistant y", "x y"), +] + + +@pytest.mark.parametrize("text,kept", KEPT) +def test_bare_end_survives_stripping(text, kept): + assert kept in strip_tool_blocks(text) + + +@pytest.mark.parametrize("text,expected", STRIPPED) +def test_piped_end_markers_are_still_stripped(text, expected): + assert strip_tool_blocks(text) == expected + + +def test_bare_end_inside_a_fenced_block_survives(): + """The scrub runs over the whole message, fenced regions included.""" + out = strip_tool_blocks("Here:\n```ruby\nloop do\n puts 1\nend\n```\nDone.") + assert "\nend\n" in out + + +def _js_bare_marker_regex_source(): + src = _CHAT_RENDERER.read_text(encoding="utf-8") + m = re.search(r"^const QWEN_BARE_MARKER_RE = (/.*/[gimsuy]*);$", src, re.MULTILINE) + assert m, "QWEN_BARE_MARKER_RE literal not found in chatRenderer.js" + return m.group(1) + + +def test_js_copy_of_the_pattern_matches_the_python_one(): + """Guard the duplication: the JS branch must require a pipe too.""" + if shutil.which("node") is None: + pytest.skip("node binary not on PATH") + + cases = [text for text, _ in KEPT] + [text for text, _ in STRIPPED] + script = ( + "const RE = %s;\n" + "const cases = JSON.parse(process.argv[1]);\n" + "console.log(JSON.stringify(cases.map(c => c.replace(RE, ' '))));" + % _js_bare_marker_regex_source() + ) + result = subprocess.run( + ["node", "--input-type=module", "-e", script, json.dumps(cases)], + cwd=_REPO, capture_output=True, timeout=15, text=True, + ) + assert result.returncode == 0, f"node failed:\n{result.stderr}" + got = json.loads(result.stdout.splitlines()[-1]) + + for (text, kept), out in zip(KEPT, got): + assert kept in out, f"JS regex dropped {kept!r} from {text!r}" + for (text, expected), out in zip(STRIPPED, got[len(KEPT):]): + assert out == expected, f"JS regex: {text!r} -> {out!r}, expected {expected!r}"