From 3c11a31207ac03d3315f0596fc0a6a0407574fea Mon Sep 17 00:00:00 2001 From: Ichhabal Singh Date: Wed, 15 Jul 2026 17:14:05 +0530 Subject: [PATCH 1/2] fix(parsing): add support for Qwen-style tool call tokens and AST parsing --- src/tool_parsing.py | 60 ++++++++++++++++++++++++++++++++++++ tests/test_qwen_tool_call.py | 29 +++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 tests/test_qwen_tool_call.py diff --git a/src/tool_parsing.py b/src/tool_parsing.py index 2885cc00f..d7d08f35c 100644 --- a/src/tool_parsing.py +++ b/src/tool_parsing.py @@ -192,6 +192,10 @@ _QWEN_BARE_MARKER_RE = re.compile( r"(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)", re.IGNORECASE, ) +_QWEN_TOOL_CALL_RE = re.compile( + r"<\|tool_call_start\|>\[(\w+)\(([\s\S]*?)\)\](?:<\|tool_call_end\|>)?", + re.IGNORECASE +) # Pattern 5: DeepSeek DSML markup leaking into content. When deepseek @@ -1110,6 +1114,52 @@ def _parse_gemma_tool_call(tool_name: str, body: str) -> Optional[ToolBlock]: return function_call_to_tool_block(tool_name, json.dumps(params)) +def _parse_qwen_tool_call(tool_name: str, args_str: str) -> Optional[ToolBlock]: + """Parse a Qwen-style call: <|tool_call_start|>[tool_name(args_str)]<|tool_call_end|>.""" + tool_name = tool_name.strip().lower().replace("-", "_") + args_str = args_str.strip() + + params = {} + if args_str: + import ast + try: + tree = ast.parse(f"dummy({args_str})") + call_node = None + for node in ast.walk(tree): + if isinstance(node, ast.Call): + call_node = node + break + if call_node: + for kw in call_node.keywords: + try: + params[kw.arg] = ast.literal_eval(kw.value) + except Exception: + pass + except Exception: + params = {} + for m in re.finditer(r"(\w+)\s*=\s*(?:['\"]([^'\"]*)['\"]|([\w.]+))", args_str): + k = m.group(1) + v = m.group(2) if m.group(2) is not None else m.group(3) + if v == "True": + v = True + elif v == "False": + v = False + elif v == "None": + v = None + else: + try: + if "." in v: + v = float(v) + else: + v = int(v) + except ValueError: + pass + params[k] = v + + from src.tool_schemas import function_call_to_tool_block + return function_call_to_tool_block(tool_name, json.dumps(params)) + + def _parse_function_model_call(body: str) -> Optional[ToolBlock]: """Parse tool....""" name_match = _FUNCTION_MODEL_NAME_RE.search(body or "") @@ -1379,6 +1429,15 @@ def parse_tool_blocks(text: str, skip_fenced: bool = False) -> List[ToolBlock]: if block: blocks.append(block) + # Pattern 4b_qwen: Qwen-style <|tool_call_start|> blocks + if not blocks: + for m in _QWEN_TOOL_CALL_RE.finditer(text): + tool_name = m.group(1) + args_str = m.group(2) + block = _parse_qwen_tool_call(tool_name, args_str) + if block: + blocks.append(block) + # Pattern 4c: wrapper from local MLX/Exo models. if not blocks: for _ms, inner_start, inner_end, _me in _iter_delimited( @@ -1441,6 +1500,7 @@ def strip_tool_blocks(text: str, skip_fenced: bool = False) -> str: cleaned = _XML_OPEN_TOOL_CALL_RE.sub('', cleaned) cleaned = _strip_delimited(cleaned, _TOOL_CODE_OPEN_RE, _TOOL_CODE_CLOSE_RE) cleaned = _GEMMA_TOOL_CALL_RE.sub('', cleaned) + cleaned = _QWEN_TOOL_CALL_RE.sub('', cleaned) cleaned = _strip_delimited(cleaned, _FUNCTION_MODEL_OPEN_RE, _FUNCTION_MODEL_CLOSE_RE) cleaned = _strip_raw_openai_tool_call_json(cleaned) cleaned = _QWEN_ROLE_MARKER_RE.sub('', cleaned) diff --git a/tests/test_qwen_tool_call.py b/tests/test_qwen_tool_call.py new file mode 100644 index 000000000..bda59ccfa --- /dev/null +++ b/tests/test_qwen_tool_call.py @@ -0,0 +1,29 @@ +import src.agent_tools # noqa: F401 +from src.tool_parsing import parse_tool_blocks, strip_tool_blocks + + +def test_qwen_tool_call_parsing_and_stripping(): + raw = """Sure, let me check your accounts. + +<|tool_call_start|>[list_email_accounts()]<|tool_call_end|>""" + + blocks = parse_tool_blocks(raw, skip_fenced=True) + + assert len(blocks) == 1 + assert blocks[0].tool_type == "mcp__email__list_email_accounts" + assert blocks[0].content == "{}" + assert strip_tool_blocks(raw, skip_fenced=True) == "Sure, let me check your accounts." + + +def test_qwen_tool_call_with_args(): + raw = """Okay, fetching recent messages. +<|tool_call_start|>[list_emails(account="Gmail", unread_only=True, max_results=5)]<|tool_call_end|>""" + + blocks = parse_tool_blocks(raw, skip_fenced=True) + + assert len(blocks) == 1 + assert blocks[0].tool_type == "mcp__email__list_emails" + assert "Gmail" in blocks[0].content + + cleaned = strip_tool_blocks(raw, skip_fenced=True) + assert cleaned == "Okay, fetching recent messages." From 941fed3f4cfdd8f2be69e85592e1f7ab5da1ad90 Mon Sep 17 00:00:00 2001 From: Ichhabal Singh Date: Fri, 24 Jul 2026 11:13:33 +0530 Subject: [PATCH 2/2] fix(parsing): enhance Qwen parser with positional arguments and robust fallbacks --- src/tool_parsing.py | 55 ++++++++++++++++++++++++++++++++++-- tests/test_qwen_tool_call.py | 46 ++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/src/tool_parsing.py b/src/tool_parsing.py index d7d08f35c..6c810b752 100644 --- a/src/tool_parsing.py +++ b/src/tool_parsing.py @@ -193,7 +193,7 @@ _QWEN_BARE_MARKER_RE = re.compile( re.IGNORECASE, ) _QWEN_TOOL_CALL_RE = re.compile( - r"<\|tool_call_start\|>\[(\w+)\(([\s\S]*?)\)\](?:<\|tool_call_end\|>)?", + r"<\|tool_call_start\|>\[(\w+)\(([\s\S]*?)\)\](?:\s*<\|tool_call_end\|>)?", re.IGNORECASE ) @@ -1114,6 +1114,35 @@ def _parse_gemma_tool_call(tool_name: str, body: str) -> Optional[ToolBlock]: return function_call_to_tool_block(tool_name, json.dumps(params)) +_BUILTIN_POSITIONAL_ARGS = { + "bash": ["command"], + "python": ["code"], + "web_search": ["query", "time_filter"], + "web_fetch": ["url", "full"], + "read_file": ["path", "offset", "limit"], + "write_file": ["path", "content"], + "edit_file": ["path", "old_string", "new_string", "replace_all"], + "grep": ["pattern", "path", "glob", "ignore_case", "max_results"], + "glob": ["pattern", "path"], + "ls": ["path"], + "create_document": ["title", "language", "content"], +} + +_BUILTIN_PRIMARY_ARG = { + "bash": "command", + "python": "code", + "web_search": "query", + "web_fetch": "url", + "read_file": "path", + "write_file": "path", + "edit_file": "path", + "grep": "pattern", + "glob": "pattern", + "ls": "path", + "create_document": "title", +} + + def _parse_qwen_tool_call(tool_name: str, args_str: str) -> Optional[ToolBlock]: """Parse a Qwen-style call: <|tool_call_start|>[tool_name(args_str)]<|tool_call_end|>.""" tool_name = tool_name.strip().lower().replace("-", "_") @@ -1130,6 +1159,16 @@ def _parse_qwen_tool_call(tool_name: str, args_str: str) -> Optional[ToolBlock]: call_node = node break if call_node: + # Map positional arguments if we know the tool's signature + pos_names = _BUILTIN_POSITIONAL_ARGS.get(tool_name) + if pos_names: + for idx, arg_val in enumerate(call_node.args): + if idx < len(pos_names): + try: + params[pos_names[idx]] = ast.literal_eval(arg_val) + except Exception: + pass + # Map keyword arguments for kw in call_node.keywords: try: params[kw.arg] = ast.literal_eval(kw.value) @@ -1137,9 +1176,10 @@ def _parse_qwen_tool_call(tool_name: str, args_str: str) -> Optional[ToolBlock]: pass except Exception: params = {} - for m in re.finditer(r"(\w+)\s*=\s*(?:['\"]([^'\"]*)['\"]|([\w.]+))", args_str): + # Fallback 1: regex key-value extraction that handles spaces and quotes + for m in re.finditer(r"(\w+)\s*=\s*['\"]?(.*?)['\"]?(?=\s*,\s*\w+\s*=|\s*$)", args_str): k = m.group(1) - v = m.group(2) if m.group(2) is not None else m.group(3) + v = m.group(2).strip() if v == "True": v = True elif v == "False": @@ -1156,6 +1196,15 @@ def _parse_qwen_tool_call(tool_name: str, args_str: str) -> Optional[ToolBlock]: pass params[k] = v + # Fallback 2: single argument fallback if no key-value pairs matched + if not params: + primary_arg = _BUILTIN_PRIMARY_ARG.get(tool_name) + if primary_arg: + val = args_str.strip() + if len(val) >= 2 and ((val[0] == '"' and val[-1] == '"') or (val[0] == "'" and val[-1] == "'")): + val = val[1:-1] + params[primary_arg] = val + from src.tool_schemas import function_call_to_tool_block return function_call_to_tool_block(tool_name, json.dumps(params)) diff --git a/tests/test_qwen_tool_call.py b/tests/test_qwen_tool_call.py index bda59ccfa..7ba450cae 100644 --- a/tests/test_qwen_tool_call.py +++ b/tests/test_qwen_tool_call.py @@ -27,3 +27,49 @@ def test_qwen_tool_call_with_args(): cleaned = strip_tool_blocks(raw, skip_fenced=True) assert cleaned == "Okay, fetching recent messages." + + +def test_qwen_tool_call_positional_args(): + # Single positional argument + raw = '<|tool_call_start|>[web_search("Sweden news")]<|tool_call_end|>' + blocks = parse_tool_blocks(raw, skip_fenced=True) + assert len(blocks) == 1 + assert blocks[0].tool_type == "web_search" + assert "Sweden news" in blocks[0].content + + # Multiple positional arguments + raw = '<|tool_call_start|>[read_file("src/main.py", 10, 50)]<|tool_call_end|>' + blocks = parse_tool_blocks(raw, skip_fenced=True) + assert len(blocks) == 1 + assert blocks[0].tool_type == "read_file" + assert "src/main.py" in blocks[0].content + assert "10" in blocks[0].content + assert "50" in blocks[0].content + + +def test_qwen_tool_call_whitespace_before_end_tag(): + raw = "Okay.\n<|tool_call_start|>[web_search(query=\"Sweden news\")]\n<|tool_call_end|>" + blocks = parse_tool_blocks(raw, skip_fenced=True) + assert len(blocks) == 1 + assert blocks[0].tool_type == "web_search" + + cleaned = strip_tool_blocks(raw, skip_fenced=True) + assert cleaned == "Okay." + + +def test_qwen_tool_call_regex_fallback_and_single_arg(): + # Regex fallback with unquoted values containing spaces + raw = '<|tool_call_start|>[web_search(query=Sweden news today, time_filter=day)]<|tool_call_end|>' + blocks = parse_tool_blocks(raw, skip_fenced=True) + assert len(blocks) == 1 + assert blocks[0].tool_type == "web_search" + assert "Sweden news today" in blocks[0].content + assert "day" in blocks[0].content + + # Single argument fallback (syntax error, no keyword) + raw = '<|tool_call_start|>[web_search(Sweden news today)]<|tool_call_end|>' + blocks = parse_tool_blocks(raw, skip_fenced=True) + assert len(blocks) == 1 + assert blocks[0].tool_type == "web_search" + assert "Sweden news today" in blocks[0].content +