mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-07 14:37:55 +00:00
576abb012d
Own each agent-browser session as a browser tree: the daemon's POSIX session, its runtime files and its Chrome profile. Timeouts, launch failures, bootstrap recovery, cancellation and shutdown clean that tree and verify nothing survives, instead of killing only the daemon and orphaning Chrome. Per-call cleanup no longer sweeps every Chrome under the runtime TMPDIR. Sessionless calls get an ephemeral browser closed before returning. Actions on one session are serialized. Recovery is bounded by one deadline with at most one retry for local HTML open, and the retry flag is no longer model-visible. Observations after a failed navigation are marked stale. read URL navigates and extracts in one batch because agent-browser has no read command. Results carry a browser_lifecycle receipt with stages, timings, ownership and cleanup evidence. Playwright MCP tool calls are bounded by ODYSSEUS_BROWSER_MCP_CALL_TIMEOUT_S and are not retried. research_navigator now passes timeout_ms.
247 lines
9.2 KiB
Python
247 lines
9.2 KiB
Python
"""Research navigation primitives.
|
|
|
|
Deep Research historically had its own narrow Search -> Fetch path. This
|
|
module gives the research engine a small normalized surface for richer web
|
|
navigation while still reusing Odysseus' existing web tooling.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import urllib.parse
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
from src.constants import MAX_OUTPUT_CHARS
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ResearchAction:
|
|
"""A bounded navigation action the research planner can request."""
|
|
|
|
tool: str
|
|
args: dict[str, Any]
|
|
|
|
|
|
@dataclass
|
|
class ResearchPage:
|
|
"""Normalized readable page content for research extraction."""
|
|
|
|
url: str
|
|
title: str = ""
|
|
content: str = ""
|
|
og_image: str = ""
|
|
success: bool = False
|
|
retrieval: str = "fetch"
|
|
error: str = ""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ResearchSourceAssessment:
|
|
"""Simple quality metadata for a gathered research source."""
|
|
|
|
kind: str
|
|
score: int
|
|
reason: str
|
|
|
|
|
|
def parse_research_actions(text: str, *, allowed_tools: set[str] | None = None) -> list[ResearchAction]:
|
|
"""Parse a model reply into bounded research actions.
|
|
|
|
Accepts either a JSON array directly or an object with an ``actions`` array.
|
|
This is intentionally small and strict so a later model-planned research
|
|
loop can be added without giving the research model arbitrary tool access.
|
|
"""
|
|
allowed = allowed_tools or {
|
|
"web_search",
|
|
"web_fetch",
|
|
"browser_open",
|
|
"browser_read",
|
|
"browser_snapshot",
|
|
"private_browser",
|
|
}
|
|
raw = (text or "").strip()
|
|
if raw.startswith("```"):
|
|
raw = raw.removeprefix("```json").removeprefix("```").strip()
|
|
if raw.endswith("```"):
|
|
raw = raw[:-3].strip()
|
|
try:
|
|
parsed = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
return []
|
|
if isinstance(parsed, dict):
|
|
parsed = parsed.get("actions")
|
|
if not isinstance(parsed, list):
|
|
return []
|
|
actions: list[ResearchAction] = []
|
|
for item in parsed:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
has_tool_field = "tool" in item
|
|
tool = str(item.get("tool") or item.get("action") or "").strip()
|
|
if tool not in allowed:
|
|
continue
|
|
args = item.get("args")
|
|
if not isinstance(args, dict):
|
|
excluded = {"tool"} if has_tool_field else {"action"}
|
|
args = {k: v for k, v in item.items() if k not in excluded}
|
|
actions.append(ResearchAction(tool=tool, args=args))
|
|
return actions
|
|
|
|
|
|
def assess_source(url: str, *, title: str = "", retrieval: str = "", summary: str = "") -> ResearchSourceAssessment:
|
|
"""Estimate source usefulness for planning and reporting.
|
|
|
|
This is deliberately coarse. The model still judges evidence content; this
|
|
only gives the planner a compact map of whether it has official/primary
|
|
sources or mostly secondary/search-result material.
|
|
"""
|
|
parsed = urllib.parse.urlparse(str(url or ""))
|
|
host = (parsed.netloc or parsed.path.split("/", 1)[0]).lower().removeprefix("www.")
|
|
path = (parsed.path or "").lower()
|
|
text = " ".join([host, path, str(title or ""), str(summary or "")]).lower()
|
|
retrieval = (retrieval or "fetch").lower()
|
|
|
|
kind = "secondary"
|
|
score = 55
|
|
reason = "secondary web source"
|
|
|
|
if host.endswith((".gov", ".edu", ".ac.uk")) or ".gov." in host:
|
|
kind, score, reason = "official", 90, "government/academic domain"
|
|
elif any(part in host for part in ("github.com", "huggingface.co", "docs.", "developer.", "support.")):
|
|
kind, score, reason = "primary", 82, "primary project/vendor source"
|
|
elif any(token in text for token in ("official", "documentation", "docs", "release notes", "press release")):
|
|
kind, score, reason = "primary", 78, "primary-source wording"
|
|
elif any(part in host for part in ("reddit.com", "quora.com", "stackoverflow.com", "news.ycombinator.com")):
|
|
kind, score, reason = "community", 45, "community/forum source"
|
|
elif any(token in text for token in ("affiliate", "coupon", "best-", "top-", "review")):
|
|
kind, score, reason = "commercial", 40, "commercial/listicle source"
|
|
|
|
if retrieval == "browser":
|
|
score = min(100, score + 5)
|
|
reason += "; browser-read"
|
|
if not str(summary or "").strip():
|
|
score = max(10, score - 20)
|
|
reason += "; weak extraction"
|
|
|
|
return ResearchSourceAssessment(kind=kind, score=score, reason=reason)
|
|
|
|
|
|
class ResearchNavigator:
|
|
"""Small web-navigation facade used by Deep Research.
|
|
|
|
It delegates to the same search/fetch/browser implementations the chat
|
|
agent uses, but normalizes results for the research extraction pipeline.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
progress_callback=None,
|
|
session_id: str = "",
|
|
search_provider: str | None = None,
|
|
) -> None:
|
|
self._progress = progress_callback
|
|
self.session_id = session_id
|
|
self.search_provider = (search_provider or "").strip()
|
|
self.providers_used: list[str] = []
|
|
self.last_search_error = ""
|
|
self.browser_fetches = 0
|
|
|
|
async def search(self, query: str, *, count: int = 10) -> list[dict[str, Any]]:
|
|
"""Run a provider-chain web search and return structured results."""
|
|
try:
|
|
from src.search.providers import _get_search_settings
|
|
from src.search.core import _build_provider_chain, _call_provider
|
|
|
|
settings = _get_search_settings()
|
|
provider = self.search_provider or (settings.get("research_search_provider") or "").strip()
|
|
if not provider:
|
|
provider = settings.get("search_provider", "searxng")
|
|
if provider == "disabled":
|
|
logger.info("Search is disabled for research")
|
|
return []
|
|
|
|
chain = _build_provider_chain(provider)
|
|
raised = False
|
|
for prov in chain:
|
|
try:
|
|
results = await asyncio.to_thread(_call_provider, prov, query, count)
|
|
if results:
|
|
if prov not in self.providers_used:
|
|
self.providers_used.append(prov)
|
|
return results
|
|
except Exception as e:
|
|
raised = True
|
|
logger.warning("Research search provider %s failed: %s", prov, e)
|
|
self.last_search_error = f"{prov}: {e}"
|
|
if not raised:
|
|
self.last_search_error = (
|
|
"no results from search provider(s): "
|
|
f"{', '.join(chain) if chain else provider}"
|
|
)
|
|
return []
|
|
except Exception as e:
|
|
logger.error("Research search failed for %r: %s", query, e)
|
|
self.last_search_error = str(e)
|
|
return []
|
|
|
|
async def fetch(self, url: str, *, timeout: int = 10, max_bytes: int | None = None) -> ResearchPage:
|
|
"""Fetch readable text from a URL using Odysseus' web fetcher."""
|
|
try:
|
|
from src.search.content import fetch_webpage_content
|
|
|
|
kwargs: dict[str, Any] = {"timeout": timeout}
|
|
if max_bytes is not None:
|
|
kwargs["max_bytes"] = max_bytes
|
|
page = await asyncio.to_thread(fetch_webpage_content, url, **kwargs)
|
|
except Exception as e:
|
|
return ResearchPage(url=url, success=False, error=str(e))
|
|
return self._normalize_fetch_page(url, page)
|
|
|
|
async def browser_read(self, url: str, *, timeout: int = 45) -> ResearchPage:
|
|
"""Read a JS-heavy page through the private browser tool."""
|
|
from src.agent_tools.web_tools import PrivateBrowserTool
|
|
|
|
if self._progress:
|
|
self._progress({"phase": "navigating", "url": url, "title": url})
|
|
tool = PrivateBrowserTool()
|
|
result = await tool.execute(
|
|
json.dumps({"action": "read", "url": url, "timeout_ms": int(timeout * 1000)}),
|
|
{"session_id": self.session_id or "research"},
|
|
)
|
|
output = str(result.get("output") or "").strip()
|
|
if result.get("exit_code") != 0 or not output:
|
|
return ResearchPage(
|
|
url=url,
|
|
success=False,
|
|
retrieval="browser",
|
|
error=str(result.get("error") or output or "browser returned no readable content"),
|
|
)
|
|
self.browser_fetches += 1
|
|
return ResearchPage(
|
|
url=url,
|
|
title=url,
|
|
content=output[:MAX_OUTPUT_CHARS],
|
|
success=True,
|
|
retrieval="browser",
|
|
)
|
|
|
|
@staticmethod
|
|
def _normalize_fetch_page(url: str, page: Any) -> ResearchPage:
|
|
if not isinstance(page, dict):
|
|
return ResearchPage(url=url, success=False, error="fetch returned non-object result")
|
|
content = str(page.get("content") or "").strip()
|
|
return ResearchPage(
|
|
url=url,
|
|
title=str(page.get("title") or ""),
|
|
content=content,
|
|
og_image=str(page.get("og_image") or ""),
|
|
success=bool(page.get("success") and content),
|
|
retrieval="fetch",
|
|
error=str(page.get("error") or ""),
|
|
)
|