From 041e6bed9a0710a981d7b634c5824c0387d01828 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Fri, 17 Jul 2026 02:19:48 +0000 Subject: [PATCH 01/14] feat(models): define provider capability catalog --- src/model_behavior_quirks.py | 297 ++++++ src/model_capabilities.py | 100 ++ src/model_capability_readers/__init__.py | 87 +- src/model_capability_readers/anthropic.py | 63 ++ src/model_capability_readers/base.py | 97 +- .../chatgpt_subscription.py | 62 ++ src/model_capability_readers/cohere.py | 110 +++ src/model_capability_readers/copilot.py | 113 +++ .../generic_openai.py | 235 ++++- src/model_capability_readers/google.py | 1 + src/model_capability_readers/huggingface.py | 89 ++ src/model_capability_readers/lmstudio.py | 1 + src/model_capability_readers/mistral.py | 112 +++ src/model_capability_readers/ollama.py | 18 +- src/model_capability_readers/sglang.py | 100 ++ src/provider_capability_schemas.py | 933 ++++++++++++++++++ tests/test_model_capability_readers.py | 15 +- tests/test_provider_capability_schemas.py | 505 ++++++++++ 18 files changed, 2862 insertions(+), 76 deletions(-) create mode 100644 src/model_behavior_quirks.py create mode 100644 src/model_capability_readers/anthropic.py create mode 100644 src/model_capability_readers/chatgpt_subscription.py create mode 100644 src/model_capability_readers/cohere.py create mode 100644 src/model_capability_readers/copilot.py create mode 100644 src/model_capability_readers/huggingface.py create mode 100644 src/model_capability_readers/mistral.py create mode 100644 src/model_capability_readers/sglang.py create mode 100644 src/provider_capability_schemas.py create mode 100644 tests/test_provider_capability_schemas.py diff --git a/src/model_behavior_quirks.py b/src/model_behavior_quirks.py new file mode 100644 index 000000000..5eb5011ae --- /dev/null +++ b/src/model_behavior_quirks.py @@ -0,0 +1,297 @@ +"""Canonical, exact-match model/provider behavior observations. + +The registry captures behavior that cannot safely be promoted to a provider- +wide capability. Selectors accept already-structured identity (provider, +model ID/family/version, API dialect, and canonical capabilities); they never +extract those facts from a display name with regexes or substring matching. + +This is shape/evidence data only. Runtime request builders can consume it in a +later integration pass after their endpoint has supplied structured identity. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +from src import model_capabilities as mc +from src import provider_capability_schemas as pcs + + +def _identity(value: Any) -> str: + return str(value or "").strip().lower() + + +def _version(value: Any) -> tuple[int, ...]: + if not isinstance(value, (list, tuple)): + return () + out: list[int] = [] + for part in value: + try: + out.append(int(part)) + except (TypeError, ValueError): + return () + return tuple(out) + + +@dataclass(frozen=True) +class ModelBehaviorSelector: + providers: tuple[str, ...] = () + model_ids: tuple[str, ...] = () + model_families: tuple[str, ...] = () + minimum_model_version: tuple[int, ...] = () + minimum_provider_version: tuple[int, ...] = () + api_dialects: tuple[str, ...] = () + required_capabilities: tuple[str, ...] = () + + def matches( + self, + *, + provider: Any, + model_id: Any = "", + model_family: Any = "", + model_version: Any = (), + provider_version: Any = (), + api_dialect: Any = "", + capabilities: Any = (), + ) -> bool: + provider_id = pcs.normalize_provider_id(provider) + if self.providers and provider_id not in self.providers: + return False + + identity_constraints = bool(self.model_ids or self.model_families) + identity_match = ( + _identity(model_id) in self.model_ids + or _identity(model_family) in self.model_families + ) + if identity_constraints and not identity_match: + return False + + actual_model_version = _version(model_version) + if self.minimum_model_version and ( + not actual_model_version or actual_model_version < self.minimum_model_version + ): + return False + actual_provider_version = _version(provider_version) + if self.minimum_provider_version and ( + not actual_provider_version or actual_provider_version < self.minimum_provider_version + ): + return False + if self.api_dialects and str(api_dialect or "").strip() not in self.api_dialects: + return False + + if isinstance(capabilities, Mapping): + capability_values: Iterable[Any] = ( + key for key, enabled in capabilities.items() if enabled is True + ) + elif isinstance(capabilities, str): + capability_values = (capabilities,) + elif isinstance(capabilities, Iterable): + capability_values = capabilities + else: + capability_values = () + normalized_caps = { + normalized + for value in capability_values + if (normalized := mc.normalize_capability(value)) + } + return set(self.required_capabilities).issubset(normalized_caps) + + +@dataclass(frozen=True) +class ModelBehaviorQuirk: + quirk_id: str + selector: ModelBehaviorSelector + request_omit_paths: tuple[str, ...] = () + request_fixed_values: tuple[tuple[str, Any], ...] = () + required_history_paths: tuple[str, ...] = () + response_reasoning_paths: tuple[str, ...] = () + reasoning_controls: tuple[mc.ReasoningControl, ...] = () + status: str = mc.ASSERTION_CLAIMED + source: str = mc.SOURCE_PROVIDER_DOCS_REGISTRY + confidence: str = mc.CONFIDENCE_REGISTRY + evidence_refs: tuple[str, ...] = () + + def to_dict(self) -> dict[str, Any]: + return { + "quirk_id": self.quirk_id, + "selector": { + "providers": list(self.selector.providers), + "model_ids": list(self.selector.model_ids), + "model_families": list(self.selector.model_families), + "minimum_model_version": list(self.selector.minimum_model_version), + "minimum_provider_version": list(self.selector.minimum_provider_version), + "api_dialects": list(self.selector.api_dialects), + "required_capabilities": list(self.selector.required_capabilities), + }, + "request_omit_paths": list(self.request_omit_paths), + "request_fixed_values": dict(self.request_fixed_values), + "required_history_paths": list(self.required_history_paths), + "response_reasoning_paths": list(self.response_reasoning_paths), + "reasoning_controls": [control.to_dict() for control in self.reasoning_controls], + "status": self.status, + "source": self.source, + "confidence": self.confidence, + "evidence_refs": list(self.evidence_refs), + } + + +MODEL_BEHAVIOR_QUIRKS = ( + ModelBehaviorQuirk( + quirk_id="moonshot.kimi-k2.5-k2.6.provider-fixed-temperature", + selector=ModelBehaviorSelector( + providers=("moonshot",), + model_ids=("kimi-k2.5", "kimi-k2.6"), + model_families=("kimi-k2.5", "kimi-k2.6"), + api_dialects=(pcs.DIALECT_OPENAI_CHAT,), + ), + request_omit_paths=("temperature",), + evidence_refs=( + "github:odysseus-dev/odysseus#3960", + "commit:f5d3e509", + ), + ), + ModelBehaviorQuirk( + quirk_id="moonshot.kimi-k2.5-k2.6.tool-history-reasoning-content", + selector=ModelBehaviorSelector( + providers=("moonshot",), + model_ids=("kimi-k2.5", "kimi-k2.6"), + model_families=("kimi-k2.5", "kimi-k2.6"), + api_dialects=(pcs.DIALECT_OPENAI_CHAT,), + ), + required_history_paths=("messages[assistant+tool_calls].reasoning_content",), + response_reasoning_paths=( + "choices[].message.reasoning_content", + "choices[].delta.reasoning_content", + ), + evidence_refs=( + "github:odysseus-dev/odysseus#3118", + "commit:2e6fff22", + ), + ), + ModelBehaviorQuirk( + quirk_id="anthropic.claude-opus-4.7-plus.omit-sampling-controls", + selector=ModelBehaviorSelector( + providers=("anthropic",), + model_families=("claude-opus",), + minimum_model_version=(4, 7), + api_dialects=(pcs.DIALECT_ANTHROPIC_MESSAGES,), + ), + request_omit_paths=("temperature", "top_p", "top_k"), + evidence_refs=( + "github:odysseus-dev/odysseus#3117", + "commit:4f48cfa9", + ), + ), + ModelBehaviorQuirk( + quirk_id="mistral.reasoning.structured-content", + selector=ModelBehaviorSelector( + providers=("mistral",), + model_families=("magistral", "mistral-small", "mistral-medium"), + api_dialects=(pcs.DIALECT_OPENAI_CHAT,), + required_capabilities=(mc.CAP_REASONING,), + ), + response_reasoning_paths=( + "choices[].message.content[type=thinking].thinking[].text", + "choices[].delta.content[type=thinking].thinking[].text", + ), + reasoning_controls=( + mc.ReasoningControl.build( + mechanism=mc.REASONING_CONTROL_EFFORT, + values=(mc.REASONING_CONTROL_VALUE_ON, mc.REASONING_CONTROL_VALUE_OFF), + native_values=("high", "medium", "low", "none"), + request_path="reasoning_effort", + response_paths=("choices[].message.content[type=thinking]",), + status=mc.ASSERTION_CLAIMED, + source=mc.SOURCE_PROVIDER_DOCS_REGISTRY, + confidence=mc.CONFIDENCE_REGISTRY, + ), + ), + evidence_refs=( + "github:odysseus-dev/odysseus#4698", + "commit:bd9149f7", + "https://docs.mistral.ai/capabilities/reasoning/", + ), + ), + ModelBehaviorQuirk( + quirk_id="ollama.native.reasoning-control", + selector=ModelBehaviorSelector( + providers=("ollama",), + model_families=("qwen3", "deepseek-v3.1", "deepseek-r1"), + api_dialects=(pcs.DIALECT_OLLAMA_NATIVE,), + required_capabilities=(mc.CAP_REASONING,), + ), + response_reasoning_paths=("message.thinking", "thinking"), + reasoning_controls=( + mc.ReasoningControl.build( + mechanism=mc.REASONING_CONTROL_NATIVE_BOOL, + values=(mc.REASONING_CONTROL_VALUE_ON, mc.REASONING_CONTROL_VALUE_OFF), + native_values=(True, False), + request_path="think", + response_paths=("message.thinking", "thinking"), + status=mc.ASSERTION_CLAIMED, + source=mc.SOURCE_PROVIDER_DOCS_REGISTRY, + confidence=mc.CONFIDENCE_REGISTRY, + ), + ), + evidence_refs=( + "https://docs.ollama.com/capabilities/thinking", + "github:odysseus-dev/odysseus#3031", + ), + ), + ModelBehaviorQuirk( + quirk_id="ollama.native.gpt-oss-reasoning-level", + selector=ModelBehaviorSelector( + providers=("ollama",), + model_families=("gpt-oss", "gptoss"), + api_dialects=(pcs.DIALECT_OLLAMA_NATIVE,), + required_capabilities=(mc.CAP_REASONING,), + ), + response_reasoning_paths=("message.thinking", "thinking"), + reasoning_controls=( + mc.ReasoningControl.build( + mechanism=mc.REASONING_CONTROL_EFFORT, + values=(mc.REASONING_CONTROL_VALUE_ON,), + native_values=("low", "medium", "high"), + request_path="think", + response_paths=("message.thinking", "thinking"), + status=mc.ASSERTION_CLAIMED, + source=mc.SOURCE_PROVIDER_DOCS_REGISTRY, + confidence=mc.CONFIDENCE_REGISTRY, + ), + ), + evidence_refs=("https://docs.ollama.com/capabilities/thinking",), + ), + ModelBehaviorQuirk( + quirk_id="ollama.openai-compat.0.20.6-reasoning-disable", + selector=ModelBehaviorSelector( + providers=("ollama",), + model_families=("qwen3.5",), + minimum_provider_version=(0, 20, 6), + api_dialects=(pcs.DIALECT_OPENAI_CHAT,), + required_capabilities=(mc.CAP_REASONING,), + ), + request_fixed_values=(("reasoning_effort", "none"),), + status=mc.ASSERTION_CLAIMED, + source=mc.SOURCE_HEURISTIC, + confidence=mc.CONFIDENCE_HEURISTIC, + evidence_refs=("github:odysseus-dev/odysseus#5503",), + ), +) + + +def matching_quirks(**identity: Any) -> tuple[ModelBehaviorQuirk, ...]: + return tuple( + quirk + for quirk in MODEL_BEHAVIOR_QUIRKS + if quirk.selector.matches(**identity) + ) + + +__all__ = [ + "MODEL_BEHAVIOR_QUIRKS", + "ModelBehaviorQuirk", + "ModelBehaviorSelector", + "matching_quirks", +] diff --git a/src/model_capabilities.py b/src/model_capabilities.py index d5b3d8507..2929d68ad 100644 --- a/src/model_capabilities.py +++ b/src/model_capabilities.py @@ -657,6 +657,106 @@ class DeterministicControl: } +@dataclass(frozen=True) +class ReasoningControl: + """A provider/model-supported request mechanism for reasoning. + + This is intentionally separate from :class:`DeterministicControl` and + from the user's on/off/auto preference. The mechanism records the native + request shape that a later resolver may choose after provider and model + evidence have been reconciled. + """ + + mechanism: str = "" + values: tuple[str, ...] = () + native_values: tuple[Any, ...] = () + request_path: str = "" + response_paths: tuple[str, ...] = () + status: str = ASSERTION_UNKNOWN + source: str = SOURCE_UNKNOWN + confidence: str = CONFIDENCE_UNKNOWN + evidence: tuple[tuple[str, Any], ...] = () + tested_at: str = "" + + @classmethod + def build( + cls, + *, + mechanism: Any, + values: Any = None, + native_values: Any = None, + request_path: Any = "", + response_paths: Any = None, + status: Any = ASSERTION_UNKNOWN, + source: Any = SOURCE_UNKNOWN, + confidence: Any = CONFIDENCE_UNKNOWN, + evidence: Mapping[str, Any] | None = None, + tested_at: Any = "", + ) -> "ReasoningControl": + normalized_mechanism = normalize_reasoning_control_mechanism(mechanism) + normalized_status = normalize_assertion_status(status) + if not normalized_mechanism: + normalized_status = ASSERTION_UNKNOWN + if isinstance(response_paths, str): + response_paths = (response_paths,) + elif isinstance(response_paths, Mapping) or not isinstance(response_paths, Iterable): + response_paths = () + if native_values is None: + native_values = () + elif ( + isinstance(native_values, (str, Mapping)) + or not isinstance(native_values, Iterable) + ): + native_values = (native_values,) + return cls( + mechanism=normalized_mechanism, + values=_normalize_tokens(values, normalize_reasoning_control_value), + native_values=tuple(native_values), + request_path=str(request_path or "").strip(), + response_paths=tuple( + str(path or "").strip() + for path in response_paths + if str(path or "").strip() + ), + status=normalized_status, + source=normalize_source(source), + confidence=normalize_confidence(confidence), + evidence=_normalize_limits(evidence), + tested_at=str(tested_at or "").strip(), + ) + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "ReasoningControl": + if not isinstance(value, Mapping): + return cls.build(mechanism="") + return cls.build( + mechanism=value.get("mechanism"), + values=value.get("values"), + native_values=value.get("native_values"), + request_path=value.get("request_path"), + response_paths=value.get("response_paths"), + status=value.get("status"), + source=value.get("source"), + confidence=value.get("confidence"), + evidence=value.get("evidence"), + tested_at=value.get("tested_at"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "mechanism": self.mechanism, + "values": list(self.values), + "native_values": list(self.native_values), + "request_path": self.request_path, + "response_paths": list(self.response_paths), + "status": self.status, + "source": self.source, + "confidence": self.confidence, + "evidence": dict(self.evidence), + "tested_at": self.tested_at, + } + + @dataclass(frozen=True) class CapabilityProbeResult: provider: str diff --git a/src/model_capability_readers/__init__.py b/src/model_capability_readers/__init__.py index 74448281f..a9425c524 100644 --- a/src/model_capability_readers/__init__.py +++ b/src/model_capability_readers/__init__.py @@ -2,24 +2,54 @@ from __future__ import annotations -from collections.abc import Mapping +from dataclasses import replace from typing import Any -from src.model_capability_readers import generic_openai, google, llamacpp, lmstudio, ollama, openai, openrouter +from src import provider_capability_schemas as pcs +from src.model_capability_readers import ( + anthropic, + chatgpt_subscription, + cohere, + copilot, + generic_openai, + google, + huggingface, + llamacpp, + lmstudio, + mistral, + ollama, + openai, + openrouter, + sglang, +) from src.model_capability_readers.base import ( ModelCapabilityRecord, VENDOR_ANTHROPIC, + VENDOR_CEREBRAS, + VENDOR_CHATGPT_SUBSCRIPTION, + VENDOR_COHERE, + VENDOR_COPILOT, + VENDOR_DEEPSEEK, + VENDOR_FIREWORKS, VENDOR_GENERIC_OPENAI, VENDOR_GOOGLE, + VENDOR_GROQ, VENDOR_HUGGINGFACE, VENDOR_LLAMACPP, VENDOR_LMSTUDIO, + VENDOR_MINIMAX, + VENDOR_MISTRAL, + VENDOR_MOONSHOT, + VENDOR_NVIDIA, VENDOR_OLLAMA, VENDOR_OPENAI, VENDOR_OPENROUTER, VENDOR_SGLANG, + VENDOR_TOGETHER, VENDOR_UNKNOWN, VENDOR_VLLM, + VENDOR_XAI, + VENDOR_ZAI, detect_vendor, stable_model_id_for, ) @@ -30,46 +60,69 @@ READER_MODULES = { VENDOR_OPENAI: openai, VENDOR_OPENROUTER: openrouter, VENDOR_GOOGLE: google, + VENDOR_ANTHROPIC: anthropic, VENDOR_LLAMACPP: llamacpp, VENDOR_OLLAMA: ollama, VENDOR_LMSTUDIO: lmstudio, + VENDOR_MISTRAL: mistral, + VENDOR_COPILOT: copilot, + VENDOR_CHATGPT_SUBSCRIPTION: chatgpt_subscription, + VENDOR_COHERE: cohere, + VENDOR_SGLANG: sglang, + VENDOR_HUGGINGFACE: huggingface, } PLACEHOLDER_VENDOR_IDS = frozenset( { - VENDOR_ANTHROPIC, - VENDOR_HUGGINGFACE, - VENDOR_SGLANG, VENDOR_VLLM, } ) def reader_for_vendor(vendor: Any): - vendor_id = str(vendor or "").strip().lower().replace("-", "_") + vendor_id = pcs.normalize_provider_id(vendor) return READER_MODULES.get(vendor_id, generic_openai) def records_from_payload( - payload: Mapping[str, Any], + payload: Any, *, vendor: str | None = None, base_url: str = "", endpoint_kind: str = "", endpoint_id: str = "", ) -> tuple[ModelCapabilityRecord, ...]: - vendor_id = vendor or detect_vendor(base_url, endpoint_kind) + resolution = pcs.resolve_provider( + payload, + provider=vendor, + base_url=base_url, + endpoint_kind=endpoint_kind, + ) + vendor_id = resolution.provider_id + if vendor_id == pcs.PROVIDER_UNKNOWN: + vendor_id = detect_vendor(base_url, endpoint_kind) reader = reader_for_vendor(vendor_id) if reader is generic_openai: record_vendor = vendor_id if vendor_id not in {VENDOR_UNKNOWN, ""} else VENDOR_GENERIC_OPENAI - return reader.records_from_payload( + records = reader.records_from_payload( payload, vendor_id=record_vendor, endpoint_id=endpoint_id, base_url=base_url, ) - return reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url) + else: + records = reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url) + shape_id = resolution.catalog_shape.shape_id if resolution.catalog_shape else "" + return tuple( + replace( + record, + provider_schema_id=resolution.schema.provider_id, + catalog_shape_id=shape_id, + provider_resolution=resolution.stage, + ) + for record in records + ) __all__ = [ @@ -77,17 +130,31 @@ __all__ = [ "PLACEHOLDER_VENDOR_IDS", "READER_MODULES", "VENDOR_ANTHROPIC", + "VENDOR_CEREBRAS", + "VENDOR_CHATGPT_SUBSCRIPTION", + "VENDOR_COHERE", + "VENDOR_COPILOT", + "VENDOR_DEEPSEEK", + "VENDOR_FIREWORKS", "VENDOR_GENERIC_OPENAI", "VENDOR_GOOGLE", + "VENDOR_GROQ", "VENDOR_HUGGINGFACE", "VENDOR_LLAMACPP", "VENDOR_LMSTUDIO", + "VENDOR_MINIMAX", + "VENDOR_MISTRAL", + "VENDOR_MOONSHOT", + "VENDOR_NVIDIA", "VENDOR_OLLAMA", "VENDOR_OPENAI", "VENDOR_OPENROUTER", "VENDOR_SGLANG", + "VENDOR_TOGETHER", "VENDOR_UNKNOWN", "VENDOR_VLLM", + "VENDOR_XAI", + "VENDOR_ZAI", "detect_vendor", "reader_for_vendor", "records_from_payload", diff --git a/src/model_capability_readers/anthropic.py b/src/model_capability_readers/anthropic.py new file mode 100644 index 000000000..4c2376d52 --- /dev/null +++ b/src/model_capability_readers/anthropic.py @@ -0,0 +1,63 @@ +"""Anthropic Models API identity reader. + +The current Model resource is availability/identity metadata, not an explicit +per-model capability card, so records stay unknown. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_ANTHROPIC, + compact_str, + model_id_from, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_ANTHROPIC + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = model_id_from(raw, "id") + if not model_id: + return None + return ModelCapabilityRecord( + vendor=VENDOR_ANTHROPIC, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_ANTHROPIC, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=compact_str(raw.get("display_name")) or model_id, + capability=mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ), + raw=raw, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + return tuple( + record + for item in openai_model_items(payload) + if (record := record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)) + ) diff --git a/src/model_capability_readers/base.py b/src/model_capability_readers/base.py index ee17650a6..1aa2e6adc 100644 --- a/src/model_capability_readers/base.py +++ b/src/model_capability_readers/base.py @@ -28,6 +28,20 @@ VENDOR_LLAMACPP = "llamacpp" VENDOR_VLLM = "vllm" VENDOR_SGLANG = "sglang" VENDOR_HUGGINGFACE = "huggingface" +VENDOR_MISTRAL = "mistral" +VENDOR_COPILOT = "copilot" +VENDOR_CHATGPT_SUBSCRIPTION = "chatgpt_subscription" +VENDOR_COHERE = "cohere" +VENDOR_MINIMAX = "minimax" +VENDOR_MOONSHOT = "moonshot" +VENDOR_GROQ = "groq" +VENDOR_NVIDIA = "nvidia" +VENDOR_CEREBRAS = "cerebras" +VENDOR_DEEPSEEK = "deepseek" +VENDOR_TOGETHER = "together" +VENDOR_FIREWORKS = "fireworks" +VENDOR_XAI = "xai" +VENDOR_ZAI = "zai" VENDOR_UNKNOWN = "unknown" @@ -40,6 +54,14 @@ class ModelCapabilityRecord: stable_model_id: str = "" capability_assertions: tuple[mc.CapabilityAssertion, ...] = () deterministic_controls: tuple[mc.DeterministicControl, ...] = () + reasoning_controls: tuple[mc.ReasoningControl, ...] = () + model_family: str = "" + model_version: tuple[int, ...] = () + provider_version: tuple[int, ...] = () + api_dialect: str = "" + provider_schema_id: str = "" + catalog_shape_id: str = "" + provider_resolution: str = "" raw: Mapping[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: @@ -66,6 +88,14 @@ class ModelCapabilityRecord: "capability": self.capability.to_dict(), "capability_assertions": [assertion.to_dict() for assertion in self.capability_assertions], "deterministic_controls": [control.to_dict() for control in self.deterministic_controls], + "reasoning_controls": [control.to_dict() for control in self.reasoning_controls], + "model_family": self.model_family, + "model_version": list(self.model_version), + "provider_version": list(self.provider_version), + "api_dialect": self.api_dialect, + "provider_schema_id": self.provider_schema_id, + "catalog_shape_id": self.catalog_shape_id, + "provider_resolution": self.provider_resolution, } if include_raw: data["raw"] = dict(self.raw) @@ -77,7 +107,7 @@ class CapabilityReader(Protocol): def records_from_payload( self, - payload: Mapping[str, Any], + payload: Any, *, endpoint_id: Any = "", base_url: Any = "", @@ -168,11 +198,14 @@ def deterministic_controls_from_supported_parameters(values: Any) -> tuple[mc.De ) -def openai_model_items(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]: - payload = as_mapping(payload) - data = payload.get("data") - if data is None: - data = payload.get("models") +def openai_model_items(payload: Any) -> tuple[Mapping[str, Any], ...]: + if isinstance(payload, (list, tuple)): + data = payload + else: + payload = as_mapping(payload) + data = payload.get("data") + if data is None: + data = payload.get("models") return tuple(item for item in as_list(data) if isinstance(item, Mapping)) @@ -254,6 +287,7 @@ def build_capability( output_modalities: Iterable[str] = (), capabilities: Iterable[str] = (), limits: Mapping[str, Any] | None = None, + source: str = mc.SOURCE_PROVIDER_READER, confidence: str = mc.CONFIDENCE_PROVIDER_REPORTED, ) -> mc.ModelCapability: return mc.ModelCapability.build( @@ -263,49 +297,22 @@ def build_capability( output_modalities=tuple(output_modalities), capabilities=tuple(capabilities), limits=limits, - source=mc.SOURCE_PROVIDER_READER, + source=source, confidence=confidence, ) def detect_vendor(base_url: Any = "", endpoint_kind: Any = "") -> str: - kind = compact_str(endpoint_kind).lower().replace("-", "_") - kind_map = { - "openai": VENDOR_OPENAI, - "openrouter": VENDOR_OPENROUTER, - "google": VENDOR_GOOGLE, - "gemini": VENDOR_GOOGLE, - "anthropic": VENDOR_ANTHROPIC, - "ollama": VENDOR_OLLAMA, - "lmstudio": VENDOR_LMSTUDIO, - "lm_studio": VENDOR_LMSTUDIO, - "llamacpp": VENDOR_LLAMACPP, - "llama_cpp": VENDOR_LLAMACPP, - "vllm": VENDOR_VLLM, - "sglang": VENDOR_SGLANG, - "huggingface": VENDOR_HUGGINGFACE, - "hf": VENDOR_HUGGINGFACE, - } - if kind in kind_map: - return kind_map[kind] + # Import lazily to keep the reader primitives independent of registry load + # order. Exact endpoint kind and host identity are authoritative enough + # for provider selection; default ports are not. + from src import provider_capability_schemas as pcs + resolution = pcs.resolve_provider( + endpoint_kind=endpoint_kind, + base_url=base_url, + ) + if resolution.provider_id != pcs.PROVIDER_UNKNOWN: + return resolution.provider_id parsed = urlparse(compact_str(base_url)) - host = (parsed.hostname or "").lower() - port = parsed.port - if host.endswith("openrouter.ai"): - return VENDOR_OPENROUTER - if host.endswith("openai.com"): - return VENDOR_OPENAI - if host.endswith("anthropic.com"): - return VENDOR_ANTHROPIC - if host.endswith("googleapis.com"): - return VENDOR_GOOGLE - if host.endswith("ollama.com") or port == 11434: - return VENDOR_OLLAMA - if port == 1234: - return VENDOR_LMSTUDIO - if port == 8000: - return VENDOR_VLLM - if port == 30000: - return VENDOR_SGLANG - return VENDOR_GENERIC_OPENAI if host else VENDOR_UNKNOWN + return VENDOR_GENERIC_OPENAI if parsed.hostname else VENDOR_UNKNOWN diff --git a/src/model_capability_readers/chatgpt_subscription.py b/src/model_capability_readers/chatgpt_subscription.py new file mode 100644 index 000000000..81400f1b9 --- /dev/null +++ b/src/model_capability_readers/chatgpt_subscription.py @@ -0,0 +1,62 @@ +"""ChatGPT Subscription Codex model-list identity reader.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_CHATGPT_SUBSCRIPTION, + as_list, + as_mapping, + compact_str, + stable_model_id_for, +) + + +vendor = VENDOR_CHATGPT_SUBSCRIPTION + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = compact_str(raw.get("slug")) + if not model_id: + return None + return ModelCapabilityRecord( + vendor=VENDOR_CHATGPT_SUBSCRIPTION, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_CHATGPT_SUBSCRIPTION, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=compact_str(raw.get("display_name") or raw.get("title")) or model_id, + capability=mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ), + model_family=compact_str(raw.get("family")), + raw=raw, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + values = as_mapping(payload).get("models") + return tuple( + record + for item in as_list(values) + if isinstance(item, Mapping) + if (record := record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)) + ) diff --git a/src/model_capability_readers/cohere.py b/src/model_capability_readers/cohere.py new file mode 100644 index 000000000..e4108fc04 --- /dev/null +++ b/src/model_capability_readers/cohere.py @@ -0,0 +1,110 @@ +"""Cohere native model-catalog capability reader. + +The `/v1/models` resource reports endpoint compatibility and context size per +model. It does not prove provider-wide chat/tool support for every model, so +the reader maps only those exact model-card fields. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_COHERE, + as_list, + as_mapping, + build_capability, + compact_str, + deterministic_controls_from_supported_parameters, + int_limit, + stable_model_id_for, +) + + +vendor = VENDOR_COHERE + +_ENDPOINT_FAMILIES = { + "chat": mc.FAMILY_CHAT, + "generate": mc.FAMILY_CHAT, + "embed": mc.FAMILY_EMBEDDING, + "rerank": mc.FAMILY_RERANK, + "classify": mc.FAMILY_CLASSIFICATION, +} + + +def _family(raw: Mapping[str, Any]) -> str: + families = { + family + for value in as_list(raw.get("endpoints")) + if (family := _ENDPOINT_FAMILIES.get(compact_str(value).lower())) + } + return next(iter(families)) if len(families) == 1 else mc.FAMILY_UNKNOWN + + +def _modalities(family: str) -> tuple[tuple[str, ...], tuple[str, ...]]: + if family == mc.FAMILY_CHAT: + return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,) + if family == mc.FAMILY_EMBEDDING: + return (mc.MODALITY_TEXT,), (mc.MODALITY_EMBEDDING,) + if family in {mc.FAMILY_RERANK, mc.FAMILY_CLASSIFICATION}: + return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,) + return (), () + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = compact_str(raw.get("name")) + if not model_id: + return None + family = _family(raw) + inputs, outputs = _modalities(family) + context_tokens = int_limit(raw.get("context_length")) + limits = {"context_tokens": context_tokens} if context_tokens else {} + sampling_defaults = as_mapping(raw.get("sampling_defaults")) + sampling_controls = ( + "top_p" if key == "p" else "top_k" if key == "k" else key + for key in sampling_defaults + ) + return ModelCapabilityRecord( + vendor=VENDOR_COHERE, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_COHERE, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=model_id, + capability=build_capability( + family=family, + input_modalities=inputs, + output_modalities=outputs, + limits=limits, + ), + deterministic_controls=deterministic_controls_from_supported_parameters( + sampling_controls + ), + raw=raw, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + values = as_mapping(payload).get("models") + return tuple( + record + for item in as_list(values) + if isinstance(item, Mapping) + if (record := record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)) + ) diff --git a/src/model_capability_readers/copilot.py b/src/model_capability_readers/copilot.py new file mode 100644 index 000000000..99920d0a0 --- /dev/null +++ b/src/model_capability_readers/copilot.py @@ -0,0 +1,113 @@ +"""GitHub Copilot model-catalog capability reader.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_COPILOT, + as_mapping, + build_capability, + compact_str, + int_limit, + merge_unique, + model_id_from, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_COPILOT + +_SUPPORT_CAPABILITIES = { + "tool_calls": mc.CAP_TOOL_CALL, + "tools": mc.CAP_TOOL_CALL, + "vision": mc.CAP_VISION, + "reasoning": mc.CAP_REASONING, + "structured_outputs": mc.CAP_STRUCTURED_OUTPUT, +} + + +def _supports(raw: Mapping[str, Any]) -> Mapping[str, Any]: + return as_mapping(as_mapping(raw.get("capabilities")).get("supports")) + + +def _limits(raw: Mapping[str, Any]) -> dict[str, int]: + payload = as_mapping(raw.get("limits")) + out: dict[str, int] = {} + for keys, target in ( + (("max_prompt_tokens", "input_tokens"), "input_tokens"), + (("max_output_tokens", "output_tokens"), "output_tokens"), + (("max_context_tokens", "context_window"), "context_tokens"), + ): + for key in keys: + value = int_limit(payload.get(key)) or int_limit(raw.get(key)) + if value: + out[target] = value + break + return out + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = model_id_from(raw, "id") + if not model_id: + return None + supports = _supports(raw) + capabilities = merge_unique( + _SUPPORT_CAPABILITIES[key] + for key, enabled in supports.items() + if enabled is True and key in _SUPPORT_CAPABILITIES + ) + picker_enabled = raw.get("model_picker_enabled") is True + if picker_enabled or capabilities: + inputs = [mc.MODALITY_TEXT] + if mc.CAP_VISION in capabilities: + inputs.append(mc.MODALITY_IMAGE) + capability = build_capability( + family=mc.FAMILY_CHAT, + input_modalities=inputs, + output_modalities=(mc.MODALITY_TEXT,), + capabilities=capabilities, + limits=_limits(raw), + ) + else: + capability = mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ) + return ModelCapabilityRecord( + vendor=VENDOR_COPILOT, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_COPILOT, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=compact_str(raw.get("name")) or model_id, + capability=capability, + model_family=compact_str(raw.get("family")), + raw=raw, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + records: list[ModelCapabilityRecord] = [] + for item in openai_model_items(payload): + record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url) + if record: + records.append(record) + return tuple(records) diff --git a/src/model_capability_readers/generic_openai.py b/src/model_capability_readers/generic_openai.py index edff3ad33..5759405b2 100644 --- a/src/model_capability_readers/generic_openai.py +++ b/src/model_capability_readers/generic_openai.py @@ -1,4 +1,10 @@ -"""Reader for bare OpenAI-compatible model-list payloads.""" +"""General structural reader for OpenAI-compatible model-list payloads. + +Identity-only model cards remain unknown. Rich records are promoted only from +recognized explicit fields (modalities, task/type, capability booleans, +supported parameters, and numeric limits). Names and descriptions are never +parsed for capability hints. +""" from __future__ import annotations @@ -9,9 +15,18 @@ from src import model_capabilities as mc from src.model_capability_readers.base import ( ModelCapabilityRecord, VENDOR_GENERIC_OPENAI, + as_list, + as_mapping, + build_capability, compact_str, + deterministic_controls_from_supported_parameters, + family_from_modalities, + int_limit, + merge_unique, model_id_from, + modalities_from_value, openai_model_items, + split_modality_arrow, stable_model_id_for, ) @@ -19,6 +34,191 @@ from src.model_capability_readers.base import ( vendor = VENDOR_GENERIC_OPENAI +_TYPE_FAMILIES = { + "llm": mc.FAMILY_CHAT, + "chat": mc.FAMILY_CHAT, + "chat_completion": mc.FAMILY_CHAT, + "text_generation": mc.FAMILY_CHAT, + "causal_lm": mc.FAMILY_CHAT, + "image_text_to_text": mc.FAMILY_CHAT, + "image_question_answering": mc.FAMILY_CHAT, + "embedding": mc.FAMILY_EMBEDDING, + "embeddings": mc.FAMILY_EMBEDDING, + "text_embedding": mc.FAMILY_EMBEDDING, + "feature_extraction": mc.FAMILY_EMBEDDING, + "text_to_image": mc.FAMILY_IMAGE, + "image_to_image": mc.FAMILY_IMAGE, + "text_to_video": mc.FAMILY_VIDEO, + "automatic_speech_recognition": mc.FAMILY_AUDIO, + "text_to_speech": mc.FAMILY_AUDIO, + "rerank": mc.FAMILY_RERANK, + "reranking": mc.FAMILY_RERANK, + "classification": mc.FAMILY_CLASSIFICATION, + "text_classification": mc.FAMILY_CLASSIFICATION, + "moderation": mc.FAMILY_MODERATION, +} + +_PARAMETER_CAPABILITIES = { + "tools": mc.CAP_TOOL_CALL, + "tool_choice": mc.CAP_TOOL_CALL, + "parallel_tool_calls": mc.CAP_TOOL_CALL, + "function_calling": mc.CAP_TOOL_CALL, + "response_format": mc.CAP_JSON_MODE, + "structured_output": mc.CAP_STRUCTURED_OUTPUT, + "structured_outputs": mc.CAP_STRUCTURED_OUTPUT, + "json_schema": mc.CAP_STRUCTURED_OUTPUT, + "reasoning": mc.CAP_REASONING, + "reasoning_effort": mc.CAP_REASONING, + "include_reasoning": mc.CAP_REASONING, + "web_search": mc.CAP_WEB_SEARCH, + "web_search_options": mc.CAP_WEB_SEARCH, +} + + +def _shape_token(value: Any) -> str: + return compact_str(value).lower().replace("-", "_").replace(" ", "_") + + +def _family_from_explicit_fields(raw: Mapping[str, Any]) -> str: + for key in ("type", "model_type", "task", "pipeline_tag"): + family = _TYPE_FAMILIES.get(_shape_token(raw.get(key))) + if family: + return family + return mc.FAMILY_UNKNOWN + + +def _modalities(raw: Mapping[str, Any]) -> tuple[tuple[str, ...], tuple[str, ...]]: + architecture = as_mapping(raw.get("architecture")) + input_modalities = modalities_from_value( + raw.get("input_modalities") or architecture.get("input_modalities") + ) + output_modalities = modalities_from_value( + raw.get("output_modalities") or architecture.get("output_modalities") + ) + if not input_modalities or not output_modalities: + arrow_input, arrow_output = split_modality_arrow( + raw.get("modality") or architecture.get("modality") + ) + input_modalities = input_modalities or arrow_input + output_modalities = output_modalities or arrow_output + return input_modalities, output_modalities + + +def _capabilities_from_modalities( + input_modalities: tuple[str, ...], + output_modalities: tuple[str, ...], +) -> tuple[str, ...]: + input_set = set(input_modalities) + output_set = set(output_modalities) + out: list[str] = [] + if mc.MODALITY_IMAGE in input_set and mc.MODALITY_TEXT in output_set: + out.append(mc.CAP_VISION) + if mc.MODALITY_FILE in input_set: + out.append(mc.CAP_FILES) + if mc.MODALITY_PDF in input_set: + out.append(mc.CAP_PDF) + if mc.MODALITY_AUDIO in input_set: + out.append(mc.CAP_AUDIO_INPUT) + if mc.MODALITY_AUDIO in output_set: + out.append(mc.CAP_AUDIO_OUTPUT) + if mc.MODALITY_IMAGE in output_set: + out.append(mc.CAP_IMAGE_GENERATION) + if mc.MODALITY_IMAGE in input_set: + out.append(mc.CAP_IMAGE_EDITING) + if mc.MODALITY_VIDEO in output_set: + out.append(mc.CAP_VIDEO_GENERATION) + return tuple(out) + + +def _explicit_capabilities(raw: Mapping[str, Any]) -> tuple[str, ...]: + values: list[Any] = [] + payload = raw.get("capabilities") + if isinstance(payload, Mapping): + supports = payload.get("supports") + if isinstance(supports, Mapping): + values.extend(key for key, enabled in supports.items() if enabled is True) + values.extend(key for key, enabled in payload.items() if enabled is True) + elif isinstance(payload, (list, tuple)): + values.extend(payload) + + out: list[str] = [] + for value in values: + cap = mc.normalize_capability(value) + if cap and cap not in out: + out.append(cap) + for value in as_list(raw.get("supported_parameters")): + cap = _PARAMETER_CAPABILITIES.get(_shape_token(value)) + if cap and cap not in out: + out.append(cap) + task = next( + (_shape_token(raw.get(key)) for key in ("type", "model_type", "task", "pipeline_tag") if raw.get(key)), + "", + ) + task_capability = { + "automatic_speech_recognition": mc.CAP_TRANSCRIPTION, + "text_to_speech": mc.CAP_TTS, + "text_to_image": mc.CAP_IMAGE_GENERATION, + "image_to_image": mc.CAP_IMAGE_EDITING, + "text_to_video": mc.CAP_VIDEO_GENERATION, + "image_text_to_text": mc.CAP_VISION, + "image_question_answering": mc.CAP_VISION, + }.get(task) + if task_capability and task_capability not in out: + out.append(task_capability) + return tuple(out) + + +def _limits(raw: Mapping[str, Any]) -> dict[str, int]: + architecture = as_mapping(raw.get("architecture")) + top_provider = as_mapping(raw.get("top_provider")) + limits: dict[str, int] = {} + for keys, target in ( + (("context_length", "max_context_length", "max_model_len"), "context_tokens"), + (("input_token_limit", "inputTokenLimit"), "input_tokens"), + (("output_token_limit", "outputTokenLimit", "max_completion_tokens"), "output_tokens"), + ): + for key in keys: + value = int_limit(raw.get(key)) or int_limit(architecture.get(key)) or int_limit(top_provider.get(key)) + if value: + limits[target] = value + break + return limits + + +def _default_modalities( + family: str, + raw: Mapping[str, Any], +) -> tuple[tuple[str, ...], tuple[str, ...]]: + task = next( + (_shape_token(raw.get(key)) for key in ("type", "model_type", "task", "pipeline_tag") if raw.get(key)), + "", + ) + task_modalities = { + "automatic_speech_recognition": ((mc.MODALITY_AUDIO,), (mc.MODALITY_TEXT,)), + "text_to_speech": ((mc.MODALITY_TEXT,), (mc.MODALITY_AUDIO,)), + "text_to_image": ((mc.MODALITY_TEXT,), (mc.MODALITY_IMAGE,)), + "image_to_image": ((mc.MODALITY_IMAGE,), (mc.MODALITY_IMAGE,)), + "text_to_video": ((mc.MODALITY_TEXT,), (mc.MODALITY_VIDEO,)), + "image_text_to_text": ((mc.MODALITY_TEXT, mc.MODALITY_IMAGE), (mc.MODALITY_TEXT,)), + "image_question_answering": ((mc.MODALITY_TEXT, mc.MODALITY_IMAGE), (mc.MODALITY_TEXT,)), + }.get(task) + if task_modalities: + return task_modalities + if family == mc.FAMILY_CHAT: + return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,) + if family == mc.FAMILY_EMBEDDING: + return (mc.MODALITY_TEXT,), (mc.MODALITY_EMBEDDING,) + if family == mc.FAMILY_IMAGE: + return (mc.MODALITY_TEXT,), (mc.MODALITY_IMAGE,) + if family == mc.FAMILY_VIDEO: + return (mc.MODALITY_TEXT,), (mc.MODALITY_VIDEO,) + if family == mc.FAMILY_AUDIO: + return (), () + if family in {mc.FAMILY_RERANK, mc.FAMILY_CLASSIFICATION, mc.FAMILY_MODERATION}: + return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,) + return (), () + + def record_from_model( raw: Mapping[str, Any], *, @@ -29,22 +229,47 @@ def record_from_model( model_id = model_id_from(raw, "id", "name", "model") if not model_id: return None - capability = mc.unknown_capability( - source=mc.SOURCE_PROVIDER_READER, - confidence=mc.CONFIDENCE_UNKNOWN, + + family = _family_from_explicit_fields(raw) + input_modalities, output_modalities = _modalities(raw) + if family == mc.FAMILY_UNKNOWN: + family = family_from_modalities(input_modalities, output_modalities) + if family != mc.FAMILY_UNKNOWN and not input_modalities and not output_modalities: + input_modalities, output_modalities = _default_modalities(family, raw) + capabilities = merge_unique( + _explicit_capabilities(raw), + _capabilities_from_modalities(input_modalities, output_modalities), ) + limits = _limits(raw) + if family == mc.FAMILY_UNKNOWN and not capabilities and not limits: + capability = mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ) + else: + capability = build_capability( + family=family, + input_modalities=input_modalities, + output_modalities=output_modalities, + capabilities=capabilities, + limits=limits, + ) return ModelCapabilityRecord( vendor=vendor_id, model_id=model_id, stable_model_id=stable_model_id_for(vendor_id, model_id, endpoint_id=endpoint_id, base_url=base_url), display_name=compact_str(raw.get("display_name") or raw.get("name")), capability=capability, + deterministic_controls=deterministic_controls_from_supported_parameters( + raw.get("supported_parameters") + ), + model_family=compact_str(raw.get("root") or raw.get("model_family")), raw=raw, ) def records_from_payload( - payload: Mapping[str, Any], + payload: Any, *, vendor_id: str = VENDOR_GENERIC_OPENAI, endpoint_id: Any = "", diff --git a/src/model_capability_readers/google.py b/src/model_capability_readers/google.py index 9edb57bdb..f6d0bfe28 100644 --- a/src/model_capability_readers/google.py +++ b/src/model_capability_readers/google.py @@ -42,6 +42,7 @@ def record_from_model( display_name=compact_str(raw.get("displayName")) or model_id, capability=ai_studio.capability_from_model(raw), deterministic_controls=ai_studio.deterministic_controls_from_model(raw), + model_family=compact_str(raw.get("baseModelId")), raw=raw, ) diff --git a/src/model_capability_readers/huggingface.py b/src/model_capability_readers/huggingface.py new file mode 100644 index 000000000..edb34d471 --- /dev/null +++ b/src/model_capability_readers/huggingface.py @@ -0,0 +1,89 @@ +"""Hugging Face Hub model-info reader using explicit pipeline metadata.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers import generic_openai +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_HUGGINGFACE, + as_mapping, + compact_str, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_HUGGINGFACE + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = compact_str(raw.get("modelId") or raw.get("id")) + if not model_id: + return None + structural = generic_openai.record_from_model( + {**raw, "id": model_id}, + vendor_id=VENDOR_HUGGINGFACE, + endpoint_id=endpoint_id, + base_url=base_url, + ) + if not structural: + return None + capability = mc.ModelCapability.build( + family=structural.capability.family, + primary_task=structural.capability.primary_task, + input_modalities=structural.capability.modalities.input, + output_modalities=structural.capability.modalities.output, + capabilities=structural.capability.capabilities, + limits=dict(structural.capability.limits), + source=mc.SOURCE_COOKBOOK_HF, + confidence=mc.CONFIDENCE_REGISTRY, + ) + config = as_mapping(raw.get("config")) + return ModelCapabilityRecord( + vendor=VENDOR_HUGGINGFACE, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_HUGGINGFACE, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=( + compact_str( + raw.get("cardData", {}).get("pretty_name") + if isinstance(raw.get("cardData"), Mapping) + else "" + ) + or model_id + ), + capability=capability, + deterministic_controls=structural.deterministic_controls, + model_family=compact_str(config.get("model_type")), + raw=raw, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + if isinstance(payload, Mapping) and (payload.get("modelId") or payload.get("pipeline_tag")): + record = record_from_model(payload, endpoint_id=endpoint_id, base_url=base_url) + return (record,) if record else () + records: list[ModelCapabilityRecord] = [] + for item in openai_model_items(payload): + record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url) + if record: + records.append(record) + return tuple(records) diff --git a/src/model_capability_readers/lmstudio.py b/src/model_capability_readers/lmstudio.py index 960649959..919055936 100644 --- a/src/model_capability_readers/lmstudio.py +++ b/src/model_capability_readers/lmstudio.py @@ -160,6 +160,7 @@ def record_from_native_model( ), display_name=compact_str(raw.get("display_name") or raw.get("name")) or model_id, capability=capability, + model_family=compact_str(raw.get("architecture") or raw.get("arch")), raw=raw, ) diff --git a/src/model_capability_readers/mistral.py b/src/model_capability_readers/mistral.py new file mode 100644 index 000000000..6a316169a --- /dev/null +++ b/src/model_capability_readers/mistral.py @@ -0,0 +1,112 @@ +"""Mistral native model-catalog capability reader.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_MISTRAL, + as_mapping, + build_capability, + compact_str, + int_limit, + merge_unique, + model_id_from, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_MISTRAL + + +def _family(raw: Mapping[str, Any]) -> str: + capabilities = as_mapping(raw.get("capabilities")) + if capabilities.get("classification") is True and not ( + capabilities.get("completion_chat") is True + or capabilities.get("completion_fim") is True + ): + return mc.FAMILY_CLASSIFICATION + if capabilities.get("completion_chat") is True or capabilities.get("completion_fim") is True: + return mc.FAMILY_CHAT + return mc.FAMILY_UNKNOWN + + +def _capabilities(raw: Mapping[str, Any]) -> tuple[str, ...]: + payload = as_mapping(raw.get("capabilities")) + values: list[str] = [] + for key, capability in ( + ("vision", mc.CAP_VISION), + ("function_calling", mc.CAP_TOOL_CALL), + ("reasoning", mc.CAP_REASONING), + ("structured_output", mc.CAP_STRUCTURED_OUTPUT), + ("structured_outputs", mc.CAP_STRUCTURED_OUTPUT), + ): + if payload.get(key) is True: + values.append(capability) + return merge_unique(values) + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = model_id_from(raw, "id") + if not model_id: + return None + family = _family(raw) + capabilities = _capabilities(raw) + if family == mc.FAMILY_CHAT: + inputs = [mc.MODALITY_TEXT] + if mc.CAP_VISION in capabilities: + inputs.append(mc.MODALITY_IMAGE) + input_modalities = tuple(inputs) + output_modalities = (mc.MODALITY_TEXT,) + elif family == mc.FAMILY_CLASSIFICATION: + input_modalities = (mc.MODALITY_TEXT,) + output_modalities = (mc.MODALITY_TEXT,) + else: + input_modalities = () + output_modalities = () + context_tokens = int_limit(raw.get("max_context_length")) + limits = {"context_tokens": context_tokens} if context_tokens else {} + capability = build_capability( + family=family, + input_modalities=input_modalities, + output_modalities=output_modalities, + capabilities=capabilities, + limits=limits, + ) + return ModelCapabilityRecord( + vendor=VENDOR_MISTRAL, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_MISTRAL, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=compact_str(raw.get("name")) or model_id, + capability=capability, + model_family=compact_str(raw.get("root")), + raw=raw, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + records: list[ModelCapabilityRecord] = [] + for item in openai_model_items(payload): + record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url) + if record: + records.append(record) + return tuple(records) diff --git a/src/model_capability_readers/ollama.py b/src/model_capability_readers/ollama.py index 8ad7a5e54..cd21f6c0e 100644 --- a/src/model_capability_readers/ollama.py +++ b/src/model_capability_readers/ollama.py @@ -59,17 +59,11 @@ def _family_from_ollama_capabilities(values: Any) -> str: def _parameters_mapping(value: Any) -> Mapping[str, Any]: - if isinstance(value, Mapping): - return value - text = compact_str(value) - if not text: - return {} - parsed: dict[str, str] = {} - for line in text.splitlines(): - parts = line.strip().split(None, 1) - if len(parts) == 2: - parsed[parts[0]] = parts[1] - return parsed + # `/api/show` currently serializes this field as Modelfile text. Do not + # recover capability truth by reparsing that late text; prefer the native + # structured `model_info.*.context_length` shape. Mapping support remains + # for compatible servers that already return structured parameters. + return value if isinstance(value, Mapping) else {} def _modalities_for_family(family: str, capabilities: tuple[str, ...]) -> tuple[tuple[str, ...], tuple[str, ...]]: @@ -148,6 +142,7 @@ def record_from_show_payload( stable_model_id=stable_model_id_for(VENDOR_OLLAMA, model_id, endpoint_id=endpoint_id, base_url=base_url), display_name=model_id, capability=capability, + model_family=compact_str(as_mapping(payload.get("details")).get("family")), raw=payload, ) @@ -180,6 +175,7 @@ def records_from_tags_payload( source=mc.SOURCE_PROVIDER_READER, confidence=mc.CONFIDENCE_UNKNOWN, ), + model_family=compact_str(as_mapping(item.get("details")).get("family")), raw=item, ) ) diff --git a/src/model_capability_readers/sglang.py b/src/model_capability_readers/sglang.py new file mode 100644 index 000000000..5b6b96c63 --- /dev/null +++ b/src/model_capability_readers/sglang.py @@ -0,0 +1,100 @@ +"""SGLang `/model_info` and OpenAI model-card reader.""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import PurePosixPath +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers import generic_openai +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_SGLANG, + as_mapping, + build_capability, + compact_str, + deterministic_controls_from_supported_parameters, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_SGLANG + + +def _model_id(payload: Mapping[str, Any]) -> str: + value = compact_str(payload.get("served_model_name") or payload.get("model_path")) + if not value: + return "" + return PurePosixPath(value).name if value.startswith("/") else value + + +def record_from_model_info( + payload: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = _model_id(payload) + if not model_id: + return None + capabilities: list[str] = [] + inputs: list[str] = [] + outputs: list[str] = [] + family = mc.FAMILY_UNKNOWN + if payload.get("is_generation") is True: + family = mc.FAMILY_CHAT + inputs.append(mc.MODALITY_TEXT) + outputs.append(mc.MODALITY_TEXT) + if payload.get("has_image_understanding") is True: + inputs.append(mc.MODALITY_IMAGE) + capabilities.append(mc.CAP_VISION) + if payload.get("has_audio_understanding") is True: + inputs.append(mc.MODALITY_AUDIO) + capabilities.append(mc.CAP_AUDIO_INPUT) + capability = build_capability( + family=family, + input_modalities=inputs, + output_modalities=outputs, + capabilities=capabilities, + ) + sampling = as_mapping(payload.get("preferred_sampling_params")) + return ModelCapabilityRecord( + vendor=VENDOR_SGLANG, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_SGLANG, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=model_id, + capability=capability, + deterministic_controls=deterministic_controls_from_supported_parameters(sampling.keys()), + model_family=compact_str(payload.get("model_type")), + raw=payload, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + mapping = as_mapping(payload) + if "is_generation" in mapping and "model_path" in mapping: + record = record_from_model_info(mapping, endpoint_id=endpoint_id, base_url=base_url) + return (record,) if record else () + records: list[ModelCapabilityRecord] = [] + for item in openai_model_items(payload): + record = generic_openai.record_from_model( + item, + vendor_id=VENDOR_SGLANG, + endpoint_id=endpoint_id, + base_url=base_url, + ) + if record: + records.append(record) + return tuple(records) diff --git a/src/provider_capability_schemas.py b/src/provider_capability_schemas.py new file mode 100644 index 000000000..531cbd19b --- /dev/null +++ b/src/provider_capability_schemas.py @@ -0,0 +1,933 @@ +"""Canonical serving-provider and payload-shape metadata. + +This module describes provider/API dialects and model-catalog JSON shapes. It +does not perform network I/O, parse model names, or claim that every model on a +provider supports every feature exposed by that provider. Model capability is +still read from each model record (or remains unknown). + +Resolution is deliberately stepped and deterministic: + +1. an explicit provider/endpoint kind; +2. an exact known provider host; +3. a discriminating native payload shape; +4. a general structural model-list shape; +5. unknown. + +Unknown keys are left to the reader's ``raw`` evidence. They never become a +capability merely because a future provider happens to add them. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlparse + + +PROVIDER_UNKNOWN = "unknown" +PROVIDER_GENERIC_OPENAI = "generic_openai" + +DIALECT_OPENAI_CHAT = "openai_chat_completions" +DIALECT_OPENAI_RESPONSES = "openai_responses" +DIALECT_ANTHROPIC_MESSAGES = "anthropic_messages" +DIALECT_COHERE_V2 = "cohere_v2" +DIALECT_GOOGLE_GENERATE_CONTENT = "google_generate_content" +DIALECT_OLLAMA_NATIVE = "ollama_native" +DIALECT_LMSTUDIO_NATIVE_V1 = "lmstudio_native_v1" +DIALECT_LLAMACPP_NATIVE = "llamacpp_native" +DIALECT_SGLANG_NATIVE = "sglang_native" +DIALECT_HUGGINGFACE_HUB = "huggingface_hub" +DIALECT_CHATGPT_SUBSCRIPTION = "chatgpt_subscription_responses" + +RESOLUTION_EXPLICIT = "explicit" +RESOLUTION_ENDPOINT_KIND = "endpoint_kind" +RESOLUTION_HOST = "host" +RESOLUTION_NATIVE_SHAPE = "native_shape" +RESOLUTION_GENERAL_SHAPE = "general_shape" +RESOLUTION_UNKNOWN = "unknown" + +ENVELOPE_DATA = "data" +ENVELOPE_MODELS = "models" +ENVELOPE_BARE_LIST = "bare_list" +ENVELOPE_SINGLE = "single" + +_MISSING = object() + + +def _token(value: Any) -> str: + return str(value or "").strip().lower().replace("-", "_").replace(" ", "_") + + +def _path_value(value: Any, path: str) -> Any: + current = value + for part in path.split("."): + if not isinstance(current, Mapping) or part not in current: + return _MISSING + current = current[part] + return current + + +def _path_present(value: Any, path: str) -> bool: + return _path_value(value, path) is not _MISSING + + +def _items_for_envelope(payload: Any, envelope: str) -> tuple[Mapping[str, Any], ...]: + if envelope == ENVELOPE_BARE_LIST: + values = payload if isinstance(payload, (list, tuple)) else () + elif envelope == ENVELOPE_SINGLE: + values = (payload,) if isinstance(payload, Mapping) else () + elif isinstance(payload, Mapping): + values = payload.get(envelope) + values = values if isinstance(values, (list, tuple)) else () + else: + values = () + return tuple(item for item in values if isinstance(item, Mapping)) + + +@dataclass(frozen=True) +class ProviderCatalogShape: + """A declarative, versioned provider model-catalog shape.""" + + shape_id: str + provider_id: str + endpoint_path: str + envelope: str + identity_paths: tuple[str, ...] + required_root_paths: tuple[str, ...] = () + required_item_paths: tuple[str, ...] = () + required_item_any_paths: tuple[str, ...] = () + item_types: tuple[tuple[str, tuple[Any, ...]], ...] = () + item_values: tuple[tuple[str, tuple[Any, ...]], ...] = () + capability_paths: tuple[str, ...] = () + api_version: str = "" + priority: int = 0 + latest: bool = True + + def items(self, payload: Any) -> tuple[Mapping[str, Any], ...]: + return _items_for_envelope(payload, self.envelope) + + def matches(self, payload: Any) -> bool: + if self.required_root_paths: + if not isinstance(payload, Mapping): + return False + if not all(_path_present(payload, path) for path in self.required_root_paths): + return False + + items = self.items(payload) + if not items: + return False + for item in items: + if self.identity_paths: + has_identity = False + for path in self.identity_paths: + value = _path_value(item, path) + if value is not _MISSING and value is not None and value != "": + has_identity = True + break + if not has_identity: + continue + if not all(_path_present(item, path) for path in self.required_item_paths): + continue + if self.required_item_any_paths and not any( + _path_present(item, path) for path in self.required_item_any_paths + ): + continue + if any( + not isinstance(_path_value(item, path), expected_types) + for path, expected_types in self.item_types + ): + continue + if any(_path_value(item, path) not in expected for path, expected in self.item_values): + continue + return True + return False + + +@dataclass(frozen=True) +class ProviderApiShape: + """Stable request/response field paths for one API dialect. + + Paths are documentation and validation inputs, not late response parsers. + A model-specific exception can narrow these fields in the model quirk + registry without changing the provider's general transport contract. + """ + + dialect: str + request_path: str + stream_path: str = "" + model_field: str = "model" + message_field: str = "messages" + tool_request_paths: tuple[str, ...] = () + tool_response_paths: tuple[str, ...] = () + text_response_paths: tuple[str, ...] = () + reasoning_response_paths: tuple[str, ...] = () + request_control_paths: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ProviderCapabilitySchema: + provider_id: str + display_name: str + aliases: tuple[str, ...] = () + host_suffixes: tuple[str, ...] = () + api_shapes: tuple[ProviderApiShape, ...] = () + catalog_shapes: tuple[ProviderCatalogShape, ...] = () + fallback_provider_id: str = PROVIDER_GENERIC_OPENAI + model_capabilities_are_per_model: bool = True + + +@dataclass(frozen=True) +class ProviderResolution: + provider_id: str + stage: str + schema: ProviderCapabilitySchema + catalog_shape: ProviderCatalogShape | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "provider_id": self.provider_id, + "stage": self.stage, + "schema_id": self.schema.provider_id, + "catalog_shape_id": self.catalog_shape.shape_id if self.catalog_shape else "", + } + + +OPENAI_CHAT_SHAPE = ProviderApiShape( + dialect=DIALECT_OPENAI_CHAT, + request_path="/v1/chat/completions", + stream_path="/v1/chat/completions", + tool_request_paths=("tools[].function", "tool_choice", "parallel_tool_calls"), + tool_response_paths=("choices[].message.tool_calls[].function", "choices[].delta.tool_calls[].function"), + text_response_paths=("choices[].message.content", "choices[].delta.content"), + reasoning_response_paths=( + "choices[].message.reasoning_content", + "choices[].delta.reasoning_content", + "choices[].delta.reasoning", + "choices[].delta.thinking", + ), + request_control_paths=( + "temperature", + "top_p", + "seed", + "response_format", + "reasoning_effort", + ), +) + +OPENAI_RESPONSES_SHAPE = ProviderApiShape( + dialect=DIALECT_OPENAI_RESPONSES, + request_path="/v1/responses", + stream_path="/v1/responses", + message_field="input", + tool_request_paths=("tools[]", "tool_choice", "parallel_tool_calls"), + tool_response_paths=( + "output[type=function_call].arguments", + "response.function_call_arguments.delta", + ), + text_response_paths=( + "output[type=message].content[type=output_text].text", + "response.output_text.delta", + ), + reasoning_response_paths=( + "output[type=reasoning].summary[].text", + "output[type=reasoning].encrypted_content", + "response.reasoning_summary_text.delta", + ), + request_control_paths=("temperature", "top_p", "reasoning", "text.format"), +) + +ANTHROPIC_MESSAGES_SHAPE = ProviderApiShape( + dialect=DIALECT_ANTHROPIC_MESSAGES, + request_path="/v1/messages", + stream_path="/v1/messages", + tool_request_paths=("tools[].input_schema", "tool_choice", "messages[].content[].tool_result"), + tool_response_paths=("content[].tool_use", "content_block_delta.delta.partial_json"), + text_response_paths=("content[].text", "content_block_delta.delta.text"), + reasoning_response_paths=("content[].thinking", "content[].signature"), + request_control_paths=("temperature", "top_p", "top_k", "thinking", "output_config"), +) + +COHERE_V2_SHAPE = ProviderApiShape( + dialect=DIALECT_COHERE_V2, + request_path="/v2/chat", + stream_path="/v2/chat", + tool_request_paths=("tools[].parameters", "messages[].tool_calls", "messages[].tool_results"), + tool_response_paths=("message.tool_calls", "tool-call-start", "tool-call-delta", "tool-call-end"), + text_response_paths=("message.content[].text", "content-delta.delta.message.content.text"), + reasoning_response_paths=("message.content[].thinking",), + request_control_paths=( + "temperature", + "p", + "k", + "seed", + "response_format", + "thinking", + ), +) + +GOOGLE_CONTENT_SHAPE = ProviderApiShape( + dialect=DIALECT_GOOGLE_GENERATE_CONTENT, + request_path="/v1beta/models/{model}:generateContent", + stream_path="/v1beta/models/{model}:streamGenerateContent?alt=sse", + message_field="contents", + tool_request_paths=("tools[].functionDeclarations", "toolConfig"), + tool_response_paths=("candidates[].content.parts[].functionCall", "candidates[].content.parts[].functionResponse"), + text_response_paths=("candidates[].content.parts[].text",), + reasoning_response_paths=("candidates[].content.parts[].thought", "candidates[].content.parts[].thoughtSignature"), + request_control_paths=( + "generationConfig.temperature", + "generationConfig.topP", + "generationConfig.topK", + "generationConfig.thinkingConfig", + "generationConfig.responseJsonSchema", + ), +) + +OLLAMA_NATIVE_SHAPE = ProviderApiShape( + dialect=DIALECT_OLLAMA_NATIVE, + request_path="/api/chat", + stream_path="/api/chat", + tool_request_paths=("tools[].function", "messages[].tool_calls[].function"), + tool_response_paths=("message.tool_calls[].function",), + text_response_paths=("message.content",), + reasoning_response_paths=("message.thinking",), + request_control_paths=("think", "format", "options.temperature", "options.top_p", "options.seed"), +) + +LMSTUDIO_NATIVE_SHAPE = ProviderApiShape( + dialect=DIALECT_LMSTUDIO_NATIVE_V1, + request_path="/api/v1/chat", + stream_path="/api/v1/chat", + message_field="input", + tool_request_paths=("integrations[].mcp",), + text_response_paths=("output[].content",), + reasoning_response_paths=("output[].reasoning",), + request_control_paths=("temperature", "top_p", "reasoning", "response_format"), +) + +SGLANG_NATIVE_SHAPE = ProviderApiShape( + dialect=DIALECT_SGLANG_NATIVE, + request_path="/generate", + stream_path="/generate", + message_field="text", + text_response_paths=("text",), + request_control_paths=("sampling_params.temperature", "sampling_params.top_p", "sampling_params.seed"), +) + +CHATGPT_SUBSCRIPTION_SHAPE = ProviderApiShape( + dialect=DIALECT_CHATGPT_SUBSCRIPTION, + request_path="/backend-api/codex/responses", + stream_path="/backend-api/codex/responses", + message_field="input", + tool_request_paths=("tools[]", "input[].function_call_output"), + tool_response_paths=("response.function_call_arguments.delta", "response.output_item.done"), + text_response_paths=("response.output_text.delta",), + reasoning_response_paths=("response.reasoning_summary_text.delta", "reasoning.encrypted_content"), + request_control_paths=("reasoning", "text", "parallel_tool_calls"), +) + + +GENERAL_DATA_SHAPE = ProviderCatalogShape( + shape_id="openai-compatible.models.data.v1", + provider_id=PROVIDER_GENERIC_OPENAI, + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id", "name", "model"), +) +GENERAL_MODELS_SHAPE = ProviderCatalogShape( + shape_id="general.models-envelope.v1", + provider_id=PROVIDER_GENERIC_OPENAI, + endpoint_path="/models", + envelope=ENVELOPE_MODELS, + identity_paths=("id", "key", "slug", "name", "model"), +) +GENERAL_BARE_SHAPE = ProviderCatalogShape( + shape_id="openai-compatible.models.bare-list.v1", + provider_id=PROVIDER_GENERIC_OPENAI, + endpoint_path="/models", + envelope=ENVELOPE_BARE_LIST, + identity_paths=("id", "name", "model"), +) + +OPENAI_MODELS_SHAPE = ProviderCatalogShape( + shape_id="openai.models.identity.v1", + provider_id="openai", + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("object", "created", "owned_by"), + item_values=(("object", ("model",)),), + api_version="v1", +) +OPENROUTER_MODELS_SHAPE = ProviderCatalogShape( + shape_id="openrouter.models.rich.v1", + provider_id="openrouter", + endpoint_path="/api/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_any_paths=("architecture", "supported_parameters", "top_provider", "canonical_slug"), + capability_paths=( + "architecture.input_modalities", + "architecture.output_modalities", + "supported_parameters", + "context_length", + "top_provider.max_completion_tokens", + ), + api_version="v1", + priority=90, +) +GOOGLE_MODELS_SHAPE = ProviderCatalogShape( + shape_id="google.generative-language.models.v1beta", + provider_id="google", + endpoint_path="/v1beta/models", + envelope=ENVELOPE_MODELS, + identity_paths=("baseModelId", "name"), + required_item_paths=("supportedGenerationMethods",), + item_types=(("supportedGenerationMethods", (list, tuple)),), + capability_paths=( + "supportedGenerationMethods", + "inputTokenLimit", + "outputTokenLimit", + "thinking", + "temperature", + "topP", + "topK", + ), + api_version="v1beta", + priority=100, +) +OLLAMA_TAGS_SHAPE = ProviderCatalogShape( + shape_id="ollama.tags.v1", + provider_id="ollama", + endpoint_path="/api/tags", + envelope=ENVELOPE_MODELS, + identity_paths=("model", "name"), + required_item_any_paths=("digest", "details.family", "details.families"), + capability_paths=("details.family", "details.families"), + priority=90, +) +OLLAMA_SHOW_SHAPE = ProviderCatalogShape( + shape_id="ollama.show.v1", + provider_id="ollama", + endpoint_path="/api/show", + envelope=ENVELOPE_SINGLE, + identity_paths=(), + required_item_paths=("capabilities",), + required_item_any_paths=("model_info", "details", "template", "parameters"), + item_types=(("capabilities", (list, tuple)),), + capability_paths=("capabilities", "model_info.*.context_length"), + priority=100, +) +LMSTUDIO_MODELS_V1_SHAPE = ProviderCatalogShape( + shape_id="lmstudio.models.native.v1", + provider_id="lmstudio", + endpoint_path="/api/v1/models", + envelope=ENVELOPE_MODELS, + identity_paths=("key",), + required_item_paths=("type",), + required_item_any_paths=("capabilities", "loaded_instances", "max_context_length", "architecture", "quantization"), + item_types=(("type", (str,)),), + capability_paths=("type", "capabilities", "max_context_length", "loaded_instances[].config.context_length"), + api_version="1", + priority=100, +) +LMSTUDIO_MODELS_V0_SHAPE = ProviderCatalogShape( + shape_id="lmstudio.models.native.v0", + provider_id="lmstudio", + endpoint_path="/api/v0/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("type",), + required_item_any_paths=("arch", "compatibility_type", "state", "max_context_length"), + item_types=(("type", (str,)),), + capability_paths=("type", "max_context_length"), + api_version="0", + priority=80, + latest=False, +) +LLAMACPP_PROPS_SHAPE = ProviderCatalogShape( + shape_id="llamacpp.props.v1", + provider_id="llamacpp", + endpoint_path="/props", + envelope=ENVELOPE_SINGLE, + identity_paths=("model_alias", "model_path"), + required_item_paths=("default_generation_settings",), + required_item_any_paths=("chat_template_caps", "modalities", "total_slots"), + item_types=(("default_generation_settings", (Mapping,)),), + capability_paths=( + "chat_template_caps", + "modalities", + "default_generation_settings.n_ctx", + "default_generation_settings.params", + ), + priority=100, +) +MISTRAL_MODELS_SHAPE = ProviderCatalogShape( + shape_id="mistral.models.rich.v1", + provider_id="mistral", + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("capabilities",), + required_item_any_paths=( + "capabilities.completion_chat", + "capabilities.completion_fim", + "capabilities.function_calling", + "capabilities.vision", + "capabilities.classification", + ), + item_types=(("capabilities", (Mapping,)),), + capability_paths=("capabilities", "max_context_length"), + api_version="v1", + priority=100, +) +COPILOT_MODELS_SHAPE = ProviderCatalogShape( + shape_id="github-copilot.models.v1", + provider_id="copilot", + endpoint_path="/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("model_picker_enabled", "capabilities.supports"), + item_types=( + ("model_picker_enabled", (bool,)), + ("capabilities.supports", (Mapping,)), + ), + capability_paths=("capabilities.supports.tool_calls", "capabilities.supports.vision"), + priority=100, +) +ANTHROPIC_MODELS_SHAPE = ProviderCatalogShape( + shape_id="anthropic.models.identity.v1", + provider_id="anthropic", + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("type", "display_name", "created_at"), + item_values=(("type", ("model",)),), + api_version="2023-06-01", + priority=70, +) +CHATGPT_MODELS_SHAPE = ProviderCatalogShape( + shape_id="chatgpt-subscription.codex-models.v1", + provider_id="chatgpt_subscription", + endpoint_path="/backend-api/codex/models?client_version=1.0.0", + envelope=ENVELOPE_MODELS, + identity_paths=("slug",), + required_item_any_paths=("visibility", "priority"), + priority=90, +) +SGLANG_MODEL_INFO_SHAPE = ProviderCatalogShape( + shape_id="sglang.model-info.v2", + provider_id="sglang", + endpoint_path="/model_info", + envelope=ENVELOPE_SINGLE, + identity_paths=("model_path",), + required_item_paths=("is_generation",), + required_item_any_paths=("tokenizer_path", "has_image_understanding", "has_audio_understanding"), + item_types=(("is_generation", (bool,)),), + capability_paths=("is_generation", "has_image_understanding", "has_audio_understanding", "preferred_sampling_params"), + api_version="2", + priority=100, +) +SGLANG_MODELS_SHAPE = ProviderCatalogShape( + shape_id="sglang.models.openai.v1", + provider_id="sglang", + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("root", "max_model_len"), + item_values=(("owned_by", ("sglang",)),), + capability_paths=("max_model_len",), + priority=80, +) +VLLM_MODELS_SHAPE = ProviderCatalogShape( + shape_id="vllm.models.openai.v1", + provider_id="vllm", + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("root", "max_model_len", "permission"), + item_values=(("owned_by", ("vllm",)),), + capability_paths=("max_model_len",), + priority=80, +) +HUGGINGFACE_MODEL_SHAPE = ProviderCatalogShape( + shape_id="huggingface.hub.model-info.v1", + provider_id="huggingface", + endpoint_path="/api/models/{model}", + envelope=ENVELOPE_SINGLE, + identity_paths=("modelId", "id"), + required_item_paths=("pipeline_tag",), + item_types=(("pipeline_tag", (str,)),), + capability_paths=("pipeline_tag", "tags", "config"), + priority=80, +) +COHERE_MODELS_SHAPE = ProviderCatalogShape( + shape_id="cohere.models.rich.v1", + provider_id="cohere", + endpoint_path="/v1/models", + envelope=ENVELOPE_MODELS, + identity_paths=("name",), + required_item_paths=("endpoints",), + required_item_any_paths=("context_length", "default_endpoints", "features", "sampling_defaults"), + item_types=(("endpoints", (list, tuple)),), + capability_paths=("endpoints", "context_length", "sampling_defaults"), + api_version="v1", + priority=100, +) +MINIMAX_MODELS_SHAPE = ProviderCatalogShape( + shape_id="minimax.models.identity.v1", + provider_id="minimax", + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("object", "owned_by"), + item_values=(("object", ("model",)), ("owned_by", ("minimax",))), + api_version="v1", + priority=90, +) + + +def _provider( + provider_id: str, + display_name: str, + *, + aliases: tuple[str, ...] = (), + hosts: tuple[str, ...] = (), + api_shapes: tuple[ProviderApiShape, ...] = (OPENAI_CHAT_SHAPE,), + catalog_shapes: tuple[ProviderCatalogShape, ...] = (), +) -> ProviderCapabilitySchema: + return ProviderCapabilitySchema( + provider_id=provider_id, + display_name=display_name, + aliases=aliases, + host_suffixes=hosts, + api_shapes=api_shapes, + catalog_shapes=catalog_shapes, + ) + + +PROVIDER_SCHEMAS = { + PROVIDER_GENERIC_OPENAI: _provider( + PROVIDER_GENERIC_OPENAI, + "General OpenAI-compatible", + aliases=("openai_compatible", "proxy"), + api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE), + catalog_shapes=(GENERAL_DATA_SHAPE, GENERAL_MODELS_SHAPE, GENERAL_BARE_SHAPE), + ), + "openai": _provider( + "openai", + "OpenAI", + hosts=("openai.com",), + api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE), + catalog_shapes=(OPENAI_MODELS_SHAPE,), + ), + "openrouter": _provider( + "openrouter", + "OpenRouter", + hosts=("openrouter.ai",), + catalog_shapes=(OPENROUTER_MODELS_SHAPE,), + ), + "google": _provider( + "google", + "Google Gemini", + aliases=("gemini", "google_ai_studio"), + hosts=("generativelanguage.googleapis.com",), + api_shapes=(GOOGLE_CONTENT_SHAPE, OPENAI_CHAT_SHAPE), + catalog_shapes=(GOOGLE_MODELS_SHAPE,), + ), + "anthropic": _provider( + "anthropic", + "Anthropic", + hosts=("anthropic.com",), + api_shapes=(ANTHROPIC_MESSAGES_SHAPE,), + catalog_shapes=(ANTHROPIC_MODELS_SHAPE,), + ), + "ollama": _provider( + "ollama", + "Ollama", + hosts=("ollama.com",), + api_shapes=(OLLAMA_NATIVE_SHAPE, OPENAI_CHAT_SHAPE), + catalog_shapes=(OLLAMA_SHOW_SHAPE, OLLAMA_TAGS_SHAPE), + ), + "lmstudio": _provider( + "lmstudio", + "LM Studio", + aliases=("lm_studio",), + api_shapes=(LMSTUDIO_NATIVE_SHAPE, OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE), + catalog_shapes=(LMSTUDIO_MODELS_V1_SHAPE, LMSTUDIO_MODELS_V0_SHAPE), + ), + "llamacpp": _provider( + "llamacpp", + "llama.cpp", + aliases=("llama.cpp", "llama_cpp", "llama_server"), + api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE, ANTHROPIC_MESSAGES_SHAPE), + catalog_shapes=(LLAMACPP_PROPS_SHAPE,), + ), + "mistral": _provider( + "mistral", + "Mistral", + hosts=("mistral.ai",), + catalog_shapes=(MISTRAL_MODELS_SHAPE,), + ), + "copilot": _provider( + "copilot", + "GitHub Copilot", + aliases=("github_copilot",), + hosts=("api.githubcopilot.com",), + catalog_shapes=(COPILOT_MODELS_SHAPE,), + ), + "chatgpt_subscription": _provider( + "chatgpt_subscription", + "ChatGPT Subscription", + aliases=("chatgpt-subscription", "chatgpt", "codex_subscription"), + hosts=("chatgpt.com",), + api_shapes=(CHATGPT_SUBSCRIPTION_SHAPE,), + catalog_shapes=(CHATGPT_MODELS_SHAPE,), + ), + "sglang": _provider( + "sglang", + "SGLang", + api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE, SGLANG_NATIVE_SHAPE), + catalog_shapes=(SGLANG_MODEL_INFO_SHAPE, SGLANG_MODELS_SHAPE), + ), + "vllm": _provider( + "vllm", + "vLLM", + api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE), + catalog_shapes=(VLLM_MODELS_SHAPE,), + ), + "huggingface": _provider( + "huggingface", + "Hugging Face", + aliases=("hf", "hugging_face"), + hosts=("huggingface.co",), + api_shapes=(OPENAI_CHAT_SHAPE,), + catalog_shapes=(HUGGINGFACE_MODEL_SHAPE,), + ), + "cohere": _provider( + "cohere", + "Cohere", + hosts=("cohere.ai", "cohere.com"), + api_shapes=(COHERE_V2_SHAPE, OPENAI_CHAT_SHAPE), + catalog_shapes=(COHERE_MODELS_SHAPE,), + ), + "minimax": _provider( + "minimax", + "MiniMax", + hosts=("minimax.io", "minimaxi.com"), + api_shapes=(ANTHROPIC_MESSAGES_SHAPE, OPENAI_CHAT_SHAPE), + catalog_shapes=(MINIMAX_MODELS_SHAPE,), + ), +} + +# These providers are currently OpenAI-compatible in Odysseus and have no +# provider-reported capability catalog shape that is stronger than the general +# structural fallback. Keeping individual identities prevents transport +# quirks from being flattened into "OpenAI" while capability stays per-model. +_GENERAL_PROVIDER_ALIASES = { + "moonshot": ("moonshot_ai",), + "nvidia": ("nvidia_nim", "nim"), + "xai": ("x_ai",), + "zai": ("z.ai", "z_ai"), + "opencode": ("opencode_go", "opencode_zen"), + "together": ("together_ai",), + "fireworks": ("fireworks_ai",), + "atlas_cloud": ("atlas",), + "azure_openai": ("azure",), + "bedrock": ("aws_bedrock",), + "cloudflare_workers_ai": ("workers_ai",), + "mlx_lm": ("mlx",), + "text_generation_inference": ("tgi", "huggingface_tgi", "hugging_face_tgi"), +} +for _provider_id, _display, _hosts in ( + ("moonshot", "Moonshot AI", ("moonshot.ai", "moonshot.cn")), + ("groq", "Groq", ("groq.com",)), + ("nvidia", "NVIDIA NIM", ("nvidia.com",)), + ("cerebras", "Cerebras", ("cerebras.ai",)), + ("deepseek", "DeepSeek", ("deepseek.com",)), + ("together", "Together AI", ("together.xyz", "together.ai")), + ("fireworks", "Fireworks AI", ("fireworks.ai",)), + ("xai", "xAI", ("x.ai",)), + ("zai", "Z.AI", ("z.ai",)), + ("opencode", "OpenCode", ("opencode.ai",)), + ("perplexity", "Perplexity", ("perplexity.ai",)), + ("github_models", "GitHub Models", ("models.inference.ai.azure.com",)), + ("atlas_cloud", "Atlas Cloud", ("atlascloud.ai",)), + ("siliconflow", "SiliconFlow", ("siliconflow.cn", "siliconflow.com",)), + ("kimi_code", "Kimi Code", ("kimi.com",)), + ("venice", "Venice", ("venice.ai",)), + ("azure_openai", "Azure OpenAI", ("openai.azure.com",)), + ("bedrock", "AWS Bedrock", ()), + ("cloudflare_workers_ai", "Cloudflare Workers AI", ()), + ("mlx_lm", "MLX LM", ()), + ("text_generation_inference", "Hugging Face TGI", ()), + ("lmdeploy", "LMDeploy", ()), + ("litellm", "LiteLLM", ()), +): + PROVIDER_SCHEMAS[_provider_id] = _provider( + _provider_id, + _display, + aliases=_GENERAL_PROVIDER_ALIASES.get(_provider_id, ()), + hosts=_hosts, + ) + +UNKNOWN_SCHEMA = ProviderCapabilitySchema( + provider_id=PROVIDER_UNKNOWN, + display_name="Unknown provider", + api_shapes=(), + catalog_shapes=(), +) + +_ALIASES = { + _token(alias): provider_id + for provider_id, schema in PROVIDER_SCHEMAS.items() + for alias in (provider_id, *schema.aliases) +} + + +def normalize_provider_id(value: Any) -> str: + token = _token(value) + return _ALIASES.get(token, token if token in PROVIDER_SCHEMAS else PROVIDER_UNKNOWN) + + +def schema_for_provider(value: Any) -> ProviderCapabilitySchema: + return PROVIDER_SCHEMAS.get(normalize_provider_id(value), UNKNOWN_SCHEMA) + + +def _host_matches(host: str, suffix: str) -> bool: + return host == suffix or host.endswith("." + suffix) + + +def provider_from_host(base_url: Any) -> str: + try: + host = (urlparse(str(base_url or "")).hostname or "").lower().rstrip(".") + except Exception: + return PROVIDER_UNKNOWN + if not host: + return PROVIDER_UNKNOWN + if host.startswith("copilot-api.") and host.endswith(".ghe.com"): + return "copilot" + matches = [ + schema.provider_id + for schema in PROVIDER_SCHEMAS.values() + if any(_host_matches(host, suffix) for suffix in schema.host_suffixes) + ] + return matches[0] if len(set(matches)) == 1 else PROVIDER_UNKNOWN + + +def catalog_shape_for_payload( + payload: Any, + *, + provider_id: Any = None, + include_general: bool = True, +) -> ProviderCatalogShape | None: + normalized = normalize_provider_id(provider_id) + provider_is_explicit = normalized not in {PROVIDER_UNKNOWN, PROVIDER_GENERIC_OPENAI} + if provider_is_explicit: + schemas = (PROVIDER_SCHEMAS[normalized],) + else: + schemas = tuple(PROVIDER_SCHEMAS.values()) + + candidates = [ + shape + for schema in schemas + for shape in schema.catalog_shapes + if (provider_is_explicit or shape.priority > 0) and shape.matches(payload) + ] + if candidates: + best_priority = max(shape.priority for shape in candidates) + best = [shape for shape in candidates if shape.priority == best_priority] + providers = {shape.provider_id for shape in best} + if len(providers) == 1: + return sorted(best, key=lambda shape: shape.shape_id)[0] + + if not include_general: + return None + for shape in PROVIDER_SCHEMAS[PROVIDER_GENERIC_OPENAI].catalog_shapes: + if shape.matches(payload): + return shape + return None + + +def resolve_provider( + payload: Any = None, + *, + provider: Any = None, + endpoint_kind: Any = None, + base_url: Any = None, +) -> ProviderResolution: + explicit = normalize_provider_id(provider) + if explicit != PROVIDER_UNKNOWN: + shape = catalog_shape_for_payload(payload, provider_id=explicit) if payload is not None else None + return ProviderResolution(explicit, RESOLUTION_EXPLICIT, PROVIDER_SCHEMAS[explicit], shape) + + kind = normalize_provider_id(endpoint_kind) + if kind != PROVIDER_UNKNOWN: + shape = catalog_shape_for_payload(payload, provider_id=kind) if payload is not None else None + return ProviderResolution(kind, RESOLUTION_ENDPOINT_KIND, PROVIDER_SCHEMAS[kind], shape) + + host_provider = provider_from_host(base_url) + if host_provider != PROVIDER_UNKNOWN: + shape = catalog_shape_for_payload(payload, provider_id=host_provider) if payload is not None else None + return ProviderResolution( + host_provider, + RESOLUTION_HOST, + PROVIDER_SCHEMAS[host_provider], + shape, + ) + + shape = catalog_shape_for_payload(payload, include_general=False) if payload is not None else None + if shape: + return ProviderResolution( + shape.provider_id, + RESOLUTION_NATIVE_SHAPE, + PROVIDER_SCHEMAS[shape.provider_id], + shape, + ) + + shape = catalog_shape_for_payload(payload) if payload is not None else None + if shape: + return ProviderResolution( + PROVIDER_GENERIC_OPENAI, + RESOLUTION_GENERAL_SHAPE, + PROVIDER_SCHEMAS[PROVIDER_GENERIC_OPENAI], + shape, + ) + + return ProviderResolution(PROVIDER_UNKNOWN, RESOLUTION_UNKNOWN, UNKNOWN_SCHEMA, None) + + +__all__ = [ + "ANTHROPIC_MESSAGES_SHAPE", + "CHATGPT_SUBSCRIPTION_SHAPE", + "COHERE_V2_SHAPE", + "DIALECT_ANTHROPIC_MESSAGES", + "DIALECT_CHATGPT_SUBSCRIPTION", + "DIALECT_COHERE_V2", + "DIALECT_GOOGLE_GENERATE_CONTENT", + "DIALECT_HUGGINGFACE_HUB", + "DIALECT_LLAMACPP_NATIVE", + "DIALECT_LMSTUDIO_NATIVE_V1", + "DIALECT_OLLAMA_NATIVE", + "DIALECT_OPENAI_CHAT", + "DIALECT_OPENAI_RESPONSES", + "DIALECT_SGLANG_NATIVE", + "GOOGLE_CONTENT_SHAPE", + "LMSTUDIO_NATIVE_SHAPE", + "OLLAMA_NATIVE_SHAPE", + "OPENAI_CHAT_SHAPE", + "OPENAI_RESPONSES_SHAPE", + "PROVIDER_GENERIC_OPENAI", + "PROVIDER_SCHEMAS", + "PROVIDER_UNKNOWN", + "ProviderApiShape", + "ProviderCapabilitySchema", + "ProviderCatalogShape", + "ProviderResolution", + "catalog_shape_for_payload", + "normalize_provider_id", + "provider_from_host", + "resolve_provider", + "schema_for_provider", +] diff --git a/tests/test_model_capability_readers.py b/tests/test_model_capability_readers.py index 36ef93ab1..17734d448 100644 --- a/tests/test_model_capability_readers.py +++ b/tests/test_model_capability_readers.py @@ -18,14 +18,14 @@ def surfaces(record): return set(mc.display_surfaces_for(record.capability)) -def test_detect_vendor_uses_endpoint_kind_then_host_and_common_local_ports(): +def test_detect_vendor_uses_endpoint_kind_and_host_but_not_ambiguous_local_ports(): assert detect_vendor("https://example.test/v1", endpoint_kind="ollama") == VENDOR_OLLAMA assert detect_vendor("http://127.0.0.1:8080", endpoint_kind="llama_cpp") == VENDOR_LLAMACPP assert detect_vendor("https://openrouter.ai/api/v1") == VENDOR_OPENROUTER assert detect_vendor("https://api.openai.com/v1") == VENDOR_OPENAI assert detect_vendor("https://generativelanguage.googleapis.com/v1beta/openai") == VENDOR_GOOGLE - assert detect_vendor("http://127.0.0.1:11434") == VENDOR_OLLAMA - assert detect_vendor("http://127.0.0.1:1234") == VENDOR_LMSTUDIO + assert detect_vendor("http://127.0.0.1:11434") == VENDOR_GENERIC_OPENAI + assert detect_vendor("http://127.0.0.1:1234") == VENDOR_GENERIC_OPENAI assert detect_vendor("http://127.0.0.1:8080") == VENDOR_GENERIC_OPENAI assert detect_vendor("http://localhost:7000/v1") == VENDOR_GENERIC_OPENAI @@ -345,7 +345,9 @@ def test_ollama_reader_maps_show_capabilities_and_tags_are_unknown(): "nomic-embed-text:latest", {"capabilities": ["embedding"]}, ) - tags = ollama.records_from_tags_payload({"models": [{"name": "qwen3:latest"}]}) + tags = ollama.records_from_tags_payload( + {"models": [{"name": "qwen3:latest", "details": {"family": "qwen3"}}]} + ) assert vision is not None assert vision.capability.family == mc.FAMILY_CHAT @@ -360,6 +362,7 @@ def test_ollama_reader_maps_show_capabilities_and_tags_are_unknown(): assert len(tags) == 1 assert tags[0].capability.family == mc.FAMILY_UNKNOWN + assert tags[0].model_family == "qwen3" assert surfaces(tags[0]) == set() @@ -381,7 +384,9 @@ def test_ollama_reader_uses_show_shape_without_architecture_name_matching(): assert record.capability.modalities.input == (mc.MODALITY_TEXT,) assert record.capability.modalities.output == (mc.MODALITY_TEXT,) assert record.capability.capabilities == (mc.CAP_REASONING, mc.CAP_TOOL_CALL) - assert dict(record.capability.limits) == {"context_tokens": 8192} + # Serialized Modelfile text is not reparsed for capability truth. The + # structured native `model_info.*.context_length` field wins. + assert dict(record.capability.limits) == {"context_tokens": 32768} assert surfaces(record) == {"chat"} diff --git a/tests/test_provider_capability_schemas.py b/tests/test_provider_capability_schemas.py new file mode 100644 index 000000000..855f85f5c --- /dev/null +++ b/tests/test_provider_capability_schemas.py @@ -0,0 +1,505 @@ +from src import model_behavior_quirks as quirks +from src import model_capabilities as mc +from src import provider_capability_schemas as pcs +from src.model_capability_readers import ( + anthropic, + chatgpt_subscription, + cohere, + copilot, + generic_openai, + huggingface, + mistral, + records_from_payload, + sglang, +) + + +def test_provider_resolution_order_explicit_then_host_then_native_then_general(): + google_payload = { + "models": [ + { + "name": "models/example", + "supportedGenerationMethods": ["generateContent"], + } + ] + } + + explicit = pcs.resolve_provider(google_payload, provider="openrouter") + host = pcs.resolve_provider(google_payload, base_url="https://api.mistral.ai/v1") + native = pcs.resolve_provider(google_payload) + general = pcs.resolve_provider([{"id": "future-model", "future": {"x": True}}]) + unknown = pcs.resolve_provider({"future": [{"not_an_identity": True}]}) + + assert (explicit.provider_id, explicit.stage) == ("openrouter", pcs.RESOLUTION_EXPLICIT) + assert (host.provider_id, host.stage) == ("mistral", pcs.RESOLUTION_HOST) + assert (native.provider_id, native.stage) == ("google", pcs.RESOLUTION_NATIVE_SHAPE) + assert native.catalog_shape.shape_id == "google.generative-language.models.v1beta" + assert (general.provider_id, general.stage) == ( + pcs.PROVIDER_GENERIC_OPENAI, + pcs.RESOLUTION_GENERAL_SHAPE, + ) + assert (unknown.provider_id, unknown.stage) == ( + pcs.PROVIDER_UNKNOWN, + pcs.RESOLUTION_UNKNOWN, + ) + + +def test_provider_host_matching_rejects_lookalikes_and_does_not_use_ports(): + assert pcs.provider_from_host("https://api.openrouter.ai/v1") == "openrouter" + assert pcs.provider_from_host("https://openrouter.ai.evil.test/v1") == pcs.PROVIDER_UNKNOWN + assert pcs.provider_from_host("http://127.0.0.1:11434") == pcs.PROVIDER_UNKNOWN + assert pcs.provider_from_host("http://127.0.0.1:1234") == pcs.PROVIDER_UNKNOWN + assert pcs.provider_from_host("http://127.0.0.1:8000") == pcs.PROVIDER_UNKNOWN + assert pcs.provider_from_host("http://127.0.0.1:30000") == pcs.PROVIDER_UNKNOWN + + +def test_provider_aliases_collapse_runtime_names_without_url_path_guessing(): + assert pcs.normalize_provider_id("opencode-go") == "opencode" + assert pcs.normalize_provider_id("opencode-zen") == "opencode" + assert pcs.normalize_provider_id("nvidia-nim") == "nvidia" + assert pcs.normalize_provider_id("tgi") == "text_generation_inference" + assert pcs.normalize_provider_id("llama.cpp") == "llamacpp" + assert pcs.normalize_provider_id("Z.AI") == "zai" + + +def test_current_native_catalog_shapes_are_discriminating_and_versioned(): + cases = ( + ( + {"models": [{"key": "local/model", "type": "llm", "capabilities": {"vision": True}}]}, + "lmstudio.models.native.v1", + ), + ( + {"data": [{"id": "legacy", "type": "vlm", "arch": "gemma"}]}, + "lmstudio.models.native.v0", + ), + ( + {"models": [{"name": "local", "digest": "abc", "details": {"family": "qwen3"}}]}, + "ollama.tags.v1", + ), + ( + {"capabilities": ["completion", "vision"], "model_info": {"x.context_length": 4096}}, + "ollama.show.v1", + ), + ( + { + "model_alias": "local", + "default_generation_settings": {"n_ctx": 4096}, + "chat_template_caps": {"supports_tools": True}, + }, + "llamacpp.props.v1", + ), + ( + {"data": [{"id": "mistral", "capabilities": {"completion_chat": True, "vision": False}}]}, + "mistral.models.rich.v1", + ), + ( + { + "data": [ + { + "id": "copilot-model", + "model_picker_enabled": True, + "capabilities": {"supports": {"tool_calls": True}}, + } + ] + }, + "github-copilot.models.v1", + ), + ( + { + "model_path": "org/model", + "tokenizer_path": "org/model", + "is_generation": True, + "has_image_understanding": False, + }, + "sglang.model-info.v2", + ), + ( + { + "object": "list", + "data": [ + { + "id": "served-model", + "object": "model", + "owned_by": "vllm", + "root": "org/model", + "max_model_len": 131072, + "permission": [], + } + ], + }, + "vllm.models.openai.v1", + ), + ( + {"models": [{"slug": "gpt-example", "visibility": "list", "priority": 1}]}, + "chatgpt-subscription.codex-models.v1", + ), + ( + { + "models": [ + { + "name": "command-example", + "endpoints": ["chat"], + "context_length": 131072, + } + ] + }, + "cohere.models.rich.v1", + ), + ( + { + "object": "list", + "data": [ + { + "id": "MiniMax-M2-example", + "object": "model", + "owned_by": "minimax", + } + ], + }, + "minimax.models.identity.v1", + ), + ) + + for payload, expected_shape in cases: + resolution = pcs.resolve_provider(payload) + assert resolution.stage == pcs.RESOLUTION_NATIVE_SHAPE + assert resolution.catalog_shape.shape_id == expected_shape + + +def test_native_shape_detection_rejects_wrong_field_types_before_general_fallback(): + malformed_cohere = pcs.resolve_provider( + {"models": [{"name": "future", "endpoints": "chat", "context_length": 4096}]} + ) + malformed_mistral = pcs.resolve_provider( + {"data": [{"id": "future", "capabilities": ["completion_chat"]}]} + ) + + assert (malformed_cohere.provider_id, malformed_cohere.stage) == ( + pcs.PROVIDER_GENERIC_OPENAI, + pcs.RESOLUTION_GENERAL_SHAPE, + ) + assert (malformed_mistral.provider_id, malformed_mistral.stage) == ( + pcs.PROVIDER_GENERIC_OPENAI, + pcs.RESOLUTION_GENERAL_SHAPE, + ) + + +def test_general_reader_promotes_only_explicit_structural_fields_and_accepts_bare_lists(): + records = generic_openai.records_from_payload( + [ + { + "id": "future-rich-model", + "type": "chat", + "architecture": { + "input_modalities": ["text", "image"], + "output_modalities": ["text"], + }, + "supported_parameters": ["tools", "structured_outputs", "temperature"], + "max_model_len": 131072, + "future_capability": {"may_be_important_later": True}, + }, + { + "id": "vision-reasoning-tools-in-the-name-only", + "description": "Claims every capability in prose", + "type": "image", + "future_capability": True, + }, + ] + ) + + rich, identity_only = records + assert rich.capability.family == mc.FAMILY_CHAT + assert rich.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) + assert rich.capability.capabilities == ( + mc.CAP_TOOL_CALL, + mc.CAP_STRUCTURED_OUTPUT, + mc.CAP_VISION, + ) + assert dict(rich.capability.limits) == {"context_tokens": 131072} + assert [control.control for control in rich.deterministic_controls] == [mc.CONTROL_TEMPERATURE] + assert rich.raw["future_capability"] == {"may_be_important_later": True} + + assert identity_only.capability.family == mc.FAMILY_UNKNOWN + assert identity_only.capability.capabilities == () + assert identity_only.raw["future_capability"] is True + + +def test_general_reader_fails_soft_for_null_and_malformed_envelopes(): + for payload in ( + {"data": None}, + {"models": None}, + {"data": "not-a-list"}, + [None, "model", 42, {"id": None}], + None, + ): + assert generic_openai.records_from_payload(payload) == () + + +def test_mistral_reader_maps_per_model_capabilities_without_provider_wide_inheritance(): + records = mistral.records_from_payload( + { + "data": [ + { + "id": "vision-chat", + "root": "mistral-small", + "capabilities": { + "completion_chat": True, + "function_calling": True, + "vision": True, + "classification": False, + }, + "max_context_length": 32768, + }, + { + "id": "classifier", + "capabilities": { + "completion_chat": False, + "classification": True, + "vision": False, + }, + }, + { + "id": "future-card", + "capabilities": {"future_only": True}, + }, + ] + } + ) + + assert records[0].capability.family == mc.FAMILY_CHAT + assert records[0].capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) + assert records[0].capability.capabilities == (mc.CAP_VISION, mc.CAP_TOOL_CALL) + assert dict(records[0].capability.limits) == {"context_tokens": 32768} + assert records[0].model_family == "mistral-small" + assert records[1].capability.family == mc.FAMILY_CLASSIFICATION + assert records[2].capability.family == mc.FAMILY_UNKNOWN + assert records[2].capability.capabilities == () + + +def test_copilot_reader_uses_picker_and_nested_supports_shape(): + records = copilot.records_from_payload( + { + "data": [ + { + "id": "picker-model", + "model_picker_enabled": True, + "capabilities": {"supports": {"tool_calls": True, "vision": True}}, + "limits": {"max_prompt_tokens": 64000, "max_output_tokens": 8192}, + }, + { + "id": "utility-model", + "model_picker_enabled": False, + "capabilities": {"supports": {}}, + }, + ] + } + ) + + assert records[0].capability.family == mc.FAMILY_CHAT + assert records[0].capability.capabilities == (mc.CAP_TOOL_CALL, mc.CAP_VISION) + assert dict(records[0].capability.limits) == {"input_tokens": 64000, "output_tokens": 8192} + assert records[1].capability.family == mc.FAMILY_UNKNOWN + + +def test_sglang_model_info_is_structural_and_non_generation_stays_unknown(): + generation = sglang.records_from_payload( + { + "model_path": "org/vision-model", + "tokenizer_path": "org/vision-model", + "is_generation": True, + "has_image_understanding": True, + "has_audio_understanding": True, + "model_type": "future_arch", + "preferred_sampling_params": {"temperature": 0.2, "top_p": 0.9}, + } + )[0] + pooling = sglang.records_from_payload( + { + "model_path": "org/pooling-model", + "tokenizer_path": "org/pooling-model", + "is_generation": False, + "has_image_understanding": False, + } + )[0] + + assert generation.capability.family == mc.FAMILY_CHAT + assert generation.capability.modalities.input == ( + mc.MODALITY_TEXT, + mc.MODALITY_IMAGE, + mc.MODALITY_AUDIO, + ) + assert generation.capability.capabilities == (mc.CAP_VISION, mc.CAP_AUDIO_INPUT) + assert [control.control for control in generation.deterministic_controls] == [ + mc.CONTROL_TEMPERATURE, + mc.CONTROL_TOP_P, + ] + assert generation.model_family == "future_arch" + assert pooling.capability.family == mc.FAMILY_UNKNOWN + + +def test_identity_only_catalogs_do_not_claim_model_capability(): + anthropic_record = anthropic.records_from_payload( + { + "data": [ + { + "id": "claude-example", + "type": "model", + "display_name": "Claude Example", + "created_at": "2026-01-01T00:00:00Z", + } + ] + } + )[0] + chatgpt_record = chatgpt_subscription.records_from_payload( + {"models": [{"slug": "gpt-example", "visibility": "list", "priority": 1}]} + )[0] + minimax_record = records_from_payload( + { + "object": "list", + "data": [ + { + "id": "MiniMax-M2-example", + "object": "model", + "owned_by": "minimax", + } + ], + } + )[0] + + assert anthropic_record.capability.family == mc.FAMILY_UNKNOWN + assert chatgpt_record.capability.family == mc.FAMILY_UNKNOWN + assert minimax_record.vendor == "minimax" + assert minimax_record.capability.family == mc.FAMILY_UNKNOWN + + +def test_huggingface_reader_maps_pipeline_tag_as_registry_evidence(): + record = huggingface.records_from_payload( + { + "modelId": "org/vision-model", + "pipeline_tag": "image-text-to-text", + "config": {"model_type": "future_vlm"}, + "tags": ["untrusted-prose-tag"], + } + )[0] + + assert record.capability.family == mc.FAMILY_CHAT + assert record.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) + assert record.capability.capabilities == (mc.CAP_VISION,) + assert record.capability.source == mc.SOURCE_COOKBOOK_HF + assert record.capability.confidence == mc.CONFIDENCE_REGISTRY + assert record.model_family == "future_vlm" + + +def test_cohere_reader_maps_only_native_endpoint_and_limit_fields(): + chat, ambiguous = cohere.records_from_payload( + { + "models": [ + { + "name": "command-example", + "endpoints": ["chat", "generate"], + "context_length": 131072, + "sampling_defaults": {"temperature": 0.3, "p": 0.9, "k": 40}, + "features": ["unmapped-future-feature"], + }, + { + "name": "multi-endpoint-example", + "endpoints": ["chat", "embed"], + "context_length": 4096, + }, + ] + } + ) + + assert chat.capability.family == mc.FAMILY_CHAT + assert chat.capability.modalities.input == (mc.MODALITY_TEXT,) + assert dict(chat.capability.limits) == {"context_tokens": 131072} + assert [control.control for control in chat.deterministic_controls] == [ + mc.CONTROL_TEMPERATURE, + mc.CONTROL_TOP_P, + mc.CONTROL_TOP_K, + ] + assert chat.raw["features"] == ["unmapped-future-feature"] + assert ambiguous.capability.family == mc.FAMILY_UNKNOWN + + +def test_registry_wrapper_records_resolution_and_preserves_compatible_provider_identity(): + mistral_records = records_from_payload( + { + "data": [ + { + "id": "mistral-model", + "capabilities": {"completion_chat": True, "function_calling": True}, + } + ] + } + ) + together_records = records_from_payload( + [{"id": "served/model", "type": "chat", "supported_parameters": ["tools"]}], + vendor="together", + ) + + assert mistral_records[0].vendor == "mistral" + assert mistral_records[0].provider_schema_id == "mistral" + assert mistral_records[0].catalog_shape_id == "mistral.models.rich.v1" + assert mistral_records[0].provider_resolution == pcs.RESOLUTION_NATIVE_SHAPE + + assert together_records[0].vendor == "together" + assert together_records[0].capability.family == mc.FAMILY_CHAT + assert together_records[0].provider_schema_id == "together" + assert together_records[0].provider_resolution == pcs.RESOLUTION_EXPLICIT + + +def test_reasoning_control_preserves_canonical_and_native_values(): + control = mc.ReasoningControl.build( + mechanism="reasoning_effort", + values=("enabled", "disabled"), + native_values=("high", "medium", "low", "none"), + request_path="reasoning_effort", + response_paths=("choices[].delta.reasoning",), + status="claimed", + source="provider_docs_registry", + confidence="registry", + ) + + assert control.values == (mc.REASONING_CONTROL_VALUE_ON, mc.REASONING_CONTROL_VALUE_OFF) + assert control.native_values == ("high", "medium", "low", "none") + assert mc.ReasoningControl.from_dict(control.to_dict()) == control + + +def test_model_quirks_require_structured_exact_identity_not_name_parsing(): + matching = quirks.matching_quirks( + provider="moonshot", + model_id="kimi-k2.5", + model_family="", + api_dialect=pcs.DIALECT_OPENAI_CHAT, + capabilities=(mc.CAP_REASONING,), + ) + lookalike = quirks.matching_quirks( + provider="moonshot", + model_id="proxy/kimi-k2.5-lookalike", + model_family="", + api_dialect=pcs.DIALECT_OPENAI_CHAT, + capabilities=(mc.CAP_REASONING,), + ) + opus_without_version = quirks.matching_quirks( + provider="anthropic", + model_family="claude-opus", + model_id="claude-opus-4-8-in-name-only", + api_dialect=pcs.DIALECT_ANTHROPIC_MESSAGES, + ) + opus_structured = quirks.matching_quirks( + provider="anthropic", + model_family="claude-opus", + model_version=(4, 8), + api_dialect=pcs.DIALECT_ANTHROPIC_MESSAGES, + ) + + assert {quirk.quirk_id for quirk in matching} == { + "moonshot.kimi-k2.5-k2.6.provider-fixed-temperature", + "moonshot.kimi-k2.5-k2.6.tool-history-reasoning-content", + } + assert lookalike == () + assert opus_without_version == () + assert [quirk.quirk_id for quirk in opus_structured] == [ + "anthropic.claude-opus-4.7-plus.omit-sampling-controls" + ] From ef3409ed536a1dd43d6234ce84d60dfb48888a04 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:42:41 +0000 Subject: [PATCH 02/14] refactor(models): simplify provider capability catalog --- src/model_behavior_quirks.py | 297 -------- src/model_capabilities.py | 100 --- src/model_capability_readers/__init__.py | 11 +- src/model_capability_readers/base.py | 53 +- .../chatgpt_subscription.py | 1 - src/model_capability_readers/copilot.py | 1 - .../generic_openai.py | 258 +------ src/model_capability_readers/google.py | 1 - src/model_capability_readers/huggingface.py | 106 ++- src/model_capability_readers/lmstudio.py | 1 - src/model_capability_readers/mistral.py | 1 - src/model_capability_readers/ollama.py | 2 - src/model_capability_readers/sglang.py | 1 - src/provider_capability_schemas.py | 697 ++++++------------ tests/test_model_capability_readers.py | 10 +- tests/test_provider_capability_schemas.py | 348 ++++----- 16 files changed, 520 insertions(+), 1368 deletions(-) delete mode 100644 src/model_behavior_quirks.py diff --git a/src/model_behavior_quirks.py b/src/model_behavior_quirks.py deleted file mode 100644 index 5eb5011ae..000000000 --- a/src/model_behavior_quirks.py +++ /dev/null @@ -1,297 +0,0 @@ -"""Canonical, exact-match model/provider behavior observations. - -The registry captures behavior that cannot safely be promoted to a provider- -wide capability. Selectors accept already-structured identity (provider, -model ID/family/version, API dialect, and canonical capabilities); they never -extract those facts from a display name with regexes or substring matching. - -This is shape/evidence data only. Runtime request builders can consume it in a -later integration pass after their endpoint has supplied structured identity. -""" - -from __future__ import annotations - -from collections.abc import Iterable, Mapping -from dataclasses import dataclass -from typing import Any - -from src import model_capabilities as mc -from src import provider_capability_schemas as pcs - - -def _identity(value: Any) -> str: - return str(value or "").strip().lower() - - -def _version(value: Any) -> tuple[int, ...]: - if not isinstance(value, (list, tuple)): - return () - out: list[int] = [] - for part in value: - try: - out.append(int(part)) - except (TypeError, ValueError): - return () - return tuple(out) - - -@dataclass(frozen=True) -class ModelBehaviorSelector: - providers: tuple[str, ...] = () - model_ids: tuple[str, ...] = () - model_families: tuple[str, ...] = () - minimum_model_version: tuple[int, ...] = () - minimum_provider_version: tuple[int, ...] = () - api_dialects: tuple[str, ...] = () - required_capabilities: tuple[str, ...] = () - - def matches( - self, - *, - provider: Any, - model_id: Any = "", - model_family: Any = "", - model_version: Any = (), - provider_version: Any = (), - api_dialect: Any = "", - capabilities: Any = (), - ) -> bool: - provider_id = pcs.normalize_provider_id(provider) - if self.providers and provider_id not in self.providers: - return False - - identity_constraints = bool(self.model_ids or self.model_families) - identity_match = ( - _identity(model_id) in self.model_ids - or _identity(model_family) in self.model_families - ) - if identity_constraints and not identity_match: - return False - - actual_model_version = _version(model_version) - if self.minimum_model_version and ( - not actual_model_version or actual_model_version < self.minimum_model_version - ): - return False - actual_provider_version = _version(provider_version) - if self.minimum_provider_version and ( - not actual_provider_version or actual_provider_version < self.minimum_provider_version - ): - return False - if self.api_dialects and str(api_dialect or "").strip() not in self.api_dialects: - return False - - if isinstance(capabilities, Mapping): - capability_values: Iterable[Any] = ( - key for key, enabled in capabilities.items() if enabled is True - ) - elif isinstance(capabilities, str): - capability_values = (capabilities,) - elif isinstance(capabilities, Iterable): - capability_values = capabilities - else: - capability_values = () - normalized_caps = { - normalized - for value in capability_values - if (normalized := mc.normalize_capability(value)) - } - return set(self.required_capabilities).issubset(normalized_caps) - - -@dataclass(frozen=True) -class ModelBehaviorQuirk: - quirk_id: str - selector: ModelBehaviorSelector - request_omit_paths: tuple[str, ...] = () - request_fixed_values: tuple[tuple[str, Any], ...] = () - required_history_paths: tuple[str, ...] = () - response_reasoning_paths: tuple[str, ...] = () - reasoning_controls: tuple[mc.ReasoningControl, ...] = () - status: str = mc.ASSERTION_CLAIMED - source: str = mc.SOURCE_PROVIDER_DOCS_REGISTRY - confidence: str = mc.CONFIDENCE_REGISTRY - evidence_refs: tuple[str, ...] = () - - def to_dict(self) -> dict[str, Any]: - return { - "quirk_id": self.quirk_id, - "selector": { - "providers": list(self.selector.providers), - "model_ids": list(self.selector.model_ids), - "model_families": list(self.selector.model_families), - "minimum_model_version": list(self.selector.minimum_model_version), - "minimum_provider_version": list(self.selector.minimum_provider_version), - "api_dialects": list(self.selector.api_dialects), - "required_capabilities": list(self.selector.required_capabilities), - }, - "request_omit_paths": list(self.request_omit_paths), - "request_fixed_values": dict(self.request_fixed_values), - "required_history_paths": list(self.required_history_paths), - "response_reasoning_paths": list(self.response_reasoning_paths), - "reasoning_controls": [control.to_dict() for control in self.reasoning_controls], - "status": self.status, - "source": self.source, - "confidence": self.confidence, - "evidence_refs": list(self.evidence_refs), - } - - -MODEL_BEHAVIOR_QUIRKS = ( - ModelBehaviorQuirk( - quirk_id="moonshot.kimi-k2.5-k2.6.provider-fixed-temperature", - selector=ModelBehaviorSelector( - providers=("moonshot",), - model_ids=("kimi-k2.5", "kimi-k2.6"), - model_families=("kimi-k2.5", "kimi-k2.6"), - api_dialects=(pcs.DIALECT_OPENAI_CHAT,), - ), - request_omit_paths=("temperature",), - evidence_refs=( - "github:odysseus-dev/odysseus#3960", - "commit:f5d3e509", - ), - ), - ModelBehaviorQuirk( - quirk_id="moonshot.kimi-k2.5-k2.6.tool-history-reasoning-content", - selector=ModelBehaviorSelector( - providers=("moonshot",), - model_ids=("kimi-k2.5", "kimi-k2.6"), - model_families=("kimi-k2.5", "kimi-k2.6"), - api_dialects=(pcs.DIALECT_OPENAI_CHAT,), - ), - required_history_paths=("messages[assistant+tool_calls].reasoning_content",), - response_reasoning_paths=( - "choices[].message.reasoning_content", - "choices[].delta.reasoning_content", - ), - evidence_refs=( - "github:odysseus-dev/odysseus#3118", - "commit:2e6fff22", - ), - ), - ModelBehaviorQuirk( - quirk_id="anthropic.claude-opus-4.7-plus.omit-sampling-controls", - selector=ModelBehaviorSelector( - providers=("anthropic",), - model_families=("claude-opus",), - minimum_model_version=(4, 7), - api_dialects=(pcs.DIALECT_ANTHROPIC_MESSAGES,), - ), - request_omit_paths=("temperature", "top_p", "top_k"), - evidence_refs=( - "github:odysseus-dev/odysseus#3117", - "commit:4f48cfa9", - ), - ), - ModelBehaviorQuirk( - quirk_id="mistral.reasoning.structured-content", - selector=ModelBehaviorSelector( - providers=("mistral",), - model_families=("magistral", "mistral-small", "mistral-medium"), - api_dialects=(pcs.DIALECT_OPENAI_CHAT,), - required_capabilities=(mc.CAP_REASONING,), - ), - response_reasoning_paths=( - "choices[].message.content[type=thinking].thinking[].text", - "choices[].delta.content[type=thinking].thinking[].text", - ), - reasoning_controls=( - mc.ReasoningControl.build( - mechanism=mc.REASONING_CONTROL_EFFORT, - values=(mc.REASONING_CONTROL_VALUE_ON, mc.REASONING_CONTROL_VALUE_OFF), - native_values=("high", "medium", "low", "none"), - request_path="reasoning_effort", - response_paths=("choices[].message.content[type=thinking]",), - status=mc.ASSERTION_CLAIMED, - source=mc.SOURCE_PROVIDER_DOCS_REGISTRY, - confidence=mc.CONFIDENCE_REGISTRY, - ), - ), - evidence_refs=( - "github:odysseus-dev/odysseus#4698", - "commit:bd9149f7", - "https://docs.mistral.ai/capabilities/reasoning/", - ), - ), - ModelBehaviorQuirk( - quirk_id="ollama.native.reasoning-control", - selector=ModelBehaviorSelector( - providers=("ollama",), - model_families=("qwen3", "deepseek-v3.1", "deepseek-r1"), - api_dialects=(pcs.DIALECT_OLLAMA_NATIVE,), - required_capabilities=(mc.CAP_REASONING,), - ), - response_reasoning_paths=("message.thinking", "thinking"), - reasoning_controls=( - mc.ReasoningControl.build( - mechanism=mc.REASONING_CONTROL_NATIVE_BOOL, - values=(mc.REASONING_CONTROL_VALUE_ON, mc.REASONING_CONTROL_VALUE_OFF), - native_values=(True, False), - request_path="think", - response_paths=("message.thinking", "thinking"), - status=mc.ASSERTION_CLAIMED, - source=mc.SOURCE_PROVIDER_DOCS_REGISTRY, - confidence=mc.CONFIDENCE_REGISTRY, - ), - ), - evidence_refs=( - "https://docs.ollama.com/capabilities/thinking", - "github:odysseus-dev/odysseus#3031", - ), - ), - ModelBehaviorQuirk( - quirk_id="ollama.native.gpt-oss-reasoning-level", - selector=ModelBehaviorSelector( - providers=("ollama",), - model_families=("gpt-oss", "gptoss"), - api_dialects=(pcs.DIALECT_OLLAMA_NATIVE,), - required_capabilities=(mc.CAP_REASONING,), - ), - response_reasoning_paths=("message.thinking", "thinking"), - reasoning_controls=( - mc.ReasoningControl.build( - mechanism=mc.REASONING_CONTROL_EFFORT, - values=(mc.REASONING_CONTROL_VALUE_ON,), - native_values=("low", "medium", "high"), - request_path="think", - response_paths=("message.thinking", "thinking"), - status=mc.ASSERTION_CLAIMED, - source=mc.SOURCE_PROVIDER_DOCS_REGISTRY, - confidence=mc.CONFIDENCE_REGISTRY, - ), - ), - evidence_refs=("https://docs.ollama.com/capabilities/thinking",), - ), - ModelBehaviorQuirk( - quirk_id="ollama.openai-compat.0.20.6-reasoning-disable", - selector=ModelBehaviorSelector( - providers=("ollama",), - model_families=("qwen3.5",), - minimum_provider_version=(0, 20, 6), - api_dialects=(pcs.DIALECT_OPENAI_CHAT,), - required_capabilities=(mc.CAP_REASONING,), - ), - request_fixed_values=(("reasoning_effort", "none"),), - status=mc.ASSERTION_CLAIMED, - source=mc.SOURCE_HEURISTIC, - confidence=mc.CONFIDENCE_HEURISTIC, - evidence_refs=("github:odysseus-dev/odysseus#5503",), - ), -) - - -def matching_quirks(**identity: Any) -> tuple[ModelBehaviorQuirk, ...]: - return tuple( - quirk - for quirk in MODEL_BEHAVIOR_QUIRKS - if quirk.selector.matches(**identity) - ) - - -__all__ = [ - "MODEL_BEHAVIOR_QUIRKS", - "ModelBehaviorQuirk", - "ModelBehaviorSelector", - "matching_quirks", -] diff --git a/src/model_capabilities.py b/src/model_capabilities.py index 2929d68ad..d5b3d8507 100644 --- a/src/model_capabilities.py +++ b/src/model_capabilities.py @@ -657,106 +657,6 @@ class DeterministicControl: } -@dataclass(frozen=True) -class ReasoningControl: - """A provider/model-supported request mechanism for reasoning. - - This is intentionally separate from :class:`DeterministicControl` and - from the user's on/off/auto preference. The mechanism records the native - request shape that a later resolver may choose after provider and model - evidence have been reconciled. - """ - - mechanism: str = "" - values: tuple[str, ...] = () - native_values: tuple[Any, ...] = () - request_path: str = "" - response_paths: tuple[str, ...] = () - status: str = ASSERTION_UNKNOWN - source: str = SOURCE_UNKNOWN - confidence: str = CONFIDENCE_UNKNOWN - evidence: tuple[tuple[str, Any], ...] = () - tested_at: str = "" - - @classmethod - def build( - cls, - *, - mechanism: Any, - values: Any = None, - native_values: Any = None, - request_path: Any = "", - response_paths: Any = None, - status: Any = ASSERTION_UNKNOWN, - source: Any = SOURCE_UNKNOWN, - confidence: Any = CONFIDENCE_UNKNOWN, - evidence: Mapping[str, Any] | None = None, - tested_at: Any = "", - ) -> "ReasoningControl": - normalized_mechanism = normalize_reasoning_control_mechanism(mechanism) - normalized_status = normalize_assertion_status(status) - if not normalized_mechanism: - normalized_status = ASSERTION_UNKNOWN - if isinstance(response_paths, str): - response_paths = (response_paths,) - elif isinstance(response_paths, Mapping) or not isinstance(response_paths, Iterable): - response_paths = () - if native_values is None: - native_values = () - elif ( - isinstance(native_values, (str, Mapping)) - or not isinstance(native_values, Iterable) - ): - native_values = (native_values,) - return cls( - mechanism=normalized_mechanism, - values=_normalize_tokens(values, normalize_reasoning_control_value), - native_values=tuple(native_values), - request_path=str(request_path or "").strip(), - response_paths=tuple( - str(path or "").strip() - for path in response_paths - if str(path or "").strip() - ), - status=normalized_status, - source=normalize_source(source), - confidence=normalize_confidence(confidence), - evidence=_normalize_limits(evidence), - tested_at=str(tested_at or "").strip(), - ) - - @classmethod - def from_dict(cls, value: Mapping[str, Any]) -> "ReasoningControl": - if not isinstance(value, Mapping): - return cls.build(mechanism="") - return cls.build( - mechanism=value.get("mechanism"), - values=value.get("values"), - native_values=value.get("native_values"), - request_path=value.get("request_path"), - response_paths=value.get("response_paths"), - status=value.get("status"), - source=value.get("source"), - confidence=value.get("confidence"), - evidence=value.get("evidence"), - tested_at=value.get("tested_at"), - ) - - def to_dict(self) -> dict[str, Any]: - return { - "mechanism": self.mechanism, - "values": list(self.values), - "native_values": list(self.native_values), - "request_path": self.request_path, - "response_paths": list(self.response_paths), - "status": self.status, - "source": self.source, - "confidence": self.confidence, - "evidence": dict(self.evidence), - "tested_at": self.tested_at, - } - - @dataclass(frozen=True) class CapabilityProbeResult: provider: str diff --git a/src/model_capability_readers/__init__.py b/src/model_capability_readers/__init__.py index a9425c524..56ff56e13 100644 --- a/src/model_capability_readers/__init__.py +++ b/src/model_capability_readers/__init__.py @@ -23,6 +23,7 @@ from src.model_capability_readers import ( sglang, ) from src.model_capability_readers.base import ( + CANONICAL_MODEL_SHAPE_VERSION, ModelCapabilityRecord, VENDOR_ANTHROPIC, VENDOR_CEREBRAS, @@ -104,7 +105,7 @@ def records_from_payload( vendor_id = detect_vendor(base_url, endpoint_kind) reader = reader_for_vendor(vendor_id) if reader is generic_openai: - record_vendor = vendor_id if vendor_id not in {VENDOR_UNKNOWN, ""} else VENDOR_GENERIC_OPENAI + record_vendor = vendor_id if vendor_id else VENDOR_UNKNOWN records = reader.records_from_payload( payload, vendor_id=record_vendor, @@ -113,13 +114,12 @@ def records_from_payload( ) else: records = reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url) - shape_id = resolution.catalog_shape.shape_id if resolution.catalog_shape else "" return tuple( replace( record, - provider_schema_id=resolution.schema.provider_id, - catalog_shape_id=shape_id, - provider_resolution=resolution.stage, + provider_source=resolution.provider_source, + catalog_shape_id=resolution.shape_id, + fallback=resolution.fallback, ) for record in records ) @@ -127,6 +127,7 @@ def records_from_payload( __all__ = [ "ModelCapabilityRecord", + "CANONICAL_MODEL_SHAPE_VERSION", "PLACEHOLDER_VENDOR_IDS", "READER_MODULES", "VENDOR_ANTHROPIC", diff --git a/src/model_capability_readers/base.py b/src/model_capability_readers/base.py index 1aa2e6adc..c2651a823 100644 --- a/src/model_capability_readers/base.py +++ b/src/model_capability_readers/base.py @@ -44,6 +44,8 @@ VENDOR_XAI = "xai" VENDOR_ZAI = "zai" VENDOR_UNKNOWN = "unknown" +CANONICAL_MODEL_SHAPE_VERSION = 1 + @dataclass(frozen=True) class ModelCapabilityRecord: @@ -54,14 +56,9 @@ class ModelCapabilityRecord: stable_model_id: str = "" capability_assertions: tuple[mc.CapabilityAssertion, ...] = () deterministic_controls: tuple[mc.DeterministicControl, ...] = () - reasoning_controls: tuple[mc.ReasoningControl, ...] = () - model_family: str = "" - model_version: tuple[int, ...] = () - provider_version: tuple[int, ...] = () - api_dialect: str = "" - provider_schema_id: str = "" + provider_source: str = "unknown" catalog_shape_id: str = "" - provider_resolution: str = "" + fallback: bool = False raw: Mapping[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: @@ -80,22 +77,31 @@ class ModelCapabilityRecord: ) def to_dict(self, *, include_raw: bool = False) -> dict[str, Any]: + controls = tuple( + dict.fromkeys( + control.control + for control in self.deterministic_controls + if control.control + ) + ) data = { - "vendor": self.vendor, - "model_id": self.model_id, - "stable_model_id": self.stable_model_id, - "display_name": self.display_name, - "capability": self.capability.to_dict(), - "capability_assertions": [assertion.to_dict() for assertion in self.capability_assertions], - "deterministic_controls": [control.to_dict() for control in self.deterministic_controls], - "reasoning_controls": [control.to_dict() for control in self.reasoning_controls], - "model_family": self.model_family, - "model_version": list(self.model_version), - "provider_version": list(self.provider_version), - "api_dialect": self.api_dialect, - "provider_schema_id": self.provider_schema_id, - "catalog_shape_id": self.catalog_shape_id, - "provider_resolution": self.provider_resolution, + "schema_version": CANONICAL_MODEL_SHAPE_VERSION, + "provider": self.vendor, + "model": self.model_id, + "stable_id": self.stable_model_id, + "family": self.capability.family, + "task": self.capability.primary_task, + "modalities": self.capability.modalities.to_dict(), + "features": list(self.capability.capabilities), + "limits": dict(self.capability.limits), + "controls": list(controls), + "evidence": { + "source": self.capability.source, + "confidence": self.capability.confidence, + "provider_source": self.provider_source, + "shape": self.catalog_shape_id, + "fallback": self.fallback, + }, } if include_raw: data["raw"] = dict(self.raw) @@ -314,5 +320,4 @@ def detect_vendor(base_url: Any = "", endpoint_kind: Any = "") -> str: ) if resolution.provider_id != pcs.PROVIDER_UNKNOWN: return resolution.provider_id - parsed = urlparse(compact_str(base_url)) - return VENDOR_GENERIC_OPENAI if parsed.hostname else VENDOR_UNKNOWN + return VENDOR_UNKNOWN diff --git a/src/model_capability_readers/chatgpt_subscription.py b/src/model_capability_readers/chatgpt_subscription.py index 81400f1b9..a1ddc4399 100644 --- a/src/model_capability_readers/chatgpt_subscription.py +++ b/src/model_capability_readers/chatgpt_subscription.py @@ -42,7 +42,6 @@ def record_from_model( source=mc.SOURCE_PROVIDER_READER, confidence=mc.CONFIDENCE_UNKNOWN, ), - model_family=compact_str(raw.get("family")), raw=raw, ) diff --git a/src/model_capability_readers/copilot.py b/src/model_capability_readers/copilot.py index 99920d0a0..ffc0c670e 100644 --- a/src/model_capability_readers/copilot.py +++ b/src/model_capability_readers/copilot.py @@ -94,7 +94,6 @@ def record_from_model( ), display_name=compact_str(raw.get("name")) or model_id, capability=capability, - model_family=compact_str(raw.get("family")), raw=raw, ) diff --git a/src/model_capability_readers/generic_openai.py b/src/model_capability_readers/generic_openai.py index 5759405b2..b101251ba 100644 --- a/src/model_capability_readers/generic_openai.py +++ b/src/model_capability_readers/generic_openai.py @@ -1,9 +1,8 @@ -"""General structural reader for OpenAI-compatible model-list payloads. +"""Inventory-only reader for unrecognized model-list envelopes. -Identity-only model cards remain unknown. Rich records are promoted only from -recognized explicit fields (modalities, task/type, capability booleans, -supported parameters, and numeric limits). Names and descriptions are never -parsed for capability hints. +Common field names are not a cross-provider capability contract. This reader +therefore recovers model identity and preserves the original record, but never +promotes tasks, modalities, parameters, limits, or capability booleans. """ from __future__ import annotations @@ -15,18 +14,9 @@ from src import model_capabilities as mc from src.model_capability_readers.base import ( ModelCapabilityRecord, VENDOR_GENERIC_OPENAI, - as_list, - as_mapping, - build_capability, compact_str, - deterministic_controls_from_supported_parameters, - family_from_modalities, - int_limit, - merge_unique, model_id_from, - modalities_from_value, openai_model_items, - split_modality_arrow, stable_model_id_for, ) @@ -34,191 +24,6 @@ from src.model_capability_readers.base import ( vendor = VENDOR_GENERIC_OPENAI -_TYPE_FAMILIES = { - "llm": mc.FAMILY_CHAT, - "chat": mc.FAMILY_CHAT, - "chat_completion": mc.FAMILY_CHAT, - "text_generation": mc.FAMILY_CHAT, - "causal_lm": mc.FAMILY_CHAT, - "image_text_to_text": mc.FAMILY_CHAT, - "image_question_answering": mc.FAMILY_CHAT, - "embedding": mc.FAMILY_EMBEDDING, - "embeddings": mc.FAMILY_EMBEDDING, - "text_embedding": mc.FAMILY_EMBEDDING, - "feature_extraction": mc.FAMILY_EMBEDDING, - "text_to_image": mc.FAMILY_IMAGE, - "image_to_image": mc.FAMILY_IMAGE, - "text_to_video": mc.FAMILY_VIDEO, - "automatic_speech_recognition": mc.FAMILY_AUDIO, - "text_to_speech": mc.FAMILY_AUDIO, - "rerank": mc.FAMILY_RERANK, - "reranking": mc.FAMILY_RERANK, - "classification": mc.FAMILY_CLASSIFICATION, - "text_classification": mc.FAMILY_CLASSIFICATION, - "moderation": mc.FAMILY_MODERATION, -} - -_PARAMETER_CAPABILITIES = { - "tools": mc.CAP_TOOL_CALL, - "tool_choice": mc.CAP_TOOL_CALL, - "parallel_tool_calls": mc.CAP_TOOL_CALL, - "function_calling": mc.CAP_TOOL_CALL, - "response_format": mc.CAP_JSON_MODE, - "structured_output": mc.CAP_STRUCTURED_OUTPUT, - "structured_outputs": mc.CAP_STRUCTURED_OUTPUT, - "json_schema": mc.CAP_STRUCTURED_OUTPUT, - "reasoning": mc.CAP_REASONING, - "reasoning_effort": mc.CAP_REASONING, - "include_reasoning": mc.CAP_REASONING, - "web_search": mc.CAP_WEB_SEARCH, - "web_search_options": mc.CAP_WEB_SEARCH, -} - - -def _shape_token(value: Any) -> str: - return compact_str(value).lower().replace("-", "_").replace(" ", "_") - - -def _family_from_explicit_fields(raw: Mapping[str, Any]) -> str: - for key in ("type", "model_type", "task", "pipeline_tag"): - family = _TYPE_FAMILIES.get(_shape_token(raw.get(key))) - if family: - return family - return mc.FAMILY_UNKNOWN - - -def _modalities(raw: Mapping[str, Any]) -> tuple[tuple[str, ...], tuple[str, ...]]: - architecture = as_mapping(raw.get("architecture")) - input_modalities = modalities_from_value( - raw.get("input_modalities") or architecture.get("input_modalities") - ) - output_modalities = modalities_from_value( - raw.get("output_modalities") or architecture.get("output_modalities") - ) - if not input_modalities or not output_modalities: - arrow_input, arrow_output = split_modality_arrow( - raw.get("modality") or architecture.get("modality") - ) - input_modalities = input_modalities or arrow_input - output_modalities = output_modalities or arrow_output - return input_modalities, output_modalities - - -def _capabilities_from_modalities( - input_modalities: tuple[str, ...], - output_modalities: tuple[str, ...], -) -> tuple[str, ...]: - input_set = set(input_modalities) - output_set = set(output_modalities) - out: list[str] = [] - if mc.MODALITY_IMAGE in input_set and mc.MODALITY_TEXT in output_set: - out.append(mc.CAP_VISION) - if mc.MODALITY_FILE in input_set: - out.append(mc.CAP_FILES) - if mc.MODALITY_PDF in input_set: - out.append(mc.CAP_PDF) - if mc.MODALITY_AUDIO in input_set: - out.append(mc.CAP_AUDIO_INPUT) - if mc.MODALITY_AUDIO in output_set: - out.append(mc.CAP_AUDIO_OUTPUT) - if mc.MODALITY_IMAGE in output_set: - out.append(mc.CAP_IMAGE_GENERATION) - if mc.MODALITY_IMAGE in input_set: - out.append(mc.CAP_IMAGE_EDITING) - if mc.MODALITY_VIDEO in output_set: - out.append(mc.CAP_VIDEO_GENERATION) - return tuple(out) - - -def _explicit_capabilities(raw: Mapping[str, Any]) -> tuple[str, ...]: - values: list[Any] = [] - payload = raw.get("capabilities") - if isinstance(payload, Mapping): - supports = payload.get("supports") - if isinstance(supports, Mapping): - values.extend(key for key, enabled in supports.items() if enabled is True) - values.extend(key for key, enabled in payload.items() if enabled is True) - elif isinstance(payload, (list, tuple)): - values.extend(payload) - - out: list[str] = [] - for value in values: - cap = mc.normalize_capability(value) - if cap and cap not in out: - out.append(cap) - for value in as_list(raw.get("supported_parameters")): - cap = _PARAMETER_CAPABILITIES.get(_shape_token(value)) - if cap and cap not in out: - out.append(cap) - task = next( - (_shape_token(raw.get(key)) for key in ("type", "model_type", "task", "pipeline_tag") if raw.get(key)), - "", - ) - task_capability = { - "automatic_speech_recognition": mc.CAP_TRANSCRIPTION, - "text_to_speech": mc.CAP_TTS, - "text_to_image": mc.CAP_IMAGE_GENERATION, - "image_to_image": mc.CAP_IMAGE_EDITING, - "text_to_video": mc.CAP_VIDEO_GENERATION, - "image_text_to_text": mc.CAP_VISION, - "image_question_answering": mc.CAP_VISION, - }.get(task) - if task_capability and task_capability not in out: - out.append(task_capability) - return tuple(out) - - -def _limits(raw: Mapping[str, Any]) -> dict[str, int]: - architecture = as_mapping(raw.get("architecture")) - top_provider = as_mapping(raw.get("top_provider")) - limits: dict[str, int] = {} - for keys, target in ( - (("context_length", "max_context_length", "max_model_len"), "context_tokens"), - (("input_token_limit", "inputTokenLimit"), "input_tokens"), - (("output_token_limit", "outputTokenLimit", "max_completion_tokens"), "output_tokens"), - ): - for key in keys: - value = int_limit(raw.get(key)) or int_limit(architecture.get(key)) or int_limit(top_provider.get(key)) - if value: - limits[target] = value - break - return limits - - -def _default_modalities( - family: str, - raw: Mapping[str, Any], -) -> tuple[tuple[str, ...], tuple[str, ...]]: - task = next( - (_shape_token(raw.get(key)) for key in ("type", "model_type", "task", "pipeline_tag") if raw.get(key)), - "", - ) - task_modalities = { - "automatic_speech_recognition": ((mc.MODALITY_AUDIO,), (mc.MODALITY_TEXT,)), - "text_to_speech": ((mc.MODALITY_TEXT,), (mc.MODALITY_AUDIO,)), - "text_to_image": ((mc.MODALITY_TEXT,), (mc.MODALITY_IMAGE,)), - "image_to_image": ((mc.MODALITY_IMAGE,), (mc.MODALITY_IMAGE,)), - "text_to_video": ((mc.MODALITY_TEXT,), (mc.MODALITY_VIDEO,)), - "image_text_to_text": ((mc.MODALITY_TEXT, mc.MODALITY_IMAGE), (mc.MODALITY_TEXT,)), - "image_question_answering": ((mc.MODALITY_TEXT, mc.MODALITY_IMAGE), (mc.MODALITY_TEXT,)), - }.get(task) - if task_modalities: - return task_modalities - if family == mc.FAMILY_CHAT: - return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,) - if family == mc.FAMILY_EMBEDDING: - return (mc.MODALITY_TEXT,), (mc.MODALITY_EMBEDDING,) - if family == mc.FAMILY_IMAGE: - return (mc.MODALITY_TEXT,), (mc.MODALITY_IMAGE,) - if family == mc.FAMILY_VIDEO: - return (mc.MODALITY_TEXT,), (mc.MODALITY_VIDEO,) - if family == mc.FAMILY_AUDIO: - return (), () - if family in {mc.FAMILY_RERANK, mc.FAMILY_CLASSIFICATION, mc.FAMILY_MODERATION}: - return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,) - return (), () - - def record_from_model( raw: Mapping[str, Any], *, @@ -226,44 +31,28 @@ def record_from_model( endpoint_id: Any = "", base_url: Any = "", ) -> ModelCapabilityRecord | None: - model_id = model_id_from(raw, "id", "name", "model") + model_id = model_id_from(raw, "id", "name", "model", "key", "slug") if not model_id: return None - - family = _family_from_explicit_fields(raw) - input_modalities, output_modalities = _modalities(raw) - if family == mc.FAMILY_UNKNOWN: - family = family_from_modalities(input_modalities, output_modalities) - if family != mc.FAMILY_UNKNOWN and not input_modalities and not output_modalities: - input_modalities, output_modalities = _default_modalities(family, raw) - capabilities = merge_unique( - _explicit_capabilities(raw), - _capabilities_from_modalities(input_modalities, output_modalities), - ) - limits = _limits(raw) - if family == mc.FAMILY_UNKNOWN and not capabilities and not limits: - capability = mc.unknown_capability( - source=mc.SOURCE_PROVIDER_READER, - confidence=mc.CONFIDENCE_UNKNOWN, - ) - else: - capability = build_capability( - family=family, - input_modalities=input_modalities, - output_modalities=output_modalities, - capabilities=capabilities, - limits=limits, - ) return ModelCapabilityRecord( vendor=vendor_id, model_id=model_id, - stable_model_id=stable_model_id_for(vendor_id, model_id, endpoint_id=endpoint_id, base_url=base_url), - display_name=compact_str(raw.get("display_name") or raw.get("name")), - capability=capability, - deterministic_controls=deterministic_controls_from_supported_parameters( - raw.get("supported_parameters") + stable_model_id=stable_model_id_for( + vendor_id, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=compact_str( + raw.get("display_name") + or raw.get("name") + or raw.get("key") + or raw.get("slug") + ), + capability=mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, ), - model_family=compact_str(raw.get("root") or raw.get("model_family")), raw=raw, ) @@ -277,7 +66,12 @@ def records_from_payload( ) -> tuple[ModelCapabilityRecord, ...]: records: list[ModelCapabilityRecord] = [] for item in openai_model_items(payload): - record = record_from_model(item, vendor_id=vendor_id, endpoint_id=endpoint_id, base_url=base_url) + record = record_from_model( + item, + vendor_id=vendor_id, + endpoint_id=endpoint_id, + base_url=base_url, + ) if record: records.append(record) return tuple(records) diff --git a/src/model_capability_readers/google.py b/src/model_capability_readers/google.py index f6d0bfe28..9edb57bdb 100644 --- a/src/model_capability_readers/google.py +++ b/src/model_capability_readers/google.py @@ -42,7 +42,6 @@ def record_from_model( display_name=compact_str(raw.get("displayName")) or model_id, capability=ai_studio.capability_from_model(raw), deterministic_controls=ai_studio.deterministic_controls_from_model(raw), - model_family=compact_str(raw.get("baseModelId")), raw=raw, ) diff --git a/src/model_capability_readers/huggingface.py b/src/model_capability_readers/huggingface.py index edb34d471..c260513ea 100644 --- a/src/model_capability_readers/huggingface.py +++ b/src/model_capability_readers/huggingface.py @@ -6,11 +6,10 @@ from collections.abc import Mapping from typing import Any from src import model_capabilities as mc -from src.model_capability_readers import generic_openai from src.model_capability_readers.base import ( ModelCapabilityRecord, VENDOR_HUGGINGFACE, - as_mapping, + build_capability, compact_str, openai_model_items, stable_model_id_for, @@ -20,6 +19,86 @@ from src.model_capability_readers.base import ( vendor = VENDOR_HUGGINGFACE +# Hugging Face publishes ``pipeline_tag`` as a provider-owned task enum. Keep +# its interpretation here, rather than teaching the inventory fallback that a +# similarly named field has the same meaning for every provider. +_PIPELINE_SHAPES = { + "text-generation": (mc.FAMILY_CHAT, (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,), ()), + "image-text-to-text": ( + mc.FAMILY_CHAT, + (mc.MODALITY_TEXT, mc.MODALITY_IMAGE), + (mc.MODALITY_TEXT,), + (mc.CAP_VISION,), + ), + "image-question-answering": ( + mc.FAMILY_CHAT, + (mc.MODALITY_TEXT, mc.MODALITY_IMAGE), + (mc.MODALITY_TEXT,), + (mc.CAP_VISION,), + ), + "feature-extraction": ( + mc.FAMILY_EMBEDDING, + (mc.MODALITY_TEXT,), + (mc.MODALITY_EMBEDDING,), + (), + ), + "text-to-image": ( + mc.FAMILY_IMAGE, + (mc.MODALITY_TEXT,), + (mc.MODALITY_IMAGE,), + (mc.CAP_IMAGE_GENERATION,), + ), + "image-to-image": ( + mc.FAMILY_IMAGE, + (mc.MODALITY_IMAGE,), + (mc.MODALITY_IMAGE,), + (mc.CAP_IMAGE_GENERATION, mc.CAP_IMAGE_EDITING), + ), + "text-to-video": ( + mc.FAMILY_VIDEO, + (mc.MODALITY_TEXT,), + (mc.MODALITY_VIDEO,), + (mc.CAP_VIDEO_GENERATION,), + ), + "automatic-speech-recognition": ( + mc.FAMILY_AUDIO, + (mc.MODALITY_AUDIO,), + (mc.MODALITY_TEXT,), + (mc.CAP_TRANSCRIPTION,), + ), + "text-to-speech": ( + mc.FAMILY_AUDIO, + (mc.MODALITY_TEXT,), + (mc.MODALITY_AUDIO,), + (mc.CAP_TTS,), + ), + "text-classification": ( + mc.FAMILY_CLASSIFICATION, + (mc.MODALITY_TEXT,), + (mc.MODALITY_TEXT,), + (), + ), +} + + +def _capability_from_pipeline_tag(value: Any) -> mc.ModelCapability: + shape = _PIPELINE_SHAPES.get(compact_str(value).lower()) + if not shape: + return mc.unknown_capability( + source=mc.SOURCE_COOKBOOK_HF, + confidence=mc.CONFIDENCE_UNKNOWN, + ) + family, input_modalities, output_modalities, capabilities = shape + return build_capability( + family=family, + input_modalities=input_modalities, + output_modalities=output_modalities, + capabilities=capabilities, + source=mc.SOURCE_COOKBOOK_HF, + confidence=mc.CONFIDENCE_REGISTRY, + ) + + def record_from_model( raw: Mapping[str, Any], *, @@ -29,25 +108,6 @@ def record_from_model( model_id = compact_str(raw.get("modelId") or raw.get("id")) if not model_id: return None - structural = generic_openai.record_from_model( - {**raw, "id": model_id}, - vendor_id=VENDOR_HUGGINGFACE, - endpoint_id=endpoint_id, - base_url=base_url, - ) - if not structural: - return None - capability = mc.ModelCapability.build( - family=structural.capability.family, - primary_task=structural.capability.primary_task, - input_modalities=structural.capability.modalities.input, - output_modalities=structural.capability.modalities.output, - capabilities=structural.capability.capabilities, - limits=dict(structural.capability.limits), - source=mc.SOURCE_COOKBOOK_HF, - confidence=mc.CONFIDENCE_REGISTRY, - ) - config = as_mapping(raw.get("config")) return ModelCapabilityRecord( vendor=VENDOR_HUGGINGFACE, model_id=model_id, @@ -65,9 +125,7 @@ def record_from_model( ) or model_id ), - capability=capability, - deterministic_controls=structural.deterministic_controls, - model_family=compact_str(config.get("model_type")), + capability=_capability_from_pipeline_tag(raw.get("pipeline_tag")), raw=raw, ) diff --git a/src/model_capability_readers/lmstudio.py b/src/model_capability_readers/lmstudio.py index 919055936..960649959 100644 --- a/src/model_capability_readers/lmstudio.py +++ b/src/model_capability_readers/lmstudio.py @@ -160,7 +160,6 @@ def record_from_native_model( ), display_name=compact_str(raw.get("display_name") or raw.get("name")) or model_id, capability=capability, - model_family=compact_str(raw.get("architecture") or raw.get("arch")), raw=raw, ) diff --git a/src/model_capability_readers/mistral.py b/src/model_capability_readers/mistral.py index 6a316169a..7788984d6 100644 --- a/src/model_capability_readers/mistral.py +++ b/src/model_capability_readers/mistral.py @@ -93,7 +93,6 @@ def record_from_model( ), display_name=compact_str(raw.get("name")) or model_id, capability=capability, - model_family=compact_str(raw.get("root")), raw=raw, ) diff --git a/src/model_capability_readers/ollama.py b/src/model_capability_readers/ollama.py index cd21f6c0e..e9a1fdfeb 100644 --- a/src/model_capability_readers/ollama.py +++ b/src/model_capability_readers/ollama.py @@ -142,7 +142,6 @@ def record_from_show_payload( stable_model_id=stable_model_id_for(VENDOR_OLLAMA, model_id, endpoint_id=endpoint_id, base_url=base_url), display_name=model_id, capability=capability, - model_family=compact_str(as_mapping(payload.get("details")).get("family")), raw=payload, ) @@ -175,7 +174,6 @@ def records_from_tags_payload( source=mc.SOURCE_PROVIDER_READER, confidence=mc.CONFIDENCE_UNKNOWN, ), - model_family=compact_str(as_mapping(item.get("details")).get("family")), raw=item, ) ) diff --git a/src/model_capability_readers/sglang.py b/src/model_capability_readers/sglang.py index 5b6b96c63..3ac59b060 100644 --- a/src/model_capability_readers/sglang.py +++ b/src/model_capability_readers/sglang.py @@ -72,7 +72,6 @@ def record_from_model_info( display_name=model_id, capability=capability, deterministic_controls=deterministic_controls_from_supported_parameters(sampling.keys()), - model_family=compact_str(payload.get("model_type")), raw=payload, ) diff --git a/src/provider_capability_schemas.py b/src/provider_capability_schemas.py index 531cbd19b..55f4809da 100644 --- a/src/provider_capability_schemas.py +++ b/src/provider_capability_schemas.py @@ -1,20 +1,12 @@ -"""Canonical serving-provider and payload-shape metadata. +"""Provider identity and native model-catalog shape detection. -This module describes provider/API dialects and model-catalog JSON shapes. It -does not perform network I/O, parse model names, or claim that every model on a -provider supports every feature exposed by that provider. Model capability is -still read from each model record (or remains unknown). +The registry has one narrow job: identify a configured provider and recognize +tested provider-native catalog envelopes. Generic ``data``/``models``/list +envelopes are marked as fallback inventory only; they never promote model +capabilities. -Resolution is deliberately stepped and deterministic: - -1. an explicit provider/endpoint kind; -2. an exact known provider host; -3. a discriminating native payload shape; -4. a general structural model-list shape; -5. unknown. - -Unknown keys are left to the reader's ``raw`` evidence. They never become a -capability merely because a future provider happens to add them. +Request/response transport fields and model-specific behavior belong to their +runtime adapters, not this catalog detector. """ from __future__ import annotations @@ -28,24 +20,11 @@ from urllib.parse import urlparse PROVIDER_UNKNOWN = "unknown" PROVIDER_GENERIC_OPENAI = "generic_openai" -DIALECT_OPENAI_CHAT = "openai_chat_completions" -DIALECT_OPENAI_RESPONSES = "openai_responses" -DIALECT_ANTHROPIC_MESSAGES = "anthropic_messages" -DIALECT_COHERE_V2 = "cohere_v2" -DIALECT_GOOGLE_GENERATE_CONTENT = "google_generate_content" -DIALECT_OLLAMA_NATIVE = "ollama_native" -DIALECT_LMSTUDIO_NATIVE_V1 = "lmstudio_native_v1" -DIALECT_LLAMACPP_NATIVE = "llamacpp_native" -DIALECT_SGLANG_NATIVE = "sglang_native" -DIALECT_HUGGINGFACE_HUB = "huggingface_hub" -DIALECT_CHATGPT_SUBSCRIPTION = "chatgpt_subscription_responses" - -RESOLUTION_EXPLICIT = "explicit" -RESOLUTION_ENDPOINT_KIND = "endpoint_kind" -RESOLUTION_HOST = "host" -RESOLUTION_NATIVE_SHAPE = "native_shape" -RESOLUTION_GENERAL_SHAPE = "general_shape" -RESOLUTION_UNKNOWN = "unknown" +PROVIDER_SOURCE_EXPLICIT = "explicit" +PROVIDER_SOURCE_ENDPOINT_KIND = "endpoint_kind" +PROVIDER_SOURCE_HOST = "host" +PROVIDER_SOURCE_PAYLOAD = "payload" +PROVIDER_SOURCE_UNKNOWN = "unknown" ENVELOPE_DATA = "data" ENVELOPE_MODELS = "models" @@ -87,11 +66,10 @@ def _items_for_envelope(payload: Any, envelope: str) -> tuple[Mapping[str, Any], @dataclass(frozen=True) class ProviderCatalogShape: - """A declarative, versioned provider model-catalog shape.""" + """A tested provider-native shape or an explicit inventory fallback.""" shape_id: str provider_id: str - endpoint_path: str envelope: str identity_paths: tuple[str, ...] required_root_paths: tuple[str, ...] = () @@ -99,10 +77,8 @@ class ProviderCatalogShape: required_item_any_paths: tuple[str, ...] = () item_types: tuple[tuple[str, tuple[Any, ...]], ...] = () item_values: tuple[tuple[str, tuple[Any, ...]], ...] = () - capability_paths: tuple[str, ...] = () - api_version: str = "" - priority: int = 0 - latest: bool = True + detection_priority: int = 0 + fallback: bool = False def items(self, payload: Any) -> tuple[Mapping[str, Any], ...]: return _items_for_envelope(payload, self.envelope) @@ -114,19 +90,14 @@ class ProviderCatalogShape: if not all(_path_present(payload, path) for path in self.required_root_paths): return False - items = self.items(payload) - if not items: - return False - for item in items: - if self.identity_paths: - has_identity = False - for path in self.identity_paths: - value = _path_value(item, path) - if value is not _MISSING and value is not None and value != "": - has_identity = True - break - if not has_identity: - continue + for item in self.items(payload): + if self.identity_paths and not any( + (value := _path_value(item, path)) is not _MISSING + and value is not None + and value != "" + for path in self.identity_paths + ): + continue if not all(_path_present(item, path) for path in self.required_item_paths): continue if self.required_item_any_paths and not any( @@ -144,329 +115,148 @@ class ProviderCatalogShape: return False -@dataclass(frozen=True) -class ProviderApiShape: - """Stable request/response field paths for one API dialect. - - Paths are documentation and validation inputs, not late response parsers. - A model-specific exception can narrow these fields in the model quirk - registry without changing the provider's general transport contract. - """ - - dialect: str - request_path: str - stream_path: str = "" - model_field: str = "model" - message_field: str = "messages" - tool_request_paths: tuple[str, ...] = () - tool_response_paths: tuple[str, ...] = () - text_response_paths: tuple[str, ...] = () - reasoning_response_paths: tuple[str, ...] = () - request_control_paths: tuple[str, ...] = () - - @dataclass(frozen=True) class ProviderCapabilitySchema: provider_id: str - display_name: str aliases: tuple[str, ...] = () host_suffixes: tuple[str, ...] = () - api_shapes: tuple[ProviderApiShape, ...] = () catalog_shapes: tuple[ProviderCatalogShape, ...] = () - fallback_provider_id: str = PROVIDER_GENERIC_OPENAI - model_capabilities_are_per_model: bool = True @dataclass(frozen=True) class ProviderResolution: - provider_id: str - stage: str - schema: ProviderCapabilitySchema - catalog_shape: ProviderCatalogShape | None = None + provider_id: str = PROVIDER_UNKNOWN + provider_source: str = PROVIDER_SOURCE_UNKNOWN + shape_id: str = "" + fallback: bool = False def to_dict(self) -> dict[str, Any]: return { - "provider_id": self.provider_id, - "stage": self.stage, - "schema_id": self.schema.provider_id, - "catalog_shape_id": self.catalog_shape.shape_id if self.catalog_shape else "", + "provider": self.provider_id, + "provider_source": self.provider_source, + "shape": self.shape_id, + "fallback": self.fallback, } -OPENAI_CHAT_SHAPE = ProviderApiShape( - dialect=DIALECT_OPENAI_CHAT, - request_path="/v1/chat/completions", - stream_path="/v1/chat/completions", - tool_request_paths=("tools[].function", "tool_choice", "parallel_tool_calls"), - tool_response_paths=("choices[].message.tool_calls[].function", "choices[].delta.tool_calls[].function"), - text_response_paths=("choices[].message.content", "choices[].delta.content"), - reasoning_response_paths=( - "choices[].message.reasoning_content", - "choices[].delta.reasoning_content", - "choices[].delta.reasoning", - "choices[].delta.thinking", - ), - request_control_paths=( - "temperature", - "top_p", - "seed", - "response_format", - "reasoning_effort", - ), -) - -OPENAI_RESPONSES_SHAPE = ProviderApiShape( - dialect=DIALECT_OPENAI_RESPONSES, - request_path="/v1/responses", - stream_path="/v1/responses", - message_field="input", - tool_request_paths=("tools[]", "tool_choice", "parallel_tool_calls"), - tool_response_paths=( - "output[type=function_call].arguments", - "response.function_call_arguments.delta", - ), - text_response_paths=( - "output[type=message].content[type=output_text].text", - "response.output_text.delta", - ), - reasoning_response_paths=( - "output[type=reasoning].summary[].text", - "output[type=reasoning].encrypted_content", - "response.reasoning_summary_text.delta", - ), - request_control_paths=("temperature", "top_p", "reasoning", "text.format"), -) - -ANTHROPIC_MESSAGES_SHAPE = ProviderApiShape( - dialect=DIALECT_ANTHROPIC_MESSAGES, - request_path="/v1/messages", - stream_path="/v1/messages", - tool_request_paths=("tools[].input_schema", "tool_choice", "messages[].content[].tool_result"), - tool_response_paths=("content[].tool_use", "content_block_delta.delta.partial_json"), - text_response_paths=("content[].text", "content_block_delta.delta.text"), - reasoning_response_paths=("content[].thinking", "content[].signature"), - request_control_paths=("temperature", "top_p", "top_k", "thinking", "output_config"), -) - -COHERE_V2_SHAPE = ProviderApiShape( - dialect=DIALECT_COHERE_V2, - request_path="/v2/chat", - stream_path="/v2/chat", - tool_request_paths=("tools[].parameters", "messages[].tool_calls", "messages[].tool_results"), - tool_response_paths=("message.tool_calls", "tool-call-start", "tool-call-delta", "tool-call-end"), - text_response_paths=("message.content[].text", "content-delta.delta.message.content.text"), - reasoning_response_paths=("message.content[].thinking",), - request_control_paths=( - "temperature", - "p", - "k", - "seed", - "response_format", - "thinking", - ), -) - -GOOGLE_CONTENT_SHAPE = ProviderApiShape( - dialect=DIALECT_GOOGLE_GENERATE_CONTENT, - request_path="/v1beta/models/{model}:generateContent", - stream_path="/v1beta/models/{model}:streamGenerateContent?alt=sse", - message_field="contents", - tool_request_paths=("tools[].functionDeclarations", "toolConfig"), - tool_response_paths=("candidates[].content.parts[].functionCall", "candidates[].content.parts[].functionResponse"), - text_response_paths=("candidates[].content.parts[].text",), - reasoning_response_paths=("candidates[].content.parts[].thought", "candidates[].content.parts[].thoughtSignature"), - request_control_paths=( - "generationConfig.temperature", - "generationConfig.topP", - "generationConfig.topK", - "generationConfig.thinkingConfig", - "generationConfig.responseJsonSchema", - ), -) - -OLLAMA_NATIVE_SHAPE = ProviderApiShape( - dialect=DIALECT_OLLAMA_NATIVE, - request_path="/api/chat", - stream_path="/api/chat", - tool_request_paths=("tools[].function", "messages[].tool_calls[].function"), - tool_response_paths=("message.tool_calls[].function",), - text_response_paths=("message.content",), - reasoning_response_paths=("message.thinking",), - request_control_paths=("think", "format", "options.temperature", "options.top_p", "options.seed"), -) - -LMSTUDIO_NATIVE_SHAPE = ProviderApiShape( - dialect=DIALECT_LMSTUDIO_NATIVE_V1, - request_path="/api/v1/chat", - stream_path="/api/v1/chat", - message_field="input", - tool_request_paths=("integrations[].mcp",), - text_response_paths=("output[].content",), - reasoning_response_paths=("output[].reasoning",), - request_control_paths=("temperature", "top_p", "reasoning", "response_format"), -) - -SGLANG_NATIVE_SHAPE = ProviderApiShape( - dialect=DIALECT_SGLANG_NATIVE, - request_path="/generate", - stream_path="/generate", - message_field="text", - text_response_paths=("text",), - request_control_paths=("sampling_params.temperature", "sampling_params.top_p", "sampling_params.seed"), -) - -CHATGPT_SUBSCRIPTION_SHAPE = ProviderApiShape( - dialect=DIALECT_CHATGPT_SUBSCRIPTION, - request_path="/backend-api/codex/responses", - stream_path="/backend-api/codex/responses", - message_field="input", - tool_request_paths=("tools[]", "input[].function_call_output"), - tool_response_paths=("response.function_call_arguments.delta", "response.output_item.done"), - text_response_paths=("response.output_text.delta",), - reasoning_response_paths=("response.reasoning_summary_text.delta", "reasoning.encrypted_content"), - request_control_paths=("reasoning", "text", "parallel_tool_calls"), -) - - +# Generic envelopes are inventory fallbacks only. Their field names are not a +# portable capability contract, so readers may recover identity but nothing +# else from them. GENERAL_DATA_SHAPE = ProviderCatalogShape( - shape_id="openai-compatible.models.data.v1", - provider_id=PROVIDER_GENERIC_OPENAI, - endpoint_path="/v1/models", + shape_id="fallback.models.data.v1", + provider_id=PROVIDER_UNKNOWN, envelope=ENVELOPE_DATA, - identity_paths=("id", "name", "model"), + identity_paths=("id", "name", "model", "key", "slug"), + fallback=True, ) GENERAL_MODELS_SHAPE = ProviderCatalogShape( - shape_id="general.models-envelope.v1", - provider_id=PROVIDER_GENERIC_OPENAI, - endpoint_path="/models", + shape_id="fallback.models.envelope.v1", + provider_id=PROVIDER_UNKNOWN, envelope=ENVELOPE_MODELS, - identity_paths=("id", "key", "slug", "name", "model"), + identity_paths=("id", "name", "model", "key", "slug"), + fallback=True, ) GENERAL_BARE_SHAPE = ProviderCatalogShape( - shape_id="openai-compatible.models.bare-list.v1", - provider_id=PROVIDER_GENERIC_OPENAI, - endpoint_path="/models", + shape_id="fallback.models.list.v1", + provider_id=PROVIDER_UNKNOWN, envelope=ENVELOPE_BARE_LIST, - identity_paths=("id", "name", "model"), + identity_paths=("id", "name", "model", "key", "slug"), + fallback=True, ) +FALLBACK_CATALOG_SHAPES = ( + GENERAL_DATA_SHAPE, + GENERAL_MODELS_SHAPE, + GENERAL_BARE_SHAPE, +) + OPENAI_MODELS_SHAPE = ProviderCatalogShape( shape_id="openai.models.identity.v1", provider_id="openai", - endpoint_path="/v1/models", envelope=ENVELOPE_DATA, identity_paths=("id",), required_item_paths=("object", "created", "owned_by"), item_values=(("object", ("model",)),), - api_version="v1", ) OPENROUTER_MODELS_SHAPE = ProviderCatalogShape( shape_id="openrouter.models.rich.v1", provider_id="openrouter", - endpoint_path="/api/v1/models", envelope=ENVELOPE_DATA, identity_paths=("id",), - required_item_any_paths=("architecture", "supported_parameters", "top_provider", "canonical_slug"), - capability_paths=( - "architecture.input_modalities", - "architecture.output_modalities", + required_item_any_paths=( + "architecture", "supported_parameters", - "context_length", - "top_provider.max_completion_tokens", + "top_provider", + "canonical_slug", ), - api_version="v1", - priority=90, + detection_priority=90, ) GOOGLE_MODELS_SHAPE = ProviderCatalogShape( shape_id="google.generative-language.models.v1beta", provider_id="google", - endpoint_path="/v1beta/models", envelope=ENVELOPE_MODELS, identity_paths=("baseModelId", "name"), required_item_paths=("supportedGenerationMethods",), item_types=(("supportedGenerationMethods", (list, tuple)),), - capability_paths=( - "supportedGenerationMethods", - "inputTokenLimit", - "outputTokenLimit", - "thinking", - "temperature", - "topP", - "topK", - ), - api_version="v1beta", - priority=100, + detection_priority=100, ) OLLAMA_TAGS_SHAPE = ProviderCatalogShape( shape_id="ollama.tags.v1", provider_id="ollama", - endpoint_path="/api/tags", envelope=ENVELOPE_MODELS, identity_paths=("model", "name"), required_item_any_paths=("digest", "details.family", "details.families"), - capability_paths=("details.family", "details.families"), - priority=90, + detection_priority=90, ) OLLAMA_SHOW_SHAPE = ProviderCatalogShape( shape_id="ollama.show.v1", provider_id="ollama", - endpoint_path="/api/show", envelope=ENVELOPE_SINGLE, identity_paths=(), required_item_paths=("capabilities",), required_item_any_paths=("model_info", "details", "template", "parameters"), item_types=(("capabilities", (list, tuple)),), - capability_paths=("capabilities", "model_info.*.context_length"), - priority=100, + detection_priority=100, ) LMSTUDIO_MODELS_V1_SHAPE = ProviderCatalogShape( shape_id="lmstudio.models.native.v1", provider_id="lmstudio", - endpoint_path="/api/v1/models", envelope=ENVELOPE_MODELS, identity_paths=("key",), required_item_paths=("type",), - required_item_any_paths=("capabilities", "loaded_instances", "max_context_length", "architecture", "quantization"), + required_item_any_paths=( + "capabilities", + "loaded_instances", + "max_context_length", + "architecture", + "quantization", + ), item_types=(("type", (str,)),), - capability_paths=("type", "capabilities", "max_context_length", "loaded_instances[].config.context_length"), - api_version="1", - priority=100, + detection_priority=100, ) LMSTUDIO_MODELS_V0_SHAPE = ProviderCatalogShape( shape_id="lmstudio.models.native.v0", provider_id="lmstudio", - endpoint_path="/api/v0/models", envelope=ENVELOPE_DATA, identity_paths=("id",), required_item_paths=("type",), required_item_any_paths=("arch", "compatibility_type", "state", "max_context_length"), item_types=(("type", (str,)),), - capability_paths=("type", "max_context_length"), - api_version="0", - priority=80, - latest=False, + detection_priority=80, ) LLAMACPP_PROPS_SHAPE = ProviderCatalogShape( shape_id="llamacpp.props.v1", provider_id="llamacpp", - endpoint_path="/props", envelope=ENVELOPE_SINGLE, identity_paths=("model_alias", "model_path"), required_item_paths=("default_generation_settings",), required_item_any_paths=("chat_template_caps", "modalities", "total_slots"), item_types=(("default_generation_settings", (Mapping,)),), - capability_paths=( - "chat_template_caps", - "modalities", - "default_generation_settings.n_ctx", - "default_generation_settings.params", - ), - priority=100, + detection_priority=100, ) MISTRAL_MODELS_SHAPE = ProviderCatalogShape( shape_id="mistral.models.rich.v1", provider_id="mistral", - endpoint_path="/v1/models", envelope=ENVELOPE_DATA, identity_paths=("id",), required_item_paths=("capabilities",), @@ -478,14 +268,11 @@ MISTRAL_MODELS_SHAPE = ProviderCatalogShape( "capabilities.classification", ), item_types=(("capabilities", (Mapping,)),), - capability_paths=("capabilities", "max_context_length"), - api_version="v1", - priority=100, + detection_priority=100, ) COPILOT_MODELS_SHAPE = ProviderCatalogShape( shape_id="github-copilot.models.v1", provider_id="copilot", - endpoint_path="/models", envelope=ENVELOPE_DATA, identity_paths=("id",), required_item_paths=("model_picker_enabled", "capabilities.supports"), @@ -493,238 +280,184 @@ COPILOT_MODELS_SHAPE = ProviderCatalogShape( ("model_picker_enabled", (bool,)), ("capabilities.supports", (Mapping,)), ), - capability_paths=("capabilities.supports.tool_calls", "capabilities.supports.vision"), - priority=100, + detection_priority=100, ) ANTHROPIC_MODELS_SHAPE = ProviderCatalogShape( shape_id="anthropic.models.identity.v1", provider_id="anthropic", - endpoint_path="/v1/models", envelope=ENVELOPE_DATA, identity_paths=("id",), required_item_paths=("type", "display_name", "created_at"), item_values=(("type", ("model",)),), - api_version="2023-06-01", - priority=70, + detection_priority=70, ) CHATGPT_MODELS_SHAPE = ProviderCatalogShape( shape_id="chatgpt-subscription.codex-models.v1", provider_id="chatgpt_subscription", - endpoint_path="/backend-api/codex/models?client_version=1.0.0", envelope=ENVELOPE_MODELS, identity_paths=("slug",), required_item_any_paths=("visibility", "priority"), - priority=90, + detection_priority=90, ) SGLANG_MODEL_INFO_SHAPE = ProviderCatalogShape( shape_id="sglang.model-info.v2", provider_id="sglang", - endpoint_path="/model_info", envelope=ENVELOPE_SINGLE, identity_paths=("model_path",), required_item_paths=("is_generation",), - required_item_any_paths=("tokenizer_path", "has_image_understanding", "has_audio_understanding"), + required_item_any_paths=( + "tokenizer_path", + "has_image_understanding", + "has_audio_understanding", + ), item_types=(("is_generation", (bool,)),), - capability_paths=("is_generation", "has_image_understanding", "has_audio_understanding", "preferred_sampling_params"), - api_version="2", - priority=100, + detection_priority=100, ) SGLANG_MODELS_SHAPE = ProviderCatalogShape( shape_id="sglang.models.openai.v1", provider_id="sglang", - endpoint_path="/v1/models", envelope=ENVELOPE_DATA, identity_paths=("id",), required_item_paths=("root", "max_model_len"), item_values=(("owned_by", ("sglang",)),), - capability_paths=("max_model_len",), - priority=80, + detection_priority=80, ) VLLM_MODELS_SHAPE = ProviderCatalogShape( shape_id="vllm.models.openai.v1", provider_id="vllm", - endpoint_path="/v1/models", envelope=ENVELOPE_DATA, identity_paths=("id",), required_item_paths=("root", "max_model_len", "permission"), item_values=(("owned_by", ("vllm",)),), - capability_paths=("max_model_len",), - priority=80, + detection_priority=80, ) HUGGINGFACE_MODEL_SHAPE = ProviderCatalogShape( shape_id="huggingface.hub.model-info.v1", provider_id="huggingface", - endpoint_path="/api/models/{model}", envelope=ENVELOPE_SINGLE, identity_paths=("modelId", "id"), required_item_paths=("pipeline_tag",), item_types=(("pipeline_tag", (str,)),), - capability_paths=("pipeline_tag", "tags", "config"), - priority=80, + detection_priority=80, ) COHERE_MODELS_SHAPE = ProviderCatalogShape( shape_id="cohere.models.rich.v1", provider_id="cohere", - endpoint_path="/v1/models", envelope=ENVELOPE_MODELS, identity_paths=("name",), required_item_paths=("endpoints",), - required_item_any_paths=("context_length", "default_endpoints", "features", "sampling_defaults"), + required_item_any_paths=( + "context_length", + "default_endpoints", + "features", + "sampling_defaults", + ), item_types=(("endpoints", (list, tuple)),), - capability_paths=("endpoints", "context_length", "sampling_defaults"), - api_version="v1", - priority=100, + detection_priority=100, ) MINIMAX_MODELS_SHAPE = ProviderCatalogShape( shape_id="minimax.models.identity.v1", provider_id="minimax", - endpoint_path="/v1/models", envelope=ENVELOPE_DATA, identity_paths=("id",), required_item_paths=("object", "owned_by"), item_values=(("object", ("model",)), ("owned_by", ("minimax",))), - api_version="v1", - priority=90, + detection_priority=90, ) def _provider( provider_id: str, - display_name: str, *, aliases: tuple[str, ...] = (), hosts: tuple[str, ...] = (), - api_shapes: tuple[ProviderApiShape, ...] = (OPENAI_CHAT_SHAPE,), - catalog_shapes: tuple[ProviderCatalogShape, ...] = (), + shapes: tuple[ProviderCatalogShape, ...] = (), ) -> ProviderCapabilitySchema: return ProviderCapabilitySchema( provider_id=provider_id, - display_name=display_name, aliases=aliases, host_suffixes=hosts, - api_shapes=api_shapes, - catalog_shapes=catalog_shapes, + catalog_shapes=shapes, ) PROVIDER_SCHEMAS = { PROVIDER_GENERIC_OPENAI: _provider( PROVIDER_GENERIC_OPENAI, - "General OpenAI-compatible", - aliases=("openai_compatible", "proxy"), - api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE), - catalog_shapes=(GENERAL_DATA_SHAPE, GENERAL_MODELS_SHAPE, GENERAL_BARE_SHAPE), - ), - "openai": _provider( - "openai", - "OpenAI", - hosts=("openai.com",), - api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE), - catalog_shapes=(OPENAI_MODELS_SHAPE,), + aliases=("openai_compatible", "openai_compat"), ), + "openai": _provider("openai", hosts=("openai.com",), shapes=(OPENAI_MODELS_SHAPE,)), "openrouter": _provider( "openrouter", - "OpenRouter", hosts=("openrouter.ai",), - catalog_shapes=(OPENROUTER_MODELS_SHAPE,), + shapes=(OPENROUTER_MODELS_SHAPE,), ), "google": _provider( "google", - "Google Gemini", aliases=("gemini", "google_ai_studio"), hosts=("generativelanguage.googleapis.com",), - api_shapes=(GOOGLE_CONTENT_SHAPE, OPENAI_CHAT_SHAPE), - catalog_shapes=(GOOGLE_MODELS_SHAPE,), + shapes=(GOOGLE_MODELS_SHAPE,), ), "anthropic": _provider( "anthropic", - "Anthropic", hosts=("anthropic.com",), - api_shapes=(ANTHROPIC_MESSAGES_SHAPE,), - catalog_shapes=(ANTHROPIC_MODELS_SHAPE,), + shapes=(ANTHROPIC_MODELS_SHAPE,), ), "ollama": _provider( "ollama", - "Ollama", hosts=("ollama.com",), - api_shapes=(OLLAMA_NATIVE_SHAPE, OPENAI_CHAT_SHAPE), - catalog_shapes=(OLLAMA_SHOW_SHAPE, OLLAMA_TAGS_SHAPE), + shapes=(OLLAMA_SHOW_SHAPE, OLLAMA_TAGS_SHAPE), ), "lmstudio": _provider( "lmstudio", - "LM Studio", aliases=("lm_studio",), - api_shapes=(LMSTUDIO_NATIVE_SHAPE, OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE), - catalog_shapes=(LMSTUDIO_MODELS_V1_SHAPE, LMSTUDIO_MODELS_V0_SHAPE), + shapes=(LMSTUDIO_MODELS_V1_SHAPE, LMSTUDIO_MODELS_V0_SHAPE), ), "llamacpp": _provider( "llamacpp", - "llama.cpp", aliases=("llama.cpp", "llama_cpp", "llama_server"), - api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE, ANTHROPIC_MESSAGES_SHAPE), - catalog_shapes=(LLAMACPP_PROPS_SHAPE,), + shapes=(LLAMACPP_PROPS_SHAPE,), ), "mistral": _provider( "mistral", - "Mistral", hosts=("mistral.ai",), - catalog_shapes=(MISTRAL_MODELS_SHAPE,), + shapes=(MISTRAL_MODELS_SHAPE,), ), "copilot": _provider( "copilot", - "GitHub Copilot", aliases=("github_copilot",), hosts=("api.githubcopilot.com",), - catalog_shapes=(COPILOT_MODELS_SHAPE,), + shapes=(COPILOT_MODELS_SHAPE,), ), "chatgpt_subscription": _provider( "chatgpt_subscription", - "ChatGPT Subscription", aliases=("chatgpt-subscription", "chatgpt", "codex_subscription"), hosts=("chatgpt.com",), - api_shapes=(CHATGPT_SUBSCRIPTION_SHAPE,), - catalog_shapes=(CHATGPT_MODELS_SHAPE,), + shapes=(CHATGPT_MODELS_SHAPE,), ), "sglang": _provider( "sglang", - "SGLang", - api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE, SGLANG_NATIVE_SHAPE), - catalog_shapes=(SGLANG_MODEL_INFO_SHAPE, SGLANG_MODELS_SHAPE), - ), - "vllm": _provider( - "vllm", - "vLLM", - api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE), - catalog_shapes=(VLLM_MODELS_SHAPE,), + shapes=(SGLANG_MODEL_INFO_SHAPE, SGLANG_MODELS_SHAPE), ), + "vllm": _provider("vllm", shapes=(VLLM_MODELS_SHAPE,)), "huggingface": _provider( "huggingface", - "Hugging Face", aliases=("hf", "hugging_face"), hosts=("huggingface.co",), - api_shapes=(OPENAI_CHAT_SHAPE,), - catalog_shapes=(HUGGINGFACE_MODEL_SHAPE,), + shapes=(HUGGINGFACE_MODEL_SHAPE,), ), "cohere": _provider( "cohere", - "Cohere", hosts=("cohere.ai", "cohere.com"), - api_shapes=(COHERE_V2_SHAPE, OPENAI_CHAT_SHAPE), - catalog_shapes=(COHERE_MODELS_SHAPE,), + shapes=(COHERE_MODELS_SHAPE,), ), "minimax": _provider( "minimax", - "MiniMax", hosts=("minimax.io", "minimaxi.com"), - api_shapes=(ANTHROPIC_MESSAGES_SHAPE, OPENAI_CHAT_SHAPE), - catalog_shapes=(MINIMAX_MODELS_SHAPE,), + shapes=(MINIMAX_MODELS_SHAPE,), ), } -# These providers are currently OpenAI-compatible in Odysseus and have no -# provider-reported capability catalog shape that is stronger than the general -# structural fallback. Keeping individual identities prevents transport -# quirks from being flattened into "OpenAI" while capability stays per-model. _GENERAL_PROVIDER_ALIASES = { "moonshot": ("moonshot_ai",), "nvidia": ("nvidia_nim", "nim"), @@ -740,44 +473,38 @@ _GENERAL_PROVIDER_ALIASES = { "mlx_lm": ("mlx",), "text_generation_inference": ("tgi", "huggingface_tgi", "hugging_face_tgi"), } -for _provider_id, _display, _hosts in ( - ("moonshot", "Moonshot AI", ("moonshot.ai", "moonshot.cn")), - ("groq", "Groq", ("groq.com",)), - ("nvidia", "NVIDIA NIM", ("nvidia.com",)), - ("cerebras", "Cerebras", ("cerebras.ai",)), - ("deepseek", "DeepSeek", ("deepseek.com",)), - ("together", "Together AI", ("together.xyz", "together.ai")), - ("fireworks", "Fireworks AI", ("fireworks.ai",)), - ("xai", "xAI", ("x.ai",)), - ("zai", "Z.AI", ("z.ai",)), - ("opencode", "OpenCode", ("opencode.ai",)), - ("perplexity", "Perplexity", ("perplexity.ai",)), - ("github_models", "GitHub Models", ("models.inference.ai.azure.com",)), - ("atlas_cloud", "Atlas Cloud", ("atlascloud.ai",)), - ("siliconflow", "SiliconFlow", ("siliconflow.cn", "siliconflow.com",)), - ("kimi_code", "Kimi Code", ("kimi.com",)), - ("venice", "Venice", ("venice.ai",)), - ("azure_openai", "Azure OpenAI", ("openai.azure.com",)), - ("bedrock", "AWS Bedrock", ()), - ("cloudflare_workers_ai", "Cloudflare Workers AI", ()), - ("mlx_lm", "MLX LM", ()), - ("text_generation_inference", "Hugging Face TGI", ()), - ("lmdeploy", "LMDeploy", ()), - ("litellm", "LiteLLM", ()), +for _provider_id, _hosts in ( + ("moonshot", ("moonshot.ai", "moonshot.cn")), + ("groq", ("groq.com",)), + ("nvidia", ("nvidia.com",)), + ("cerebras", ("cerebras.ai",)), + ("deepseek", ("deepseek.com",)), + ("together", ("together.xyz", "together.ai")), + ("fireworks", ("fireworks.ai",)), + ("xai", ("x.ai",)), + ("zai", ("z.ai",)), + ("opencode", ("opencode.ai",)), + ("perplexity", ("perplexity.ai",)), + ("github_models", ("models.inference.ai.azure.com",)), + ("atlas_cloud", ("atlascloud.ai",)), + ("siliconflow", ("siliconflow.cn", "siliconflow.com")), + ("kimi_code", ("kimi.com",)), + ("venice", ("venice.ai",)), + ("azure_openai", ("openai.azure.com",)), + ("bedrock", ()), + ("cloudflare_workers_ai", ()), + ("mlx_lm", ()), + ("text_generation_inference", ()), + ("lmdeploy", ()), + ("litellm", ()), ): PROVIDER_SCHEMAS[_provider_id] = _provider( _provider_id, - _display, aliases=_GENERAL_PROVIDER_ALIASES.get(_provider_id, ()), hosts=_hosts, ) -UNKNOWN_SCHEMA = ProviderCapabilitySchema( - provider_id=PROVIDER_UNKNOWN, - display_name="Unknown provider", - api_shapes=(), - catalog_shapes=(), -) +UNKNOWN_SCHEMA = ProviderCapabilitySchema(provider_id=PROVIDER_UNKNOWN) _ALIASES = { _token(alias): provider_id @@ -788,7 +515,12 @@ _ALIASES = { def normalize_provider_id(value: Any) -> str: token = _token(value) - return _ALIASES.get(token, token if token in PROVIDER_SCHEMAS else PROVIDER_UNKNOWN) + if not token or token == PROVIDER_UNKNOWN: + return PROVIDER_UNKNOWN + # An explicit, previously unseen provider id is still useful identity. It + # selects the inventory-only reader until a native schema is added; it does + # not acquire capabilities merely by being preserved here. + return _ALIASES.get(token, token) def schema_for_provider(value: Any) -> ProviderCapabilitySchema: @@ -808,46 +540,44 @@ def provider_from_host(base_url: Any) -> str: return PROVIDER_UNKNOWN if host.startswith("copilot-api.") and host.endswith(".ghe.com"): return "copilot" - matches = [ + matches = { schema.provider_id for schema in PROVIDER_SCHEMAS.values() if any(_host_matches(host, suffix) for suffix in schema.host_suffixes) - ] - return matches[0] if len(set(matches)) == 1 else PROVIDER_UNKNOWN + } + return next(iter(matches)) if len(matches) == 1 else PROVIDER_UNKNOWN -def catalog_shape_for_payload( +def native_shape_for_payload( payload: Any, *, provider_id: Any = None, - include_general: bool = True, ) -> ProviderCatalogShape | None: normalized = normalize_provider_id(provider_id) - provider_is_explicit = normalized not in {PROVIDER_UNKNOWN, PROVIDER_GENERIC_OPENAI} - if provider_is_explicit: - schemas = (PROVIDER_SCHEMAS[normalized],) + if normalized == PROVIDER_UNKNOWN: + shapes = tuple( + shape + for schema in PROVIDER_SCHEMAS.values() + for shape in schema.catalog_shapes + if shape.detection_priority > 0 + ) else: - schemas = tuple(PROVIDER_SCHEMAS.values()) + schema = PROVIDER_SCHEMAS.get(normalized) + shapes = schema.catalog_shapes if schema else () - candidates = [ - shape - for schema in schemas - for shape in schema.catalog_shapes - if (provider_is_explicit or shape.priority > 0) and shape.matches(payload) - ] - if candidates: - best_priority = max(shape.priority for shape in candidates) - best = [shape for shape in candidates if shape.priority == best_priority] - providers = {shape.provider_id for shape in best} - if len(providers) == 1: - return sorted(best, key=lambda shape: shape.shape_id)[0] - - if not include_general: + matches = [shape for shape in shapes if shape.matches(payload)] + if not matches: return None - for shape in PROVIDER_SCHEMAS[PROVIDER_GENERIC_OPENAI].catalog_shapes: - if shape.matches(payload): - return shape - return None + priority = max(shape.detection_priority for shape in matches) + best = [shape for shape in matches if shape.detection_priority == priority] + providers = {shape.provider_id for shape in best} + if len(providers) != 1: + return None + return sorted(best, key=lambda shape: shape.shape_id)[0] + + +def fallback_shape_for_payload(payload: Any) -> ProviderCatalogShape | None: + return next((shape for shape in FALLBACK_CATALOG_SHAPES if shape.matches(payload)), None) def resolve_provider( @@ -857,75 +587,68 @@ def resolve_provider( endpoint_kind: Any = None, base_url: Any = None, ) -> ProviderResolution: - explicit = normalize_provider_id(provider) - if explicit != PROVIDER_UNKNOWN: - shape = catalog_shape_for_payload(payload, provider_id=explicit) if payload is not None else None - return ProviderResolution(explicit, RESOLUTION_EXPLICIT, PROVIDER_SCHEMAS[explicit], shape) + provider_id = normalize_provider_id(provider) + provider_source = PROVIDER_SOURCE_EXPLICIT - kind = normalize_provider_id(endpoint_kind) - if kind != PROVIDER_UNKNOWN: - shape = catalog_shape_for_payload(payload, provider_id=kind) if payload is not None else None - return ProviderResolution(kind, RESOLUTION_ENDPOINT_KIND, PROVIDER_SCHEMAS[kind], shape) + if provider_id == PROVIDER_UNKNOWN: + provider_id = normalize_provider_id(endpoint_kind) + provider_source = PROVIDER_SOURCE_ENDPOINT_KIND + if provider_id == PROVIDER_UNKNOWN: + provider_id = provider_from_host(base_url) + provider_source = PROVIDER_SOURCE_HOST - host_provider = provider_from_host(base_url) - if host_provider != PROVIDER_UNKNOWN: - shape = catalog_shape_for_payload(payload, provider_id=host_provider) if payload is not None else None + if provider_id != PROVIDER_UNKNOWN: + native = ( + native_shape_for_payload(payload, provider_id=provider_id) + if payload is not None + else None + ) + if native: + return ProviderResolution(provider_id, provider_source, native.shape_id, False) + fallback = fallback_shape_for_payload(payload) if payload is not None else None return ProviderResolution( - host_provider, - RESOLUTION_HOST, - PROVIDER_SCHEMAS[host_provider], - shape, + provider_id, + provider_source, + fallback.shape_id if fallback else "", + bool(fallback), ) - shape = catalog_shape_for_payload(payload, include_general=False) if payload is not None else None - if shape: + native = native_shape_for_payload(payload) if payload is not None else None + if native: return ProviderResolution( - shape.provider_id, - RESOLUTION_NATIVE_SHAPE, - PROVIDER_SCHEMAS[shape.provider_id], - shape, + native.provider_id, + PROVIDER_SOURCE_PAYLOAD, + native.shape_id, + False, ) - shape = catalog_shape_for_payload(payload) if payload is not None else None - if shape: - return ProviderResolution( - PROVIDER_GENERIC_OPENAI, - RESOLUTION_GENERAL_SHAPE, - PROVIDER_SCHEMAS[PROVIDER_GENERIC_OPENAI], - shape, - ) - - return ProviderResolution(PROVIDER_UNKNOWN, RESOLUTION_UNKNOWN, UNKNOWN_SCHEMA, None) + fallback = fallback_shape_for_payload(payload) if payload is not None else None + return ProviderResolution( + PROVIDER_UNKNOWN, + PROVIDER_SOURCE_UNKNOWN, + fallback.shape_id if fallback else "", + bool(fallback), + ) __all__ = [ - "ANTHROPIC_MESSAGES_SHAPE", - "CHATGPT_SUBSCRIPTION_SHAPE", - "COHERE_V2_SHAPE", - "DIALECT_ANTHROPIC_MESSAGES", - "DIALECT_CHATGPT_SUBSCRIPTION", - "DIALECT_COHERE_V2", - "DIALECT_GOOGLE_GENERATE_CONTENT", - "DIALECT_HUGGINGFACE_HUB", - "DIALECT_LLAMACPP_NATIVE", - "DIALECT_LMSTUDIO_NATIVE_V1", - "DIALECT_OLLAMA_NATIVE", - "DIALECT_OPENAI_CHAT", - "DIALECT_OPENAI_RESPONSES", - "DIALECT_SGLANG_NATIVE", - "GOOGLE_CONTENT_SHAPE", - "LMSTUDIO_NATIVE_SHAPE", - "OLLAMA_NATIVE_SHAPE", - "OPENAI_CHAT_SHAPE", - "OPENAI_RESPONSES_SHAPE", + "FALLBACK_CATALOG_SHAPES", + "GENERAL_BARE_SHAPE", + "GENERAL_DATA_SHAPE", + "GENERAL_MODELS_SHAPE", "PROVIDER_GENERIC_OPENAI", "PROVIDER_SCHEMAS", + "PROVIDER_SOURCE_ENDPOINT_KIND", + "PROVIDER_SOURCE_EXPLICIT", + "PROVIDER_SOURCE_HOST", + "PROVIDER_SOURCE_PAYLOAD", + "PROVIDER_SOURCE_UNKNOWN", "PROVIDER_UNKNOWN", - "ProviderApiShape", "ProviderCapabilitySchema", "ProviderCatalogShape", "ProviderResolution", - "catalog_shape_for_payload", + "fallback_shape_for_payload", + "native_shape_for_payload", "normalize_provider_id", "provider_from_host", "resolve_provider", diff --git a/tests/test_model_capability_readers.py b/tests/test_model_capability_readers.py index 17734d448..ca547b15a 100644 --- a/tests/test_model_capability_readers.py +++ b/tests/test_model_capability_readers.py @@ -9,6 +9,7 @@ from src.model_capability_readers.base import ( VENDOR_OLLAMA, VENDOR_OPENAI, VENDOR_OPENROUTER, + VENDOR_UNKNOWN, detect_vendor, stable_model_id_for, ) @@ -24,10 +25,10 @@ def test_detect_vendor_uses_endpoint_kind_and_host_but_not_ambiguous_local_ports assert detect_vendor("https://openrouter.ai/api/v1") == VENDOR_OPENROUTER assert detect_vendor("https://api.openai.com/v1") == VENDOR_OPENAI assert detect_vendor("https://generativelanguage.googleapis.com/v1beta/openai") == VENDOR_GOOGLE - assert detect_vendor("http://127.0.0.1:11434") == VENDOR_GENERIC_OPENAI - assert detect_vendor("http://127.0.0.1:1234") == VENDOR_GENERIC_OPENAI - assert detect_vendor("http://127.0.0.1:8080") == VENDOR_GENERIC_OPENAI - assert detect_vendor("http://localhost:7000/v1") == VENDOR_GENERIC_OPENAI + assert detect_vendor("http://127.0.0.1:11434") == VENDOR_UNKNOWN + assert detect_vendor("http://127.0.0.1:1234") == VENDOR_UNKNOWN + assert detect_vendor("http://127.0.0.1:8080") == VENDOR_UNKNOWN + assert detect_vendor("http://localhost:7000/v1") == VENDOR_UNKNOWN def test_generic_openai_reader_keeps_basic_model_payload_unknown(): @@ -362,7 +363,6 @@ def test_ollama_reader_maps_show_capabilities_and_tags_are_unknown(): assert len(tags) == 1 assert tags[0].capability.family == mc.FAMILY_UNKNOWN - assert tags[0].model_family == "qwen3" assert surfaces(tags[0]) == set() diff --git a/tests/test_provider_capability_schemas.py b/tests/test_provider_capability_schemas.py index 855f85f5c..9d5349c85 100644 --- a/tests/test_provider_capability_schemas.py +++ b/tests/test_provider_capability_schemas.py @@ -1,7 +1,7 @@ -from src import model_behavior_quirks as quirks from src import model_capabilities as mc from src import provider_capability_schemas as pcs from src.model_capability_readers import ( + CANONICAL_MODEL_SHAPE_VERSION, anthropic, chatgpt_subscription, cohere, @@ -14,7 +14,7 @@ from src.model_capability_readers import ( ) -def test_provider_resolution_order_explicit_then_host_then_native_then_general(): +def test_provider_identity_and_catalog_shape_are_resolved_separately(): google_payload = { "models": [ { @@ -27,21 +27,39 @@ def test_provider_resolution_order_explicit_then_host_then_native_then_general() explicit = pcs.resolve_provider(google_payload, provider="openrouter") host = pcs.resolve_provider(google_payload, base_url="https://api.mistral.ai/v1") native = pcs.resolve_provider(google_payload) - general = pcs.resolve_provider([{"id": "future-model", "future": {"x": True}}]) + fallback = pcs.resolve_provider([{"id": "future-model", "future": {"x": True}}]) unknown = pcs.resolve_provider({"future": [{"not_an_identity": True}]}) - assert (explicit.provider_id, explicit.stage) == ("openrouter", pcs.RESOLUTION_EXPLICIT) - assert (host.provider_id, host.stage) == ("mistral", pcs.RESOLUTION_HOST) - assert (native.provider_id, native.stage) == ("google", pcs.RESOLUTION_NATIVE_SHAPE) - assert native.catalog_shape.shape_id == "google.generative-language.models.v1beta" - assert (general.provider_id, general.stage) == ( - pcs.PROVIDER_GENERIC_OPENAI, - pcs.RESOLUTION_GENERAL_SHAPE, - ) - assert (unknown.provider_id, unknown.stage) == ( - pcs.PROVIDER_UNKNOWN, - pcs.RESOLUTION_UNKNOWN, - ) + assert explicit.to_dict() == { + "provider": "openrouter", + "provider_source": pcs.PROVIDER_SOURCE_EXPLICIT, + "shape": "fallback.models.envelope.v1", + "fallback": True, + } + assert host.to_dict() == { + "provider": "mistral", + "provider_source": pcs.PROVIDER_SOURCE_HOST, + "shape": "fallback.models.envelope.v1", + "fallback": True, + } + assert native.to_dict() == { + "provider": "google", + "provider_source": pcs.PROVIDER_SOURCE_PAYLOAD, + "shape": "google.generative-language.models.v1beta", + "fallback": False, + } + assert fallback.to_dict() == { + "provider": pcs.PROVIDER_UNKNOWN, + "provider_source": pcs.PROVIDER_SOURCE_UNKNOWN, + "shape": "fallback.models.list.v1", + "fallback": True, + } + assert unknown.to_dict() == { + "provider": pcs.PROVIDER_UNKNOWN, + "provider_source": pcs.PROVIDER_SOURCE_UNKNOWN, + "shape": "", + "fallback": False, + } def test_provider_host_matching_rejects_lookalikes_and_does_not_use_ports(): @@ -53,31 +71,57 @@ def test_provider_host_matching_rejects_lookalikes_and_does_not_use_ports(): assert pcs.provider_from_host("http://127.0.0.1:30000") == pcs.PROVIDER_UNKNOWN -def test_provider_aliases_collapse_runtime_names_without_url_path_guessing(): +def test_provider_aliases_only_normalize_explicit_identity(): assert pcs.normalize_provider_id("opencode-go") == "opencode" assert pcs.normalize_provider_id("opencode-zen") == "opencode" assert pcs.normalize_provider_id("nvidia-nim") == "nvidia" assert pcs.normalize_provider_id("tgi") == "text_generation_inference" assert pcs.normalize_provider_id("llama.cpp") == "llamacpp" assert pcs.normalize_provider_id("Z.AI") == "zai" + assert pcs.normalize_provider_id("future-provider") == "future_provider" -def test_current_native_catalog_shapes_are_discriminating_and_versioned(): +def test_unregistered_explicit_provider_is_preserved_but_stays_on_fallback(): + resolution = pcs.resolve_provider( + {"data": [{"id": "future-model", "capabilities": {"tools": True}}]}, + provider="future-provider", + ) + records = records_from_payload( + {"data": [{"id": "future-model", "capabilities": {"tools": True}}]}, + vendor="future-provider", + ) + + assert resolution.to_dict() == { + "provider": "future_provider", + "provider_source": pcs.PROVIDER_SOURCE_EXPLICIT, + "shape": "fallback.models.data.v1", + "fallback": True, + } + assert records[0].vendor == "future_provider" + assert records[0].capability.family == mc.FAMILY_UNKNOWN + assert records[0].capability.capabilities == () + + +def test_current_native_catalog_shapes_are_discriminating(): cases = ( ( {"models": [{"key": "local/model", "type": "llm", "capabilities": {"vision": True}}]}, + "lmstudio", "lmstudio.models.native.v1", ), ( {"data": [{"id": "legacy", "type": "vlm", "arch": "gemma"}]}, + "lmstudio", "lmstudio.models.native.v0", ), ( {"models": [{"name": "local", "digest": "abc", "details": {"family": "qwen3"}}]}, + "ollama", "ollama.tags.v1", ), ( {"capabilities": ["completion", "vision"], "model_info": {"x.context_length": 4096}}, + "ollama", "ollama.show.v1", ), ( @@ -86,10 +130,12 @@ def test_current_native_catalog_shapes_are_discriminating_and_versioned(): "default_generation_settings": {"n_ctx": 4096}, "chat_template_caps": {"supports_tools": True}, }, + "llamacpp", "llamacpp.props.v1", ), ( {"data": [{"id": "mistral", "capabilities": {"completion_chat": True, "vision": False}}]}, + "mistral", "mistral.models.rich.v1", ), ( @@ -102,6 +148,7 @@ def test_current_native_catalog_shapes_are_discriminating_and_versioned(): } ] }, + "copilot", "github-copilot.models.v1", ), ( @@ -111,6 +158,7 @@ def test_current_native_catalog_shapes_are_discriminating_and_versioned(): "is_generation": True, "has_image_understanding": False, }, + "sglang", "sglang.model-info.v2", ), ( @@ -127,46 +175,38 @@ def test_current_native_catalog_shapes_are_discriminating_and_versioned(): } ], }, + "vllm", "vllm.models.openai.v1", ), ( {"models": [{"slug": "gpt-example", "visibility": "list", "priority": 1}]}, + "chatgpt_subscription", "chatgpt-subscription.codex-models.v1", ), ( - { - "models": [ - { - "name": "command-example", - "endpoints": ["chat"], - "context_length": 131072, - } - ] - }, + {"models": [{"name": "command-example", "endpoints": ["chat"], "context_length": 131072}]}, + "cohere", "cohere.models.rich.v1", ), ( { "object": "list", - "data": [ - { - "id": "MiniMax-M2-example", - "object": "model", - "owned_by": "minimax", - } - ], + "data": [{"id": "MiniMax-M2", "object": "model", "owned_by": "minimax"}], }, + "minimax", "minimax.models.identity.v1", ), ) - for payload, expected_shape in cases: + for payload, expected_provider, expected_shape in cases: resolution = pcs.resolve_provider(payload) - assert resolution.stage == pcs.RESOLUTION_NATIVE_SHAPE - assert resolution.catalog_shape.shape_id == expected_shape + assert resolution.provider_id == expected_provider + assert resolution.provider_source == pcs.PROVIDER_SOURCE_PAYLOAD + assert resolution.shape_id == expected_shape + assert resolution.fallback is False -def test_native_shape_detection_rejects_wrong_field_types_before_general_fallback(): +def test_wrong_native_field_types_degrade_to_explicit_fallback_inventory(): malformed_cohere = pcs.resolve_provider( {"models": [{"name": "future", "endpoints": "chat", "context_length": 4096}]} ) @@ -174,57 +214,69 @@ def test_native_shape_detection_rejects_wrong_field_types_before_general_fallbac {"data": [{"id": "future", "capabilities": ["completion_chat"]}]} ) - assert (malformed_cohere.provider_id, malformed_cohere.stage) == ( - pcs.PROVIDER_GENERIC_OPENAI, - pcs.RESOLUTION_GENERAL_SHAPE, - ) - assert (malformed_mistral.provider_id, malformed_mistral.stage) == ( - pcs.PROVIDER_GENERIC_OPENAI, - pcs.RESOLUTION_GENERAL_SHAPE, - ) + assert malformed_cohere.provider_id == pcs.PROVIDER_UNKNOWN + assert malformed_cohere.shape_id == "fallback.models.envelope.v1" + assert malformed_cohere.fallback is True + assert malformed_mistral.provider_id == pcs.PROVIDER_UNKNOWN + assert malformed_mistral.shape_id == "fallback.models.data.v1" + assert malformed_mistral.fallback is True -def test_general_reader_promotes_only_explicit_structural_fields_and_accepts_bare_lists(): - records = generic_openai.records_from_payload( - [ - { - "id": "future-rich-model", - "type": "chat", - "architecture": { - "input_modalities": ["text", "image"], - "output_modalities": ["text"], - }, - "supported_parameters": ["tools", "structured_outputs", "temperature"], - "max_model_len": 131072, - "future_capability": {"may_be_important_later": True}, +def test_fallback_reader_is_identity_only_even_for_dangerous_looking_fields(): + payload = [ + { + "id": "future-rich-model", + "type": "chat", + "architecture": { + "input_modalities": ["text", "image"], + "output_modalities": ["text"], }, - { - "id": "vision-reasoning-tools-in-the-name-only", - "description": "Claims every capability in prose", - "type": "image", - "future_capability": True, - }, - ] - ) + "capabilities": {"supports": {"tools": True, "reasoning": True}}, + "supported_parameters": ["tools", "structured_outputs", "temperature"], + "max_model_len": 131072, + }, + {"key": "key-only-model", "pipeline_tag": "text-to-image"}, + {"slug": "slug-only-model", "modality": "text_to_image"}, + ] - rich, identity_only = records - assert rich.capability.family == mc.FAMILY_CHAT - assert rich.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) - assert rich.capability.capabilities == ( - mc.CAP_TOOL_CALL, - mc.CAP_STRUCTURED_OUTPUT, - mc.CAP_VISION, - ) - assert dict(rich.capability.limits) == {"context_tokens": 131072} - assert [control.control for control in rich.deterministic_controls] == [mc.CONTROL_TEMPERATURE] - assert rich.raw["future_capability"] == {"may_be_important_later": True} + direct = generic_openai.records_from_payload(payload) + wrapped = records_from_payload(payload, vendor="together") - assert identity_only.capability.family == mc.FAMILY_UNKNOWN - assert identity_only.capability.capabilities == () - assert identity_only.raw["future_capability"] is True + assert [record.model_id for record in direct] == [ + "future-rich-model", + "key-only-model", + "slug-only-model", + ] + for record in (*direct, *wrapped): + assert record.capability.family == mc.FAMILY_UNKNOWN + assert record.capability.capabilities == () + assert dict(record.capability.limits) == {} + assert record.deterministic_controls == () + + lean = wrapped[0].to_dict() + assert lean == { + "schema_version": CANONICAL_MODEL_SHAPE_VERSION, + "provider": "together", + "model": "future-rich-model", + "stable_id": "together|global|future-rich-model", + "family": "unknown", + "task": "unknown", + "modalities": {"input": [], "output": []}, + "features": [], + "limits": {}, + "controls": [], + "evidence": { + "source": "provider_reader", + "confidence": "unknown", + "provider_source": "explicit", + "shape": "fallback.models.list.v1", + "fallback": True, + }, + } + assert wrapped[0].to_dict(include_raw=True)["raw"] == payload[0] -def test_general_reader_fails_soft_for_null_and_malformed_envelopes(): +def test_fallback_reader_fails_soft_for_null_and_malformed_envelopes(): for payload in ( {"data": None}, {"models": None}, @@ -235,13 +287,12 @@ def test_general_reader_fails_soft_for_null_and_malformed_envelopes(): assert generic_openai.records_from_payload(payload) == () -def test_mistral_reader_maps_per_model_capabilities_without_provider_wide_inheritance(): +def test_mistral_reader_maps_per_model_capabilities_without_provider_inheritance(): records = mistral.records_from_payload( { "data": [ { "id": "vision-chat", - "root": "mistral-small", "capabilities": { "completion_chat": True, "function_calling": True, @@ -258,10 +309,7 @@ def test_mistral_reader_maps_per_model_capabilities_without_provider_wide_inheri "vision": False, }, }, - { - "id": "future-card", - "capabilities": {"future_only": True}, - }, + {"id": "future-card", "capabilities": {"future_only": True}}, ] } ) @@ -270,14 +318,12 @@ def test_mistral_reader_maps_per_model_capabilities_without_provider_wide_inheri assert records[0].capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) assert records[0].capability.capabilities == (mc.CAP_VISION, mc.CAP_TOOL_CALL) assert dict(records[0].capability.limits) == {"context_tokens": 32768} - assert records[0].model_family == "mistral-small" assert records[1].capability.family == mc.FAMILY_CLASSIFICATION assert records[2].capability.family == mc.FAMILY_UNKNOWN - assert records[2].capability.capabilities == () def test_copilot_reader_uses_picker_and_nested_supports_shape(): - records = copilot.records_from_payload( + record = copilot.records_from_payload( { "data": [ { @@ -285,23 +331,17 @@ def test_copilot_reader_uses_picker_and_nested_supports_shape(): "model_picker_enabled": True, "capabilities": {"supports": {"tool_calls": True, "vision": True}}, "limits": {"max_prompt_tokens": 64000, "max_output_tokens": 8192}, - }, - { - "id": "utility-model", - "model_picker_enabled": False, - "capabilities": {"supports": {}}, - }, + } ] } - ) + )[0] - assert records[0].capability.family == mc.FAMILY_CHAT - assert records[0].capability.capabilities == (mc.CAP_TOOL_CALL, mc.CAP_VISION) - assert dict(records[0].capability.limits) == {"input_tokens": 64000, "output_tokens": 8192} - assert records[1].capability.family == mc.FAMILY_UNKNOWN + assert record.capability.family == mc.FAMILY_CHAT + assert record.capability.capabilities == (mc.CAP_TOOL_CALL, mc.CAP_VISION) + assert dict(record.capability.limits) == {"input_tokens": 64000, "output_tokens": 8192} -def test_sglang_model_info_is_structural_and_non_generation_stays_unknown(): +def test_sglang_model_info_maps_native_generation_flags_only(): generation = sglang.records_from_payload( { "model_path": "org/vision-model", @@ -309,7 +349,6 @@ def test_sglang_model_info_is_structural_and_non_generation_stays_unknown(): "is_generation": True, "has_image_understanding": True, "has_audio_understanding": True, - "model_type": "future_arch", "preferred_sampling_params": {"temperature": 0.2, "top_p": 0.9}, } )[0] @@ -333,11 +372,10 @@ def test_sglang_model_info_is_structural_and_non_generation_stays_unknown(): mc.CONTROL_TEMPERATURE, mc.CONTROL_TOP_P, ] - assert generation.model_family == "future_arch" assert pooling.capability.family == mc.FAMILY_UNKNOWN -def test_identity_only_catalogs_do_not_claim_model_capability(): +def test_identity_only_native_catalogs_remain_unknown(): anthropic_record = anthropic.records_from_payload( { "data": [ @@ -356,13 +394,7 @@ def test_identity_only_catalogs_do_not_claim_model_capability(): minimax_record = records_from_payload( { "object": "list", - "data": [ - { - "id": "MiniMax-M2-example", - "object": "model", - "owned_by": "minimax", - } - ], + "data": [{"id": "MiniMax-M2", "object": "model", "owned_by": "minimax"}], } )[0] @@ -372,7 +404,7 @@ def test_identity_only_catalogs_do_not_claim_model_capability(): assert minimax_record.capability.family == mc.FAMILY_UNKNOWN -def test_huggingface_reader_maps_pipeline_tag_as_registry_evidence(): +def test_huggingface_reader_maps_provider_specific_pipeline_metadata(): record = huggingface.records_from_payload( { "modelId": "org/vision-model", @@ -387,7 +419,6 @@ def test_huggingface_reader_maps_pipeline_tag_as_registry_evidence(): assert record.capability.capabilities == (mc.CAP_VISION,) assert record.capability.source == mc.SOURCE_COOKBOOK_HF assert record.capability.confidence == mc.CONFIDENCE_REGISTRY - assert record.model_family == "future_vlm" def test_cohere_reader_maps_only_native_endpoint_and_limit_fields(): @@ -411,7 +442,6 @@ def test_cohere_reader_maps_only_native_endpoint_and_limit_fields(): ) assert chat.capability.family == mc.FAMILY_CHAT - assert chat.capability.modalities.input == (mc.MODALITY_TEXT,) assert dict(chat.capability.limits) == {"context_tokens": 131072} assert [control.control for control in chat.deterministic_controls] == [ mc.CONTROL_TEMPERATURE, @@ -422,8 +452,8 @@ def test_cohere_reader_maps_only_native_endpoint_and_limit_fields(): assert ambiguous.capability.family == mc.FAMILY_UNKNOWN -def test_registry_wrapper_records_resolution_and_preserves_compatible_provider_identity(): - mistral_records = records_from_payload( +def test_reader_wrapper_adds_one_lean_evidence_object(): + record = records_from_payload( { "data": [ { @@ -432,74 +462,20 @@ def test_registry_wrapper_records_resolution_and_preserves_compatible_provider_i } ] } - ) - together_records = records_from_payload( - [{"id": "served/model", "type": "chat", "supported_parameters": ["tools"]}], - vendor="together", - ) + )[0] + serialized = record.to_dict() - assert mistral_records[0].vendor == "mistral" - assert mistral_records[0].provider_schema_id == "mistral" - assert mistral_records[0].catalog_shape_id == "mistral.models.rich.v1" - assert mistral_records[0].provider_resolution == pcs.RESOLUTION_NATIVE_SHAPE - - assert together_records[0].vendor == "together" - assert together_records[0].capability.family == mc.FAMILY_CHAT - assert together_records[0].provider_schema_id == "together" - assert together_records[0].provider_resolution == pcs.RESOLUTION_EXPLICIT - - -def test_reasoning_control_preserves_canonical_and_native_values(): - control = mc.ReasoningControl.build( - mechanism="reasoning_effort", - values=("enabled", "disabled"), - native_values=("high", "medium", "low", "none"), - request_path="reasoning_effort", - response_paths=("choices[].delta.reasoning",), - status="claimed", - source="provider_docs_registry", - confidence="registry", - ) - - assert control.values == (mc.REASONING_CONTROL_VALUE_ON, mc.REASONING_CONTROL_VALUE_OFF) - assert control.native_values == ("high", "medium", "low", "none") - assert mc.ReasoningControl.from_dict(control.to_dict()) == control - - -def test_model_quirks_require_structured_exact_identity_not_name_parsing(): - matching = quirks.matching_quirks( - provider="moonshot", - model_id="kimi-k2.5", - model_family="", - api_dialect=pcs.DIALECT_OPENAI_CHAT, - capabilities=(mc.CAP_REASONING,), - ) - lookalike = quirks.matching_quirks( - provider="moonshot", - model_id="proxy/kimi-k2.5-lookalike", - model_family="", - api_dialect=pcs.DIALECT_OPENAI_CHAT, - capabilities=(mc.CAP_REASONING,), - ) - opus_without_version = quirks.matching_quirks( - provider="anthropic", - model_family="claude-opus", - model_id="claude-opus-4-8-in-name-only", - api_dialect=pcs.DIALECT_ANTHROPIC_MESSAGES, - ) - opus_structured = quirks.matching_quirks( - provider="anthropic", - model_family="claude-opus", - model_version=(4, 8), - api_dialect=pcs.DIALECT_ANTHROPIC_MESSAGES, - ) - - assert {quirk.quirk_id for quirk in matching} == { - "moonshot.kimi-k2.5-k2.6.provider-fixed-temperature", - "moonshot.kimi-k2.5-k2.6.tool-history-reasoning-content", + assert record.vendor == "mistral" + assert serialized["schema_version"] == 1 + assert serialized["provider"] == "mistral" + assert serialized["features"] == [mc.CAP_TOOL_CALL] + assert serialized["evidence"] == { + "source": mc.SOURCE_PROVIDER_READER, + "confidence": mc.CONFIDENCE_PROVIDER_REPORTED, + "provider_source": pcs.PROVIDER_SOURCE_PAYLOAD, + "shape": "mistral.models.rich.v1", + "fallback": False, } - assert lookalike == () - assert opus_without_version == () - assert [quirk.quirk_id for quirk in opus_structured] == [ - "anthropic.claude-opus-4.7-plus.omit-sampling-controls" - ] + assert "capability" not in serialized + assert "capability_assertions" not in serialized + assert "deterministic_controls" not in serialized From 5dedd262d1eb99c7a544140cde1dc178dcf8e5d5 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:46:09 +0000 Subject: [PATCH 03/14] feat(models): log capability normalization diagnostics --- app.py | 11 +++- src/model_capability_readers/__init__.py | 38 +++++++++++++- tests/test_model_capability_diagnostics.py | 59 ++++++++++++++++++++++ 3 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 tests/test_model_capability_diagnostics.py diff --git a/app.py b/app.py index e740ad518..ac90ed789 100644 --- a/app.py +++ b/app.py @@ -85,7 +85,12 @@ import logging.handlers from core.constants import DATA_DIR _root_logger = logging.getLogger() -_root_logger.setLevel(logging.INFO) +_log_level_name = os.getenv("LOG_LEVEL", "INFO").strip().upper() +_log_level = getattr(logging, _log_level_name, logging.INFO) +if not isinstance(_log_level, int): + _log_level_name = "INFO" + _log_level = logging.INFO +_root_logger.setLevel(_log_level) _formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') # Clear existing handlers to avoid duplicates @@ -93,6 +98,7 @@ for _h in list(_root_logger.handlers): _root_logger.removeHandler(_h) _console_h = logging.StreamHandler() +_console_h.setLevel(_log_level) _console_h.setFormatter(_formatter) _root_logger.addHandler(_console_h) @@ -107,6 +113,7 @@ try: _file_h = logging.handlers.RotatingFileHandler( _log_file, maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8" ) + _file_h.setLevel(_log_level) _file_h.setFormatter(_formatter) _root_logger.addHandler(_file_h) except Exception as e: @@ -1278,4 +1285,4 @@ if __name__ == "__main__": bind_host = os.getenv("APP_BIND", "127.0.0.1") bind_port = int(os.getenv("APP_PORT", "7000")) - uvicorn.run(app, host=bind_host, port=bind_port, log_level="info") + uvicorn.run(app, host=bind_host, port=bind_port, log_level=_log_level_name.lower()) diff --git a/src/model_capability_readers/__init__.py b/src/model_capability_readers/__init__.py index 56ff56e13..5a67edf3f 100644 --- a/src/model_capability_readers/__init__.py +++ b/src/model_capability_readers/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from dataclasses import replace from typing import Any @@ -56,6 +57,9 @@ from src.model_capability_readers.base import ( ) +logger = logging.getLogger(__name__) + + READER_MODULES = { VENDOR_GENERIC_OPENAI: generic_openai, VENDOR_OPENAI: openai, @@ -114,7 +118,7 @@ def records_from_payload( ) else: records = reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url) - return tuple( + normalized = tuple( replace( record, provider_source=resolution.provider_source, @@ -123,6 +127,38 @@ def records_from_payload( ) for record in records ) + if logger.isEnabledFor(logging.DEBUG): + families = sorted({record.capability.family for record in normalized}) + features = sorted( + { + feature + for record in normalized + for feature in record.capability.capabilities + } + ) + controls = sorted( + { + control.control + for record in normalized + for control in record.deterministic_controls + if control.control + } + ) + logger.debug( + "[model-capability] normalized: canonical_version=%s provider=%s " + "provider_source=%s catalog_shape=%s fallback=%s records=%d " + "families=%s features=%s controls=%s", + CANONICAL_MODEL_SHAPE_VERSION, + resolution.provider_id, + resolution.provider_source, + resolution.shape_id or "unknown", + resolution.fallback, + len(normalized), + families, + features, + controls, + ) + return normalized __all__ = [ diff --git a/tests/test_model_capability_diagnostics.py b/tests/test_model_capability_diagnostics.py new file mode 100644 index 000000000..3582ae425 --- /dev/null +++ b/tests/test_model_capability_diagnostics.py @@ -0,0 +1,59 @@ +import logging +from pathlib import Path + +from src.model_capability_readers import records_from_payload + + +def test_normalization_debug_log_reports_shape_without_payload_identity(caplog): + payload = { + "data": [ + { + "id": "sensitive-model-id", + "architecture": {"modality": "text+image->text"}, + "supported_parameters": ["tools", "temperature"], + "private_field": "secret-value", + } + ] + } + + with caplog.at_level(logging.DEBUG, logger="src.model_capability_readers"): + records = records_from_payload(payload, vendor="openrouter") + + assert len(records) == 1 + message = caplog.messages[-1] + assert "[model-capability] normalized:" in message + assert "canonical_version=1" in message + assert "provider=openrouter" in message + assert "provider_source=explicit" in message + assert "catalog_shape=openrouter.models.rich.v1" in message + assert "fallback=False" in message + assert "records=1" in message + assert "families=['chat']" in message + assert "features=['tool_call', 'vision']" in message + assert "controls=['temperature']" in message + assert "sensitive-model-id" not in message + assert "secret-value" not in message + + +def test_fallback_debug_log_is_explicit_and_has_no_capability_claims(caplog): + payload = [{"id": "future-model", "capabilities": {"tools": True}}] + + with caplog.at_level(logging.DEBUG, logger="src.model_capability_readers"): + records = records_from_payload(payload, vendor="future-provider") + + assert records[0].capability.capabilities == () + message = caplog.messages[-1] + assert "provider=future_provider" in message + assert "catalog_shape=fallback.models.list.v1" in message + assert "fallback=True" in message + assert "features=[]" in message + + +def test_web_app_logging_uses_existing_log_level_environment_toggle(): + source = (Path(__file__).resolve().parents[1] / "app.py").read_text(encoding="utf-8") + + assert 'os.getenv("LOG_LEVEL", "INFO")' in source + assert "_root_logger.setLevel(_log_level)" in source + assert "_console_h.setLevel(_log_level)" in source + assert "_file_h.setLevel(_log_level)" in source + assert "log_level=_log_level_name.lower()" in source From 9a04d98a3846e837b3441a7f3f2c9e72b5b122a4 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sat, 18 Jul 2026 11:20:19 +0000 Subject: [PATCH 04/14] fix(models): harden capability catalog normalization --- app.py | 27 ++- core/log_safety.py | 54 ++++++ src/model_capability_readers/__init__.py | 85 +++++++-- src/model_capability_readers/sglang.py | 11 ++ src/provider_capability_schemas.py | 107 ++++++++--- tests/test_log_safety.py | 56 +++++- tests/test_model_capability_diagnostics.py | 18 +- tests/test_provider_capability_schemas.py | 209 +++++++++++++++++++-- 8 files changed, 496 insertions(+), 71 deletions(-) diff --git a/app.py b/app.py index ac90ed789..bae972452 100644 --- a/app.py +++ b/app.py @@ -83,14 +83,23 @@ from starlette.responses import RedirectResponse # ========= LOGGING ========= import logging.handlers from core.constants import DATA_DIR +from core.log_safety import ( + CAPABILITY_DIAGNOSTICS_LOGGER, + ScopedDiagnosticsFilter, + application_log_settings, +) _root_logger = logging.getLogger() _log_level_name = os.getenv("LOG_LEVEL", "INFO").strip().upper() -_log_level = getattr(logging, _log_level_name, logging.INFO) -if not isinstance(_log_level, int): - _log_level_name = "INFO" - _log_level = logging.INFO -_root_logger.setLevel(_log_level) +_application_log_level, _capability_debug = application_log_settings(_log_level_name) +_root_logger.setLevel(_application_log_level) +logging.getLogger(CAPABILITY_DIAGNOSTICS_LOGGER).setLevel( + logging.DEBUG if _capability_debug else logging.NOTSET +) +_diagnostics_filter = ScopedDiagnosticsFilter( + _application_log_level, + capability_debug=_capability_debug, +) _formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') # Clear existing handlers to avoid duplicates @@ -98,7 +107,8 @@ for _h in list(_root_logger.handlers): _root_logger.removeHandler(_h) _console_h = logging.StreamHandler() -_console_h.setLevel(_log_level) +_console_h.setLevel(logging.DEBUG) +_console_h.addFilter(_diagnostics_filter) _console_h.setFormatter(_formatter) _root_logger.addHandler(_console_h) @@ -113,7 +123,8 @@ try: _file_h = logging.handlers.RotatingFileHandler( _log_file, maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8" ) - _file_h.setLevel(_log_level) + _file_h.setLevel(logging.DEBUG) + _file_h.addFilter(_diagnostics_filter) _file_h.setFormatter(_formatter) _root_logger.addHandler(_file_h) except Exception as e: @@ -1285,4 +1296,4 @@ if __name__ == "__main__": bind_host = os.getenv("APP_BIND", "127.0.0.1") bind_port = int(os.getenv("APP_PORT", "7000")) - uvicorn.run(app, host=bind_host, port=bind_port, log_level=_log_level_name.lower()) + uvicorn.run(app, host=bind_host, port=bind_port, log_level=_application_log_level) diff --git a/core/log_safety.py b/core/log_safety.py index 2339a73b6..a314e1e98 100644 --- a/core/log_safety.py +++ b/core/log_safety.py @@ -7,9 +7,63 @@ raw leaks those secrets, so route/diagnostic logs run URLs through also doubles as a sanitizer barrier for CodeQL's clear-text-logging query. """ +from __future__ import annotations + +import logging + from urllib.parse import urlparse, urlunparse +CAPABILITY_DIAGNOSTICS_LOGGER = "src.model_capability_readers" + +_LOG_LEVELS = { + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARN": logging.WARNING, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR, + "FATAL": logging.CRITICAL, + "CRITICAL": logging.CRITICAL, +} + + +def application_log_settings(value: object) -> tuple[int, bool]: + """Return the safe app level and whether scoped capability debug is on. + + Application-wide DEBUG logging can expose request bodies, provider + responses, or credentials from unrelated libraries. The model capability + catalog has a deliberately bounded DEBUG summary, so a DEBUG request is + translated into INFO for the application and enabled only for that logger. + Unknown values also fail closed to INFO. + """ + + requested = _LOG_LEVELS.get(str(value or "INFO").strip().upper(), logging.INFO) + return max(requested, logging.INFO), requested == logging.DEBUG + + +class ScopedDiagnosticsFilter(logging.Filter): + """Allow normal application records plus one explicitly scoped DEBUG log.""" + + def __init__( + self, + application_level: int, + *, + capability_debug: bool = False, + ) -> None: + super().__init__() + self.application_level = application_level + self.capability_debug = capability_debug + + def filter(self, record: logging.LogRecord) -> bool: + if record.levelno >= self.application_level: + return True + return ( + self.capability_debug + and record.levelno >= logging.DEBUG + and record.name == CAPABILITY_DIAGNOSTICS_LOGGER + ) + + def redact_url(url: str) -> str: """Return a URL safe for logs by removing userinfo and query/fragment. diff --git a/src/model_capability_readers/__init__.py b/src/model_capability_readers/__init__.py index 5a67edf3f..d25721d27 100644 --- a/src/model_capability_readers/__init__.py +++ b/src/model_capability_readers/__init__.py @@ -108,25 +108,71 @@ def records_from_payload( if vendor_id == pcs.PROVIDER_UNKNOWN: vendor_id = detect_vendor(base_url, endpoint_kind) reader = reader_for_vendor(vendor_id) - if reader is generic_openai: - record_vendor = vendor_id if vendor_id else VENDOR_UNKNOWN - records = reader.records_from_payload( + + record_vendor = vendor_id if vendor_id else VENDOR_UNKNOWN + + def annotate( + records: tuple[ModelCapabilityRecord, ...], + *, + shape_id: str, + fallback: bool, + ) -> tuple[ModelCapabilityRecord, ...]: + return tuple( + replace( + record, + provider_source=resolution.provider_source, + catalog_shape_id=shape_id, + fallback=fallback, + ) + for record in records + ) + + shape = pcs.catalog_shape_for_id(resolution.shape_id) + if resolution.fallback or reader is generic_openai or shape is None: + records = generic_openai.records_from_payload( payload, vendor_id=record_vendor, endpoint_id=endpoint_id, base_url=base_url, ) - else: - records = reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url) - normalized = tuple( - replace( - record, - provider_source=resolution.provider_source, - catalog_shape_id=resolution.shape_id, + normalized = annotate( + records, + shape_id=resolution.shape_id, fallback=resolution.fallback, ) - for record in records - ) + else: + normalized_records: list[ModelCapabilityRecord] = [] + for item in shape.items(payload): + item_payload = shape.payload_for_item(payload, item) + if shape.item_matches(item): + native_records = reader.records_from_payload( + item_payload, + endpoint_id=endpoint_id, + base_url=base_url, + ) + if native_records: + normalized_records.extend( + annotate(native_records, shape_id=shape.shape_id, fallback=False) + ) + continue + + fallback_record = generic_openai.record_from_model( + item, + vendor_id=record_vendor, + endpoint_id=endpoint_id, + base_url=base_url, + ) + if fallback_record: + fallback_shape = pcs.fallback_shape_for_payload(item_payload) + normalized_records.extend( + annotate( + (fallback_record,), + shape_id=fallback_shape.shape_id if fallback_shape else "", + fallback=True, + ) + ) + normalized = tuple(normalized_records) + if logger.isEnabledFor(logging.DEBUG): families = sorted({record.capability.family for record in normalized}) features = sorted( @@ -144,16 +190,27 @@ def records_from_payload( if control.control } ) + fallback_count = sum(record.fallback for record in normalized) + diagnostic_provider = ( + resolution.provider_id + if resolution.provider_id in pcs.PROVIDER_SCHEMAS + else "unknown" + if resolution.provider_id == pcs.PROVIDER_UNKNOWN + else "unregistered" + ) logger.debug( "[model-capability] normalized: canonical_version=%s provider=%s " "provider_source=%s catalog_shape=%s fallback=%s records=%d " + "native_records=%d fallback_records=%d " "families=%s features=%s controls=%s", CANONICAL_MODEL_SHAPE_VERSION, - resolution.provider_id, + diagnostic_provider, resolution.provider_source, resolution.shape_id or "unknown", - resolution.fallback, + bool(fallback_count), len(normalized), + len(normalized) - fallback_count, + fallback_count, families, features, controls, diff --git a/src/model_capability_readers/sglang.py b/src/model_capability_readers/sglang.py index 3ac59b060..be1ce69af 100644 --- a/src/model_capability_readers/sglang.py +++ b/src/model_capability_readers/sglang.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Mapping +from dataclasses import replace from pathlib import PurePosixPath from typing import Any @@ -15,6 +16,7 @@ from src.model_capability_readers.base import ( build_capability, compact_str, deterministic_controls_from_supported_parameters, + int_limit, openai_model_items, stable_model_id_for, ) @@ -95,5 +97,14 @@ def records_from_payload( base_url=base_url, ) if record: + context_tokens = int_limit(item.get("max_model_len")) + if context_tokens: + record = replace( + record, + capability=build_capability( + family=mc.FAMILY_UNKNOWN, + limits={"context_tokens": context_tokens}, + ), + ) records.append(record) return tuple(records) diff --git a/src/provider_capability_schemas.py b/src/provider_capability_schemas.py index 55f4809da..bafc06c05 100644 --- a/src/provider_capability_schemas.py +++ b/src/provider_capability_schemas.py @@ -83,36 +83,49 @@ class ProviderCatalogShape: def items(self, payload: Any) -> tuple[Mapping[str, Any], ...]: return _items_for_envelope(payload, self.envelope) + def item_matches(self, item: Mapping[str, Any]) -> bool: + if self.identity_paths and not any( + (value := _path_value(item, path)) is not _MISSING + and value is not None + and value != "" + for path in self.identity_paths + ): + return False + if not all(_path_present(item, path) for path in self.required_item_paths): + return False + if self.required_item_any_paths and not any( + _path_present(item, path) for path in self.required_item_any_paths + ): + return False + if any( + not isinstance(_path_value(item, path), expected_types) + for path, expected_types in self.item_types + ): + return False + if any(_path_value(item, path) not in expected for path, expected in self.item_values): + return False + return True + def matches(self, payload: Any) -> bool: if self.required_root_paths: if not isinstance(payload, Mapping): return False if not all(_path_present(payload, path) for path in self.required_root_paths): return False + return any(self.item_matches(item) for item in self.items(payload)) - for item in self.items(payload): - if self.identity_paths and not any( - (value := _path_value(item, path)) is not _MISSING - and value is not None - and value != "" - for path in self.identity_paths - ): - continue - if not all(_path_present(item, path) for path in self.required_item_paths): - continue - if self.required_item_any_paths and not any( - _path_present(item, path) for path in self.required_item_any_paths - ): - continue - if any( - not isinstance(_path_value(item, path), expected_types) - for path, expected_types in self.item_types - ): - continue - if any(_path_value(item, path) not in expected for path, expected in self.item_values): - continue - return True - return False + def payload_for_item(self, payload: Any, item: Mapping[str, Any]) -> Any: + """Return a one-item payload in the same provider-native envelope.""" + + if self.envelope == ENVELOPE_BARE_LIST: + return [item] + if self.envelope == ENVELOPE_SINGLE: + return item + if isinstance(payload, Mapping): + narrowed = dict(payload) + narrowed[self.envelope] = [item] + return narrowed + return {self.envelope: [item]} @dataclass(frozen=True) @@ -183,11 +196,19 @@ OPENROUTER_MODELS_SHAPE = ProviderCatalogShape( provider_id="openrouter", envelope=ENVELOPE_DATA, identity_paths=("id",), - required_item_any_paths=( + required_item_paths=( "architecture", + "canonical_slug", + "pricing", "supported_parameters", "top_provider", - "canonical_slug", + ), + item_types=( + ("architecture", (Mapping,)), + ("canonical_slug", (str,)), + ("pricing", (Mapping,)), + ("supported_parameters", (list, tuple)), + ("top_provider", (Mapping,)), ), detection_priority=90, ) @@ -232,7 +253,7 @@ LMSTUDIO_MODELS_V1_SHAPE = ProviderCatalogShape( "quantization", ), item_types=(("type", (str,)),), - detection_priority=100, + detection_priority=0, ) LMSTUDIO_MODELS_V0_SHAPE = ProviderCatalogShape( shape_id="lmstudio.models.native.v0", @@ -242,7 +263,7 @@ LMSTUDIO_MODELS_V0_SHAPE = ProviderCatalogShape( required_item_paths=("type",), required_item_any_paths=("arch", "compatibility_type", "state", "max_context_length"), item_types=(("type", (str,)),), - detection_priority=80, + detection_priority=0, ) LLAMACPP_PROPS_SHAPE = ProviderCatalogShape( shape_id="llamacpp.props.v1", @@ -268,7 +289,7 @@ MISTRAL_MODELS_SHAPE = ProviderCatalogShape( "capabilities.classification", ), item_types=(("capabilities", (Mapping,)),), - detection_priority=100, + detection_priority=0, ) COPILOT_MODELS_SHAPE = ProviderCatalogShape( shape_id="github-copilot.models.v1", @@ -353,7 +374,7 @@ COHERE_MODELS_SHAPE = ProviderCatalogShape( "sampling_defaults", ), item_types=(("endpoints", (list, tuple)),), - detection_priority=100, + detection_priority=0, ) MINIMAX_MODELS_SHAPE = ProviderCatalogShape( shape_id="minimax.models.identity.v1", @@ -527,6 +548,18 @@ def schema_for_provider(value: Any) -> ProviderCapabilitySchema: return PROVIDER_SCHEMAS.get(normalize_provider_id(value), UNKNOWN_SCHEMA) +def provider_from_endpoint_kind(value: Any) -> str: + """Return a provider only for registered provider-valued endpoint kinds. + + Endpoint configuration normally stores transport categories such as + ``auto``, ``local``, ``api``, and ``proxy``. Those categories and unknown + values must not preempt provider identity from a host or native payload. + """ + + provider_id = normalize_provider_id(value) + return provider_id if provider_id in PROVIDER_SCHEMAS else PROVIDER_UNKNOWN + + def _host_matches(host: str, suffix: str) -> bool: return host == suffix or host.endswith("." + suffix) @@ -576,6 +609,18 @@ def native_shape_for_payload( return sorted(best, key=lambda shape: shape.shape_id)[0] +def catalog_shape_for_id(shape_id: Any) -> ProviderCatalogShape | None: + return next( + ( + shape + for schema in PROVIDER_SCHEMAS.values() + for shape in schema.catalog_shapes + if shape.shape_id == shape_id + ), + None, + ) + + def fallback_shape_for_payload(payload: Any) -> ProviderCatalogShape | None: return next((shape for shape in FALLBACK_CATALOG_SHAPES if shape.matches(payload)), None) @@ -591,7 +636,7 @@ def resolve_provider( provider_source = PROVIDER_SOURCE_EXPLICIT if provider_id == PROVIDER_UNKNOWN: - provider_id = normalize_provider_id(endpoint_kind) + provider_id = provider_from_endpoint_kind(endpoint_kind) provider_source = PROVIDER_SOURCE_ENDPOINT_KIND if provider_id == PROVIDER_UNKNOWN: provider_id = provider_from_host(base_url) @@ -647,9 +692,11 @@ __all__ = [ "ProviderCapabilitySchema", "ProviderCatalogShape", "ProviderResolution", + "catalog_shape_for_id", "fallback_shape_for_payload", "native_shape_for_payload", "normalize_provider_id", + "provider_from_endpoint_kind", "provider_from_host", "resolve_provider", "schema_for_provider", diff --git a/tests/test_log_safety.py b/tests/test_log_safety.py index b806516d2..a4de26666 100644 --- a/tests/test_log_safety.py +++ b/tests/test_log_safety.py @@ -1,4 +1,13 @@ -from core.log_safety import redact_url +import logging + +import pytest + +from core.log_safety import ( + CAPABILITY_DIAGNOSTICS_LOGGER, + ScopedDiagnosticsFilter, + application_log_settings, + redact_url, +) def test_strips_userinfo(): @@ -30,3 +39,48 @@ def test_empty_and_none(): def test_garbage_does_not_raise(): # urlparse is lenient; just assert no credential-looking userinfo survives. assert "@" not in redact_url("::::not a url::::") + + +@pytest.mark.parametrize( + ("configured", "expected_level", "expected_capability_debug"), + ( + ("DEBUG", logging.INFO, True), + ("debug", logging.INFO, True), + ("INFO", logging.INFO, False), + ("WARNING", logging.WARNING, False), + ("ERROR", logging.ERROR, False), + ("CRITICAL", logging.CRITICAL, False), + ("not-a-level", logging.INFO, False), + (None, logging.INFO, False), + ), +) +def test_application_log_settings_scope_debug_and_fail_closed( + configured, + expected_level, + expected_capability_debug, +): + assert application_log_settings(configured) == ( + expected_level, + expected_capability_debug, + ) + + +def _record(name: str, level: int) -> logging.LogRecord: + return logging.LogRecord(name, level, __file__, 1, "message", (), None) + + +def test_scoped_diagnostics_filter_allows_only_bounded_debug_logger(): + log_filter = ScopedDiagnosticsFilter(logging.INFO, capability_debug=True) + + assert log_filter.filter(_record(CAPABILITY_DIAGNOSTICS_LOGGER, logging.DEBUG)) + assert log_filter.filter(_record("unrelated.library", logging.INFO)) + assert not log_filter.filter(_record("unrelated.library", logging.DEBUG)) + assert not log_filter.filter(_record(f"{CAPABILITY_DIAGNOSTICS_LOGGER}.raw", logging.DEBUG)) + + +def test_scoped_diagnostics_filter_respects_higher_application_level(): + log_filter = ScopedDiagnosticsFilter(logging.WARNING, capability_debug=False) + + assert log_filter.filter(_record("application", logging.WARNING)) + assert not log_filter.filter(_record("application", logging.INFO)) + assert not log_filter.filter(_record(CAPABILITY_DIAGNOSTICS_LOGGER, logging.DEBUG)) diff --git a/tests/test_model_capability_diagnostics.py b/tests/test_model_capability_diagnostics.py index 3582ae425..7b1969430 100644 --- a/tests/test_model_capability_diagnostics.py +++ b/tests/test_model_capability_diagnostics.py @@ -10,7 +10,10 @@ def test_normalization_debug_log_reports_shape_without_payload_identity(caplog): { "id": "sensitive-model-id", "architecture": {"modality": "text+image->text"}, + "canonical_slug": "provider/sensitive-model-id", + "pricing": {"prompt": "0.1", "completion": "0.2"}, "supported_parameters": ["tools", "temperature"], + "top_provider": {"context_length": 32768}, "private_field": "secret-value", } ] @@ -28,6 +31,8 @@ def test_normalization_debug_log_reports_shape_without_payload_identity(caplog): assert "catalog_shape=openrouter.models.rich.v1" in message assert "fallback=False" in message assert "records=1" in message + assert "native_records=1" in message + assert "fallback_records=0" in message assert "families=['chat']" in message assert "features=['tool_call', 'vision']" in message assert "controls=['temperature']" in message @@ -43,9 +48,11 @@ def test_fallback_debug_log_is_explicit_and_has_no_capability_claims(caplog): assert records[0].capability.capabilities == () message = caplog.messages[-1] - assert "provider=future_provider" in message + assert "provider=unregistered" in message assert "catalog_shape=fallback.models.list.v1" in message assert "fallback=True" in message + assert "native_records=0" in message + assert "fallback_records=1" in message assert "features=[]" in message @@ -53,7 +60,8 @@ def test_web_app_logging_uses_existing_log_level_environment_toggle(): source = (Path(__file__).resolve().parents[1] / "app.py").read_text(encoding="utf-8") assert 'os.getenv("LOG_LEVEL", "INFO")' in source - assert "_root_logger.setLevel(_log_level)" in source - assert "_console_h.setLevel(_log_level)" in source - assert "_file_h.setLevel(_log_level)" in source - assert "log_level=_log_level_name.lower()" in source + assert "application_log_settings(_log_level_name)" in source + assert "_root_logger.setLevel(_application_log_level)" in source + assert "_console_h.addFilter(_diagnostics_filter)" in source + assert "_file_h.addFilter(_diagnostics_filter)" in source + assert "log_level=_application_log_level" in source diff --git a/tests/test_provider_capability_schemas.py b/tests/test_provider_capability_schemas.py index 9d5349c85..87d4244a7 100644 --- a/tests/test_provider_capability_schemas.py +++ b/tests/test_provider_capability_schemas.py @@ -14,6 +14,22 @@ from src.model_capability_readers import ( ) +def _openrouter_payload(*items): + return { + "data": list(items) + or [ + { + "id": "provider/model", + "architecture": {"modality": "text->text"}, + "canonical_slug": "provider/model", + "pricing": {"prompt": "0.1", "completion": "0.2"}, + "supported_parameters": ["tools", "temperature"], + "top_provider": {"context_length": 32768}, + } + ] + } + + def test_provider_identity_and_catalog_shape_are_resolved_separately(): google_payload = { "models": [ @@ -71,6 +87,27 @@ def test_provider_host_matching_rejects_lookalikes_and_does_not_use_ports(): assert pcs.provider_from_host("http://127.0.0.1:30000") == pcs.PROVIDER_UNKNOWN +def test_transport_endpoint_kinds_do_not_preempt_host_or_payload_provider_identity(): + payload = _openrouter_payload() + + for endpoint_kind in ("auto", "local", "api", "proxy", "future-transport"): + from_host = pcs.resolve_provider( + payload, + endpoint_kind=endpoint_kind, + base_url="https://api.openrouter.ai/v1", + ) + from_payload = pcs.resolve_provider(payload, endpoint_kind=endpoint_kind) + + assert from_host.provider_id == "openrouter" + assert from_host.provider_source == pcs.PROVIDER_SOURCE_HOST + assert from_payload.provider_id == "openrouter" + assert from_payload.provider_source == pcs.PROVIDER_SOURCE_PAYLOAD + + assert pcs.provider_from_endpoint_kind("ollama") == "ollama" + assert pcs.provider_from_endpoint_kind("llama.cpp") == "llamacpp" + assert pcs.provider_from_endpoint_kind("proxy") == pcs.PROVIDER_UNKNOWN + + def test_provider_aliases_only_normalize_explicit_identity(): assert pcs.normalize_provider_id("opencode-go") == "opencode" assert pcs.normalize_provider_id("opencode-zen") == "opencode" @@ -102,8 +139,13 @@ def test_unregistered_explicit_provider_is_preserved_but_stays_on_fallback(): assert records[0].capability.capabilities == () -def test_current_native_catalog_shapes_are_discriminating(): +def test_native_catalog_shapes_resolve_with_required_provider_context(): cases = ( + ( + _openrouter_payload(), + "openrouter", + "openrouter.models.rich.v1", + ), ( {"models": [{"key": "local/model", "type": "llm", "capabilities": {"vision": True}}]}, "lmstudio", @@ -198,30 +240,135 @@ def test_current_native_catalog_shapes_are_discriminating(): ), ) + explicit_context_providers = {"cohere", "lmstudio", "mistral"} for payload, expected_provider, expected_shape in cases: - resolution = pcs.resolve_provider(payload) + explicit_provider = ( + expected_provider if expected_provider in explicit_context_providers else None + ) + resolution = pcs.resolve_provider(payload, provider=explicit_provider) assert resolution.provider_id == expected_provider - assert resolution.provider_source == pcs.PROVIDER_SOURCE_PAYLOAD + assert resolution.provider_source == ( + pcs.PROVIDER_SOURCE_EXPLICIT + if explicit_provider + else pcs.PROVIDER_SOURCE_PAYLOAD + ) assert resolution.shape_id == expected_shape assert resolution.fallback is False -def test_wrong_native_field_types_degrade_to_explicit_fallback_inventory(): - malformed_cohere = pcs.resolve_provider( - {"models": [{"name": "future", "endpoints": "chat", "context_length": 4096}]} - ) - malformed_mistral = pcs.resolve_provider( - {"data": [{"id": "future", "capabilities": ["completion_chat"]}]} +def test_ambiguous_common_fields_do_not_infer_provider_from_payload_alone(): + cases = ( + ({"data": [{"id": "generic", "architecture": {}}]}, "openrouter"), + ({"data": [{"id": "generic", "supported_parameters": ["tools"]}]}, "openrouter"), + ( + {"data": [{"id": "generic", "capabilities": {"completion_chat": True}}]}, + "mistral", + ), + ({"data": [{"id": "generic", "type": "llm", "arch": "future"}]}, "lmstudio"), + ( + {"models": [{"key": "generic", "type": "llm", "capabilities": {}}]}, + "lmstudio", + ), + ( + {"models": [{"name": "generic", "endpoints": ["chat"], "context_length": 4096}]}, + "cohere", + ), ) - assert malformed_cohere.provider_id == pcs.PROVIDER_UNKNOWN + for payload, provider_id in cases: + inferred = pcs.resolve_provider(payload) + contextual = pcs.resolve_provider(payload, provider=provider_id) + + assert inferred.provider_id == pcs.PROVIDER_UNKNOWN + assert inferred.fallback is True + assert contextual.provider_id == provider_id + + +def test_openrouter_payload_detection_requires_the_compound_official_shape(): + complete = _openrouter_payload() + assert pcs.resolve_provider(complete).shape_id == "openrouter.models.rich.v1" + + item = complete["data"][0] + for required_field in ( + "architecture", + "canonical_slug", + "pricing", + "supported_parameters", + "top_provider", + ): + partial = _openrouter_payload( + {key: value for key, value in item.items() if key != required_field} + ) + resolution = pcs.resolve_provider(partial) + + assert resolution.provider_id == pcs.PROVIDER_UNKNOWN + assert resolution.shape_id == "fallback.models.data.v1" + assert resolution.fallback is True + + +def test_wrong_native_field_types_degrade_to_explicit_fallback_inventory(): + malformed_cohere = pcs.resolve_provider( + {"models": [{"name": "future", "endpoints": "chat", "context_length": 4096}]}, + provider="cohere", + ) + malformed_mistral = pcs.resolve_provider( + {"data": [{"id": "future", "capabilities": ["completion_chat"]}]}, + provider="mistral", + ) + + assert malformed_cohere.provider_id == "cohere" assert malformed_cohere.shape_id == "fallback.models.envelope.v1" assert malformed_cohere.fallback is True - assert malformed_mistral.provider_id == pcs.PROVIDER_UNKNOWN + assert malformed_mistral.provider_id == "mistral" assert malformed_mistral.shape_id == "fallback.models.data.v1" assert malformed_mistral.fallback is True +def test_explicit_fallback_and_mixed_native_payloads_are_normalized_per_item(): + malformed = {"models": [{"name": "unsafe", "endpoints": "chat", "context_length": 4096}]} + malformed_record = records_from_payload(malformed, vendor="cohere")[0] + + assert malformed_record.vendor == "cohere" + assert malformed_record.capability.family == mc.FAMILY_UNKNOWN + assert dict(malformed_record.capability.limits) == {} + assert malformed_record.catalog_shape_id == "fallback.models.envelope.v1" + assert malformed_record.fallback is True + + valid_item = { + "name": "native", + "endpoints": ["chat"], + "context_length": 131072, + } + native, fallback = records_from_payload( + {"models": [valid_item, malformed["models"][0]]}, + vendor="cohere", + ) + + assert native.model_id == "native" + assert native.capability.family == mc.FAMILY_CHAT + assert dict(native.capability.limits) == {"context_tokens": 131072} + assert native.catalog_shape_id == "cohere.models.rich.v1" + assert native.fallback is False + assert fallback.model_id == "unsafe" + assert fallback.capability.family == mc.FAMILY_UNKNOWN + assert dict(fallback.capability.limits) == {} + assert fallback.catalog_shape_id == "fallback.models.envelope.v1" + assert fallback.fallback is True + + +def test_provider_specific_reader_is_not_used_for_a_different_fallback_envelope(): + record = records_from_payload( + [{"id": "untrusted", "architecture": {"modality": "text+image->text"}}], + vendor="openrouter", + )[0] + + assert record.vendor == "openrouter" + assert record.capability.family == mc.FAMILY_UNKNOWN + assert record.capability.capabilities == () + assert record.catalog_shape_id == "fallback.models.list.v1" + assert record.fallback is True + + def test_fallback_reader_is_identity_only_even_for_dangerous_looking_fields(): payload = [ { @@ -375,6 +522,41 @@ def test_sglang_model_info_maps_native_generation_flags_only(): assert pooling.capability.family == mc.FAMILY_UNKNOWN +def test_sglang_openai_catalog_preserves_only_valid_native_context_limit(): + valid = records_from_payload( + { + "data": [ + { + "id": "served-model", + "owned_by": "sglang", + "root": "org/model", + "max_model_len": 131072, + } + ] + } + )[0] + nonpositive = sglang.records_from_payload( + { + "data": [ + { + "id": "served-model", + "owned_by": "sglang", + "root": "org/model", + "max_model_len": 0, + } + ] + } + )[0] + + assert valid.vendor == "sglang" + assert valid.capability.family == mc.FAMILY_UNKNOWN + assert valid.capability.capabilities == () + assert dict(valid.capability.limits) == {"context_tokens": 131072} + assert valid.catalog_shape_id == "sglang.models.openai.v1" + assert valid.fallback is False + assert dict(nonpositive.capability.limits) == {} + + def test_identity_only_native_catalogs_remain_unknown(): anthropic_record = anthropic.records_from_payload( { @@ -461,7 +643,8 @@ def test_reader_wrapper_adds_one_lean_evidence_object(): "capabilities": {"completion_chat": True, "function_calling": True}, } ] - } + }, + base_url="https://api.mistral.ai/v1", )[0] serialized = record.to_dict() @@ -472,7 +655,7 @@ def test_reader_wrapper_adds_one_lean_evidence_object(): assert serialized["evidence"] == { "source": mc.SOURCE_PROVIDER_READER, "confidence": mc.CONFIDENCE_PROVIDER_REPORTED, - "provider_source": pcs.PROVIDER_SOURCE_PAYLOAD, + "provider_source": pcs.PROVIDER_SOURCE_HOST, "shape": "mistral.models.rich.v1", "fallback": False, } From 3ec6cb4b50602a6ffa0dac8fe96ba497dc3efe40 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sat, 18 Jul 2026 11:34:41 +0000 Subject: [PATCH 05/14] fix(models): honor selected catalog envelope --- src/model_capability_readers/__init__.py | 38 ++++++++++++++++++---- src/provider_capability_schemas.py | 16 ++++++++-- tests/test_provider_capability_schemas.py | 39 +++++++++++++++++++++++ 3 files changed, 84 insertions(+), 9 deletions(-) diff --git a/src/model_capability_readers/__init__.py b/src/model_capability_readers/__init__.py index d25721d27..15b208163 100644 --- a/src/model_capability_readers/__init__.py +++ b/src/model_capability_readers/__init__.py @@ -128,7 +128,7 @@ def records_from_payload( ) shape = pcs.catalog_shape_for_id(resolution.shape_id) - if resolution.fallback or reader is generic_openai or shape is None: + if shape is None: records = generic_openai.records_from_payload( payload, vendor_id=record_vendor, @@ -140,16 +140,42 @@ def records_from_payload( shape_id=resolution.shape_id, fallback=resolution.fallback, ) + elif resolution.fallback: + normalized_records: list[ModelCapabilityRecord] = [] + for item in shape.items(payload): + fallback_record = generic_openai.record_from_model( + item, + vendor_id=record_vendor, + endpoint_id=endpoint_id, + base_url=base_url, + ) + if fallback_record: + normalized_records.extend( + annotate( + (fallback_record,), + shape_id=shape.shape_id, + fallback=True, + ) + ) + normalized = tuple(normalized_records) else: normalized_records: list[ModelCapabilityRecord] = [] for item in shape.items(payload): item_payload = shape.payload_for_item(payload, item) if shape.item_matches(item): - native_records = reader.records_from_payload( - item_payload, - endpoint_id=endpoint_id, - base_url=base_url, - ) + if reader is generic_openai: + native_records = reader.records_from_payload( + item_payload, + vendor_id=record_vendor, + endpoint_id=endpoint_id, + base_url=base_url, + ) + else: + native_records = reader.records_from_payload( + item_payload, + endpoint_id=endpoint_id, + base_url=base_url, + ) if native_records: normalized_records.extend( annotate(native_records, shape_id=shape.shape_id, fallback=False) diff --git a/src/provider_capability_schemas.py b/src/provider_capability_schemas.py index bafc06c05..0e7593fbc 100644 --- a/src/provider_capability_schemas.py +++ b/src/provider_capability_schemas.py @@ -122,7 +122,11 @@ class ProviderCatalogShape: if self.envelope == ENVELOPE_SINGLE: return item if isinstance(payload, Mapping): - narrowed = dict(payload) + narrowed = { + key: value + for key, value in payload.items() + if key not in {ENVELOPE_DATA, ENVELOPE_MODELS} + } narrowed[self.envelope] = [item] return narrowed return {self.envelope: [item]} @@ -613,8 +617,14 @@ def catalog_shape_for_id(shape_id: Any) -> ProviderCatalogShape | None: return next( ( shape - for schema in PROVIDER_SCHEMAS.values() - for shape in schema.catalog_shapes + for shape in ( + *FALLBACK_CATALOG_SHAPES, + *( + provider_shape + for schema in PROVIDER_SCHEMAS.values() + for provider_shape in schema.catalog_shapes + ), + ) if shape.shape_id == shape_id ), None, diff --git a/tests/test_provider_capability_schemas.py b/tests/test_provider_capability_schemas.py index 87d4244a7..4e61087b5 100644 --- a/tests/test_provider_capability_schemas.py +++ b/tests/test_provider_capability_schemas.py @@ -369,6 +369,45 @@ def test_provider_specific_reader_is_not_used_for_a_different_fallback_envelope( assert record.fallback is True +def test_selected_native_envelope_ignores_an_unrelated_alternate_envelope(): + record = records_from_payload( + { + "data": [{"id": "unrelated-openai-card"}], + "models": [ + { + "key": "native-lmstudio-card", + "type": "llm", + "capabilities": {"vision": True}, + } + ], + }, + vendor="lmstudio", + )[0] + + assert record.model_id == "native-lmstudio-card" + assert record.vendor == "lmstudio" + assert record.capability.family == mc.FAMILY_CHAT + assert record.capability.capabilities == (mc.CAP_VISION,) + assert record.catalog_shape_id == "lmstudio.models.native.v1" + assert record.fallback is False + + +def test_selected_fallback_envelope_is_not_shadowed_by_empty_data(): + record = records_from_payload( + { + "data": [], + "models": [{"id": "fallback-model", "capabilities": {"tools": True}}], + } + )[0] + + assert record.model_id == "fallback-model" + assert record.vendor == pcs.PROVIDER_UNKNOWN + assert record.capability.family == mc.FAMILY_UNKNOWN + assert record.capability.capabilities == () + assert record.catalog_shape_id == "fallback.models.envelope.v1" + assert record.fallback is True + + def test_fallback_reader_is_identity_only_even_for_dangerous_looking_fields(): payload = [ { From 6e6543ab13f7ea97a4951f40d2f5eb0a3160ca39 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sat, 18 Jul 2026 11:38:52 +0000 Subject: [PATCH 06/14] fix(models): reject ambiguous catalog evidence --- src/model_capability_readers/base.py | 4 +- src/provider_capability_schemas.py | 8 +-- tests/test_provider_capability_schemas.py | 60 +++++++++++++++++------ 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/model_capability_readers/base.py b/src/model_capability_readers/base.py index c2651a823..813f79754 100644 --- a/src/model_capability_readers/base.py +++ b/src/model_capability_readers/base.py @@ -178,9 +178,11 @@ def model_id_from(raw: Mapping[str, Any], *keys: str) -> str: def int_limit(value: Any) -> int | None: + if isinstance(value, bool): + return None try: limit = int(value) - except (TypeError, ValueError): + except (OverflowError, TypeError, ValueError): return None return limit if limit > 0 else None diff --git a/src/provider_capability_schemas.py b/src/provider_capability_schemas.py index 0e7593fbc..3fb679c28 100644 --- a/src/provider_capability_schemas.py +++ b/src/provider_capability_schemas.py @@ -322,7 +322,7 @@ CHATGPT_MODELS_SHAPE = ProviderCatalogShape( envelope=ENVELOPE_MODELS, identity_paths=("slug",), required_item_any_paths=("visibility", "priority"), - detection_priority=90, + detection_priority=0, ) SGLANG_MODEL_INFO_SHAPE = ProviderCatalogShape( shape_id="sglang.model-info.v2", @@ -605,11 +605,11 @@ def native_shape_for_payload( matches = [shape for shape in shapes if shape.matches(payload)] if not matches: return None - priority = max(shape.detection_priority for shape in matches) - best = [shape for shape in matches if shape.detection_priority == priority] - providers = {shape.provider_id for shape in best} + providers = {shape.provider_id for shape in matches} if len(providers) != 1: return None + priority = max(shape.detection_priority for shape in matches) + best = [shape for shape in matches if shape.detection_priority == priority] return sorted(best, key=lambda shape: shape.shape_id)[0] diff --git a/tests/test_provider_capability_schemas.py b/tests/test_provider_capability_schemas.py index 4e61087b5..4a533155c 100644 --- a/tests/test_provider_capability_schemas.py +++ b/tests/test_provider_capability_schemas.py @@ -240,7 +240,12 @@ def test_native_catalog_shapes_resolve_with_required_provider_context(): ), ) - explicit_context_providers = {"cohere", "lmstudio", "mistral"} + explicit_context_providers = { + "chatgpt_subscription", + "cohere", + "lmstudio", + "mistral", + } for payload, expected_provider, expected_shape in cases: explicit_provider = ( expected_provider if expected_provider in explicit_context_providers else None @@ -273,6 +278,10 @@ def test_ambiguous_common_fields_do_not_infer_provider_from_payload_alone(): {"models": [{"name": "generic", "endpoints": ["chat"], "context_length": 4096}]}, "cohere", ), + ( + {"models": [{"slug": "generic", "visibility": "list", "priority": 1}]}, + "chatgpt_subscription", + ), ) for payload, provider_id in cases: @@ -284,6 +293,26 @@ def test_ambiguous_common_fields_do_not_infer_provider_from_payload_alone(): assert contextual.provider_id == provider_id +def test_payload_matching_multiple_providers_degrades_to_fallback(): + payload = _openrouter_payload( + { + **_openrouter_payload()["data"][0], + "model_picker_enabled": True, + "capabilities": {"supports": {"tool_calls": True}}, + } + ) + resolution = pcs.resolve_provider(payload) + record = records_from_payload(payload)[0] + + assert resolution.provider_id == pcs.PROVIDER_UNKNOWN + assert resolution.shape_id == "fallback.models.data.v1" + assert resolution.fallback is True + assert record.vendor == pcs.PROVIDER_UNKNOWN + assert record.capability.family == mc.FAMILY_UNKNOWN + assert record.capability.capabilities == () + assert record.fallback is True + + def test_openrouter_payload_detection_requires_the_compound_official_shape(): complete = _openrouter_payload() assert pcs.resolve_provider(complete).shape_id == "openrouter.models.rich.v1" @@ -574,18 +603,21 @@ def test_sglang_openai_catalog_preserves_only_valid_native_context_limit(): ] } )[0] - nonpositive = sglang.records_from_payload( - { - "data": [ - { - "id": "served-model", - "owned_by": "sglang", - "root": "org/model", - "max_model_len": 0, - } - ] - } - )[0] + malformed = [ + records_from_payload( + { + "data": [ + { + "id": "served-model", + "owned_by": "sglang", + "root": "org/model", + "max_model_len": value, + } + ] + } + )[0] + for value in (0, True, float("inf")) + ] assert valid.vendor == "sglang" assert valid.capability.family == mc.FAMILY_UNKNOWN @@ -593,7 +625,7 @@ def test_sglang_openai_catalog_preserves_only_valid_native_context_limit(): assert dict(valid.capability.limits) == {"context_tokens": 131072} assert valid.catalog_shape_id == "sglang.models.openai.v1" assert valid.fallback is False - assert dict(nonpositive.capability.limits) == {} + assert all(dict(record.capability.limits) == {} for record in malformed) def test_identity_only_native_catalogs_remain_unknown(): From 639c440d6f3187082b985135a2bd0195baac97c6 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sat, 18 Jul 2026 11:41:15 +0000 Subject: [PATCH 07/14] fix(models): require context for Hugging Face catalogs --- src/provider_capability_schemas.py | 2 +- tests/test_provider_capability_schemas.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/provider_capability_schemas.py b/src/provider_capability_schemas.py index 3fb679c28..1ef701fdd 100644 --- a/src/provider_capability_schemas.py +++ b/src/provider_capability_schemas.py @@ -363,7 +363,7 @@ HUGGINGFACE_MODEL_SHAPE = ProviderCatalogShape( identity_paths=("modelId", "id"), required_item_paths=("pipeline_tag",), item_types=(("pipeline_tag", (str,)),), - detection_priority=80, + detection_priority=0, ) COHERE_MODELS_SHAPE = ProviderCatalogShape( shape_id="cohere.models.rich.v1", diff --git a/tests/test_provider_capability_schemas.py b/tests/test_provider_capability_schemas.py index 4a533155c..39663795b 100644 --- a/tests/test_provider_capability_schemas.py +++ b/tests/test_provider_capability_schemas.py @@ -243,6 +243,7 @@ def test_native_catalog_shapes_resolve_with_required_provider_context(): explicit_context_providers = { "chatgpt_subscription", "cohere", + "huggingface", "lmstudio", "mistral", } @@ -313,6 +314,20 @@ def test_payload_matching_multiple_providers_degrades_to_fallback(): assert record.fallback is True +def test_generic_pipeline_tag_does_not_select_huggingface_without_provider_context(): + payload = {"id": "generic-model", "pipeline_tag": "image-text-to-text"} + inferred = pcs.resolve_provider(payload) + contextual = pcs.resolve_provider(payload, provider="huggingface") + + assert inferred.provider_id == pcs.PROVIDER_UNKNOWN + assert inferred.shape_id == "" + assert inferred.fallback is False + assert records_from_payload(payload) == () + assert contextual.provider_id == "huggingface" + assert contextual.shape_id == "huggingface.hub.model-info.v1" + assert contextual.fallback is False + + def test_openrouter_payload_detection_requires_the_compound_official_shape(): complete = _openrouter_payload() assert pcs.resolve_provider(complete).shape_id == "openrouter.models.rich.v1" From a71661208a9e7a01cbab030b02746ef43be99390 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sat, 18 Jul 2026 12:08:39 +0000 Subject: [PATCH 08/14] fix(models): tighten catalog evidence boundaries --- src/model_capability_readers/base.py | 5 +- src/model_capability_readers/copilot.py | 3 - src/provider_capability_schemas.py | 26 +++-- tests/test_provider_capability_schemas.py | 117 +++++++++++++++++++++- 4 files changed, 140 insertions(+), 11 deletions(-) diff --git a/src/model_capability_readers/base.py b/src/model_capability_readers/base.py index 813f79754..f184ab413 100644 --- a/src/model_capability_readers/base.py +++ b/src/model_capability_readers/base.py @@ -171,7 +171,10 @@ def stable_model_id_for(vendor: Any, model_id: Any, *, endpoint_id: Any = "", ba def model_id_from(raw: Mapping[str, Any], *keys: str) -> str: for key in keys: - value = compact_str(raw.get(key)) + raw_value = raw.get(key) + if not isinstance(raw_value, str): + continue + value = raw_value.strip() if value: return value.removeprefix("models/") return "" diff --git a/src/model_capability_readers/copilot.py b/src/model_capability_readers/copilot.py index ffc0c670e..642cc87ed 100644 --- a/src/model_capability_readers/copilot.py +++ b/src/model_capability_readers/copilot.py @@ -24,10 +24,7 @@ vendor = VENDOR_COPILOT _SUPPORT_CAPABILITIES = { "tool_calls": mc.CAP_TOOL_CALL, - "tools": mc.CAP_TOOL_CALL, "vision": mc.CAP_VISION, - "reasoning": mc.CAP_REASONING, - "structured_outputs": mc.CAP_STRUCTURED_OUTPUT, } diff --git a/src/provider_capability_schemas.py b/src/provider_capability_schemas.py index 1ef701fdd..8250ea4f6 100644 --- a/src/provider_capability_schemas.py +++ b/src/provider_capability_schemas.py @@ -86,8 +86,8 @@ class ProviderCatalogShape: def item_matches(self, item: Mapping[str, Any]) -> bool: if self.identity_paths and not any( (value := _path_value(item, path)) is not _MISSING - and value is not None - and value != "" + and isinstance(value, str) + and bool(value.strip()) for path in self.identity_paths ): return False @@ -120,7 +120,11 @@ class ProviderCatalogShape: if self.envelope == ENVELOPE_BARE_LIST: return [item] if self.envelope == ENVELOPE_SINGLE: - return item + return { + key: value + for key, value in item.items() + if key not in {ENVELOPE_DATA, ENVELOPE_MODELS} + } if isinstance(payload, Mapping): narrowed = { key: value @@ -231,7 +235,10 @@ OLLAMA_TAGS_SHAPE = ProviderCatalogShape( envelope=ENVELOPE_MODELS, identity_paths=("model", "name"), required_item_any_paths=("digest", "details.family", "details.families"), - detection_priority=90, + # `name` plus a digest/details field is not globally provider-specific. + # Configured provider context remains authoritative for local Ollama + # inventories; payload-only detection would create false provider identity. + detection_priority=0, ) OLLAMA_SHOW_SHAPE = ProviderCatalogShape( shape_id="ollama.show.v1", @@ -241,7 +248,11 @@ OLLAMA_SHOW_SHAPE = ProviderCatalogShape( required_item_paths=("capabilities",), required_item_any_paths=("model_info", "details", "template", "parameters"), item_types=(("capabilities", (list, tuple)),), - detection_priority=100, + # `/api/show` capability and parameter fields are not sufficiently unique + # to identify an otherwise unknown provider. Local/default ports are also + # deliberately non-authoritative, so require configured provider context + # before interpreting this singleton response as Ollama-native metadata. + detection_priority=0, ) LMSTUDIO_MODELS_V1_SHAPE = ProviderCatalogShape( shape_id="lmstudio.models.native.v1", @@ -610,7 +621,10 @@ def native_shape_for_payload( return None priority = max(shape.detection_priority for shape in matches) best = [shape for shape in matches if shape.detection_priority == priority] - return sorted(best, key=lambda shape: shape.shape_id)[0] + # Registry declaration order expresses preference between revisions of the + # same provider shape (for example LM Studio v1 before v0). Alphabetical + # shape ids invert that version preference for otherwise equal evidence. + return best[0] def catalog_shape_for_id(shape_id: Any) -> ProviderCatalogShape | None: diff --git a/tests/test_provider_capability_schemas.py b/tests/test_provider_capability_schemas.py index 39663795b..030c0bc4a 100644 --- a/tests/test_provider_capability_schemas.py +++ b/tests/test_provider_capability_schemas.py @@ -246,10 +246,13 @@ def test_native_catalog_shapes_resolve_with_required_provider_context(): "huggingface", "lmstudio", "mistral", + "ollama", } for payload, expected_provider, expected_shape in cases: explicit_provider = ( - expected_provider if expected_provider in explicit_context_providers else None + expected_provider + if expected_provider in explicit_context_providers + else None ) resolution = pcs.resolve_provider(payload, provider=explicit_provider) assert resolution.provider_id == expected_provider @@ -262,6 +265,59 @@ def test_native_catalog_shapes_resolve_with_required_provider_context(): assert resolution.fallback is False +def test_generic_ollama_like_fields_require_provider_context(): + show_payload = { + "name": "foreign-model", + "capabilities": ["vision"], + "parameters": {}, + } + tags_payload = {"models": [{"name": "foreign-model", "digest": None}]} + + inferred = pcs.resolve_provider(show_payload) + contextual = pcs.resolve_provider(show_payload, provider="ollama") + + assert inferred.provider_id == pcs.PROVIDER_UNKNOWN + assert inferred.shape_id == "" + assert inferred.fallback is False + assert records_from_payload(show_payload) == () + assert contextual.provider_id == "ollama" + assert contextual.shape_id == "ollama.show.v1" + assert contextual.fallback is False + contextual_record = records_from_payload(show_payload, vendor="ollama")[0] + assert contextual_record.capability.capabilities == (mc.CAP_VISION,) + + inferred_tags = pcs.resolve_provider(tags_payload) + contextual_tags = pcs.resolve_provider(tags_payload, provider="ollama") + assert inferred_tags.provider_id == pcs.PROVIDER_UNKNOWN + assert inferred_tags.shape_id == "fallback.models.envelope.v1" + assert inferred_tags.fallback is True + assert contextual_tags.provider_id == "ollama" + assert contextual_tags.shape_id == "ollama.tags.v1" + assert contextual_tags.fallback is False + + +def test_singleton_native_reader_ignores_competing_list_envelopes(): + record = records_from_payload( + { + "model": "show-model", + "capabilities": ["completion", "vision"], + "model_info": {"family.context_length": 4096}, + "models": [ + { + "name": "shadow-model", + "digest": "abc", + "details": {"family": "shadow"}, + } + ], + }, + vendor="ollama", + )[0] + + assert record.model_id == "show-model" + assert record.catalog_shape_id == "ollama.show.v1" + assert record.capability.capabilities == (mc.CAP_VISION,) + + def test_ambiguous_common_fields_do_not_infer_provider_from_payload_alone(): cases = ( ({"data": [{"id": "generic", "architecture": {}}]}, "openrouter"), @@ -436,6 +492,32 @@ def test_selected_native_envelope_ignores_an_unrelated_alternate_envelope(): assert record.fallback is False +def test_same_provider_shape_tie_prefers_declared_modern_envelope(): + record = records_from_payload( + { + "data": [ + { + "id": "legacy-v0-card", + "type": "vlm", + "arch": "legacy", + } + ], + "models": [ + { + "key": "modern-v1-card", + "type": "llm", + "capabilities": {"vision": True}, + } + ], + }, + vendor="lmstudio", + )[0] + + assert record.model_id == "modern-v1-card" + assert record.catalog_shape_id == "lmstudio.models.native.v1" + assert record.capability.capabilities == (mc.CAP_VISION,) + + def test_selected_fallback_envelope_is_not_shadowed_by_empty_data(): record = records_from_payload( { @@ -517,6 +599,15 @@ def test_fallback_reader_fails_soft_for_null_and_malformed_envelopes(): assert generic_openai.records_from_payload(payload) == () +def test_structured_identity_values_are_not_stringified_into_fallback_records(): + for key in ("id", "name", "model", "key", "slug"): + payload = [{key: {"nested": "model"}}] + + assert pcs.resolve_provider(payload).shape_id == "" + assert generic_openai.records_from_payload(payload) == () + assert records_from_payload(payload, vendor="future-provider") == () + + def test_mistral_reader_maps_per_model_capabilities_without_provider_inheritance(): records = mistral.records_from_payload( { @@ -571,6 +662,30 @@ def test_copilot_reader_uses_picker_and_nested_supports_shape(): assert dict(record.capability.limits) == {"input_tokens": 64000, "output_tokens": 8192} +def test_copilot_reader_ignores_unverified_support_aliases(): + record = records_from_payload( + { + "data": [ + { + "id": "future-supports-model", + "model_picker_enabled": True, + "capabilities": { + "supports": { + "tools": True, + "reasoning": True, + "structured_outputs": True, + } + }, + } + ] + }, + vendor="copilot", + )[0] + + assert record.capability.family == mc.FAMILY_CHAT + assert record.capability.capabilities == () + + def test_sglang_model_info_maps_native_generation_flags_only(): generation = sglang.records_from_payload( { From d3d4d03c0f21723e96efb4dfe012f1b7fc27132e Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sat, 18 Jul 2026 12:43:05 +0000 Subject: [PATCH 09/14] fix(models): preserve native catalog contracts --- src/model_capability_readers/__init__.py | 6 +- src/model_capability_readers/base.py | 16 +- .../chatgpt_subscription.py | 56 +++++- src/model_capability_readers/cohere.py | 3 +- src/model_capability_readers/copilot.py | 14 +- .../google_ai_studio_mapping.py | 9 +- src/model_capability_readers/huggingface.py | 12 +- src/model_capability_readers/llamacpp.py | 5 +- src/model_capability_readers/ollama.py | 3 +- src/model_capability_readers/sglang.py | 6 +- src/provider_capability_schemas.py | 39 +++- tests/test_provider_capability_schemas.py | 190 +++++++++++++++++- 12 files changed, 333 insertions(+), 26 deletions(-) diff --git a/src/model_capability_readers/__init__.py b/src/model_capability_readers/__init__.py index 15b208163..df5c28581 100644 --- a/src/model_capability_readers/__init__.py +++ b/src/model_capability_readers/__init__.py @@ -160,7 +160,11 @@ def records_from_payload( normalized = tuple(normalized_records) else: normalized_records: list[ModelCapabilityRecord] = [] - for item in shape.items(payload): + catalog_items = shape.items(payload) + select_catalog_items = getattr(reader, "select_catalog_items", None) + if callable(select_catalog_items): + catalog_items = tuple(select_catalog_items(catalog_items)) + for item in catalog_items: item_payload = shape.payload_for_item(payload, item) if shape.item_matches(item): if reader is generic_openai: diff --git a/src/model_capability_readers/base.py b/src/model_capability_readers/base.py index f184ab413..e07b10a55 100644 --- a/src/model_capability_readers/base.py +++ b/src/model_capability_readers/base.py @@ -9,6 +9,7 @@ labels. from __future__ import annotations import hashlib +import math from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from typing import Any, Protocol @@ -139,6 +140,12 @@ def compact_str(value: Any) -> str: return str(value or "").strip() +def identity_str(value: Any) -> str: + """Return a provider identity only when the payload supplied a string.""" + + return value.strip() if isinstance(value, str) else "" + + def _identity_part(value: Any) -> str: text = compact_str(value).lower() out = [] @@ -171,10 +178,7 @@ def stable_model_id_for(vendor: Any, model_id: Any, *, endpoint_id: Any = "", ba def model_id_from(raw: Mapping[str, Any], *keys: str) -> str: for key in keys: - raw_value = raw.get(key) - if not isinstance(raw_value, str): - continue - value = raw_value.strip() + value = identity_str(raw.get(key)) if value: return value.removeprefix("models/") return "" @@ -183,6 +187,10 @@ def model_id_from(raw: Mapping[str, Any], *keys: str) -> str: def int_limit(value: Any) -> int | None: if isinstance(value, bool): return None + if isinstance(value, float) and ( + not math.isfinite(value) or not value.is_integer() + ): + return None try: limit = int(value) except (OverflowError, TypeError, ValueError): diff --git a/src/model_capability_readers/chatgpt_subscription.py b/src/model_capability_readers/chatgpt_subscription.py index a1ddc4399..2b2257049 100644 --- a/src/model_capability_readers/chatgpt_subscription.py +++ b/src/model_capability_readers/chatgpt_subscription.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math from collections.abc import Mapping from typing import Any @@ -12,11 +13,59 @@ from src.model_capability_readers.base import ( as_list, as_mapping, compact_str, + identity_str, stable_model_id_for, ) vendor = VENDOR_CHATGPT_SUBSCRIPTION +_DEFAULT_PRIORITY = 10_000 + + +def _priority_rank(raw: Mapping[str, Any]) -> int | float: + value = raw.get("priority") + if isinstance(value, bool): + return _DEFAULT_PRIORITY + if isinstance(value, int): + return value + if isinstance(value, float) and math.isfinite(value): + return value + return _DEFAULT_PRIORITY + + +def _is_hidden(raw: Mapping[str, Any]) -> bool: + visibility = raw.get("visibility") + return ( + isinstance(visibility, str) + and visibility.strip().lower() in {"hide", "hidden"} + ) + + +def select_catalog_items( + items: tuple[Mapping[str, Any], ...], +) -> tuple[Mapping[str, Any], ...]: + """Apply the provider's visibility, priority, and slug de-duplication.""" + + sortable: list[tuple[int | float, str, int, Mapping[str, Any]]] = [] + passthrough: list[Mapping[str, Any]] = [] + for position, item in enumerate(items): + if _is_hidden(item): + continue + slug = identity_str(item.get("slug")) + if not slug: + passthrough.append(item) + continue + sortable.append((_priority_rank(item), slug, position, item)) + sortable.sort(key=lambda entry: (entry[0], entry[1], entry[2])) + + selected: list[Mapping[str, Any]] = [] + seen: set[str] = set() + for _, slug, _, item in sortable: + if slug not in seen: + selected.append(item) + seen.add(slug) + selected.extend(passthrough) + return tuple(selected) def record_from_model( @@ -25,7 +74,7 @@ def record_from_model( endpoint_id: Any = "", base_url: Any = "", ) -> ModelCapabilityRecord | None: - model_id = compact_str(raw.get("slug")) + model_id = identity_str(raw.get("slug")) if not model_id: return None return ModelCapabilityRecord( @@ -55,7 +104,8 @@ def records_from_payload( values = as_mapping(payload).get("models") return tuple( record - for item in as_list(values) - if isinstance(item, Mapping) + for item in select_catalog_items( + tuple(item for item in as_list(values) if isinstance(item, Mapping)) + ) if (record := record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)) ) diff --git a/src/model_capability_readers/cohere.py b/src/model_capability_readers/cohere.py index e4108fc04..e83b264c7 100644 --- a/src/model_capability_readers/cohere.py +++ b/src/model_capability_readers/cohere.py @@ -19,6 +19,7 @@ from src.model_capability_readers.base import ( build_capability, compact_str, deterministic_controls_from_supported_parameters, + identity_str, int_limit, stable_model_id_for, ) @@ -60,7 +61,7 @@ def record_from_model( endpoint_id: Any = "", base_url: Any = "", ) -> ModelCapabilityRecord | None: - model_id = compact_str(raw.get("name")) + model_id = identity_str(raw.get("name")) if not model_id: return None family = _family(raw) diff --git a/src/model_capability_readers/copilot.py b/src/model_capability_readers/copilot.py index 642cc87ed..6b64b6367 100644 --- a/src/model_capability_readers/copilot.py +++ b/src/model_capability_readers/copilot.py @@ -28,6 +28,18 @@ _SUPPORT_CAPABILITIES = { } +def select_catalog_items( + items: tuple[Mapping[str, Any], ...], +) -> tuple[Mapping[str, Any], ...]: + """Keep picker-enabled models when the catalog advertises any of them.""" + + if any(item.get("model_picker_enabled") is True for item in items): + return tuple( + item for item in items if item.get("model_picker_enabled") is True + ) + return items + + def _supports(raw: Mapping[str, Any]) -> Mapping[str, Any]: return as_mapping(as_mapping(raw.get("capabilities")).get("supports")) @@ -102,7 +114,7 @@ def records_from_payload( base_url: Any = "", ) -> tuple[ModelCapabilityRecord, ...]: records: list[ModelCapabilityRecord] = [] - for item in openai_model_items(payload): + for item in select_catalog_items(openai_model_items(payload)): record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url) if record: records.append(record) diff --git a/src/model_capability_readers/google_ai_studio_mapping.py b/src/model_capability_readers/google_ai_studio_mapping.py index a6f5dec19..ed06016be 100644 --- a/src/model_capability_readers/google_ai_studio_mapping.py +++ b/src/model_capability_readers/google_ai_studio_mapping.py @@ -13,7 +13,12 @@ from collections.abc import Mapping from typing import Any from src import model_capabilities as mc -from src.model_capability_readers.base import as_list, compact_str, int_limit +from src.model_capability_readers.base import ( + as_list, + compact_str, + identity_str, + int_limit, +) METHOD_GENERATE_CONTENT = "generateContent" @@ -55,7 +60,7 @@ MODEL_FIELD_MAP = { def google_model_id(raw: Mapping[str, Any]) -> str: - value = compact_str(raw.get("baseModelId")) or compact_str(raw.get("name")) + value = identity_str(raw.get("baseModelId")) or identity_str(raw.get("name")) return value.removeprefix("models/") diff --git a/src/model_capability_readers/huggingface.py b/src/model_capability_readers/huggingface.py index c260513ea..f6f5ec6fc 100644 --- a/src/model_capability_readers/huggingface.py +++ b/src/model_capability_readers/huggingface.py @@ -11,7 +11,7 @@ from src.model_capability_readers.base import ( VENDOR_HUGGINGFACE, build_capability, compact_str, - openai_model_items, + model_id_from, stable_model_id_for, ) @@ -105,7 +105,7 @@ def record_from_model( endpoint_id: Any = "", base_url: Any = "", ) -> ModelCapabilityRecord | None: - model_id = compact_str(raw.get("modelId") or raw.get("id")) + model_id = model_id_from(raw, "modelId", "id") if not model_id: return None return ModelCapabilityRecord( @@ -136,11 +136,15 @@ def records_from_payload( endpoint_id: Any = "", base_url: Any = "", ) -> tuple[ModelCapabilityRecord, ...]: - if isinstance(payload, Mapping) and (payload.get("modelId") or payload.get("pipeline_tag")): + if isinstance(payload, Mapping) and "pipeline_tag" in payload: record = record_from_model(payload, endpoint_id=endpoint_id, base_url=base_url) return (record,) if record else () + if not isinstance(payload, (list, tuple)): + return () records: list[ModelCapabilityRecord] = [] - for item in openai_model_items(payload): + for item in payload: + if not isinstance(item, Mapping): + continue record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url) if record: records.append(record) diff --git a/src/model_capability_readers/llamacpp.py b/src/model_capability_readers/llamacpp.py index 9c3beb5c0..c5a233b79 100644 --- a/src/model_capability_readers/llamacpp.py +++ b/src/model_capability_readers/llamacpp.py @@ -22,6 +22,7 @@ from src.model_capability_readers.base import ( build_capability, compact_str, deterministic_controls_from_supported_parameters, + identity_str, int_limit, merge_unique, model_id_from, @@ -53,10 +54,10 @@ def _server_model_entries(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any] def _model_id_from_props(payload: Mapping[str, Any]) -> str: payload = as_mapping(payload) - model_alias = compact_str(payload.get("model_alias")) + model_alias = identity_str(payload.get("model_alias")) if model_alias: return model_alias - model_path = compact_str(payload.get("model_path")) + model_path = identity_str(payload.get("model_path")) if model_path: return PurePosixPath(model_path).name return "" diff --git a/src/model_capability_readers/ollama.py b/src/model_capability_readers/ollama.py index e9a1fdfeb..2333b3574 100644 --- a/src/model_capability_readers/ollama.py +++ b/src/model_capability_readers/ollama.py @@ -13,6 +13,7 @@ from src.model_capability_readers.base import ( as_mapping, build_capability, compact_str, + identity_str, int_limit, merge_unique, model_id_from, @@ -116,7 +117,7 @@ def record_from_show_payload( endpoint_id: Any = "", base_url: Any = "", ) -> ModelCapabilityRecord | None: - model_id = compact_str(model_id) or model_id_from(payload, "model", "name") + model_id = identity_str(model_id) or model_id_from(payload, "model", "name") if not model_id: return None capability_values = payload.get("capabilities") diff --git a/src/model_capability_readers/sglang.py b/src/model_capability_readers/sglang.py index be1ce69af..713e3e10b 100644 --- a/src/model_capability_readers/sglang.py +++ b/src/model_capability_readers/sglang.py @@ -14,8 +14,8 @@ from src.model_capability_readers.base import ( VENDOR_SGLANG, as_mapping, build_capability, - compact_str, deterministic_controls_from_supported_parameters, + identity_str, int_limit, openai_model_items, stable_model_id_for, @@ -26,7 +26,9 @@ vendor = VENDOR_SGLANG def _model_id(payload: Mapping[str, Any]) -> str: - value = compact_str(payload.get("served_model_name") or payload.get("model_path")) + value = identity_str(payload.get("served_model_name")) or identity_str( + payload.get("model_path") + ) if not value: return "" return PurePosixPath(value).name if value.startswith("/") else value diff --git a/src/provider_capability_schemas.py b/src/provider_capability_schemas.py index 8250ea4f6..71b86162c 100644 --- a/src/provider_capability_schemas.py +++ b/src/provider_capability_schemas.py @@ -229,6 +229,15 @@ GOOGLE_MODELS_SHAPE = ProviderCatalogShape( item_types=(("supportedGenerationMethods", (list, tuple)),), detection_priority=100, ) +GOOGLE_MODEL_SHAPE = ProviderCatalogShape( + shape_id="google.generative-language.model.v1beta", + provider_id="google", + envelope=ENVELOPE_SINGLE, + identity_paths=("baseModelId", "name"), + required_item_paths=("supportedGenerationMethods",), + item_types=(("supportedGenerationMethods", (list, tuple)),), + detection_priority=100, +) OLLAMA_TAGS_SHAPE = ProviderCatalogShape( shape_id="ollama.tags.v1", provider_id="ollama", @@ -290,6 +299,17 @@ LLAMACPP_PROPS_SHAPE = ProviderCatalogShape( item_types=(("default_generation_settings", (Mapping,)),), detection_priority=100, ) +LLAMACPP_MODELS_SHAPE = ProviderCatalogShape( + shape_id="llamacpp.models.native.v1", + provider_id="llamacpp", + envelope=ENVELOPE_MODELS, + identity_paths=("id", "name", "model"), + required_item_paths=("capabilities",), + item_types=(("capabilities", (list, tuple)),), + # Model/capability fields are not globally provider-specific. Interpret + # them only after explicit llama.cpp endpoint/provider selection. + detection_priority=0, +) MISTRAL_MODELS_SHAPE = ProviderCatalogShape( shape_id="mistral.models.rich.v1", provider_id="mistral", @@ -325,7 +345,9 @@ ANTHROPIC_MODELS_SHAPE = ProviderCatalogShape( identity_paths=("id",), required_item_paths=("type", "display_name", "created_at"), item_values=(("type", ("model",)),), - detection_priority=70, + # These model-resource fields are not globally provider-specific. Require + # explicit Anthropic endpoint/provider context before assigning identity. + detection_priority=0, ) CHATGPT_MODELS_SHAPE = ProviderCatalogShape( shape_id="chatgpt-subscription.codex-models.v1", @@ -376,6 +398,15 @@ HUGGINGFACE_MODEL_SHAPE = ProviderCatalogShape( item_types=(("pipeline_tag", (str,)),), detection_priority=0, ) +HUGGINGFACE_MODELS_LIST_SHAPE = ProviderCatalogShape( + shape_id="huggingface.hub.model-info-list.v1", + provider_id="huggingface", + envelope=ENVELOPE_BARE_LIST, + identity_paths=("modelId", "id"), + required_item_paths=("pipeline_tag",), + item_types=(("pipeline_tag", (str,)),), + detection_priority=0, +) COHERE_MODELS_SHAPE = ProviderCatalogShape( shape_id="cohere.models.rich.v1", provider_id="cohere", @@ -432,7 +463,7 @@ PROVIDER_SCHEMAS = { "google", aliases=("gemini", "google_ai_studio"), hosts=("generativelanguage.googleapis.com",), - shapes=(GOOGLE_MODELS_SHAPE,), + shapes=(GOOGLE_MODELS_SHAPE, GOOGLE_MODEL_SHAPE), ), "anthropic": _provider( "anthropic", @@ -452,7 +483,7 @@ PROVIDER_SCHEMAS = { "llamacpp": _provider( "llamacpp", aliases=("llama.cpp", "llama_cpp", "llama_server"), - shapes=(LLAMACPP_PROPS_SHAPE,), + shapes=(LLAMACPP_PROPS_SHAPE, LLAMACPP_MODELS_SHAPE), ), "mistral": _provider( "mistral", @@ -480,7 +511,7 @@ PROVIDER_SCHEMAS = { "huggingface", aliases=("hf", "hugging_face"), hosts=("huggingface.co",), - shapes=(HUGGINGFACE_MODEL_SHAPE,), + shapes=(HUGGINGFACE_MODEL_SHAPE, HUGGINGFACE_MODELS_LIST_SHAPE), ), "cohere": _provider( "cohere", diff --git a/tests/test_provider_capability_schemas.py b/tests/test_provider_capability_schemas.py index 030c0bc4a..f83c104cd 100644 --- a/tests/test_provider_capability_schemas.py +++ b/tests/test_provider_capability_schemas.py @@ -241,6 +241,7 @@ def test_native_catalog_shapes_resolve_with_required_provider_context(): ) explicit_context_providers = { + "anthropic", "chatgpt_subscription", "cohere", "huggingface", @@ -608,6 +609,139 @@ def test_structured_identity_values_are_not_stringified_into_fallback_records(): assert records_from_payload(payload, vendor="future-provider") == () +def test_native_readers_skip_structured_identity_candidates(): + google_record = records_from_payload( + { + "models": [ + { + "baseModelId": {"nested": "bad"}, + "name": "models/good-google-id", + "supportedGenerationMethods": ["embedContent"], + } + ] + } + )[0] + huggingface_record = records_from_payload( + { + "modelId": {"nested": "bad"}, + "id": "org/good-hf-id", + "pipeline_tag": "text-to-image", + }, + vendor="huggingface", + )[0] + llamacpp_record = records_from_payload( + { + "model_alias": {"nested": "bad"}, + "model_path": "/models/good-llama.gguf", + "default_generation_settings": {}, + "chat_template_caps": {"supports_vision": True}, + }, + vendor="llamacpp", + )[0] + sglang_record = records_from_payload( + { + "served_model_name": {"nested": "bad"}, + "model_path": "/models/good-sglang", + "is_generation": True, + "has_image_understanding": True, + }, + vendor="sglang", + )[0] + + assert google_record.model_id == "good-google-id" + assert huggingface_record.model_id == "org/good-hf-id" + assert llamacpp_record.model_id == "good-llama.gguf" + assert sglang_record.model_id == "good-sglang" + assert chatgpt_subscription.record_from_model({"slug": {"nested": "bad"}}) is None + assert cohere.record_from_model( + {"name": {"nested": "bad"}, "endpoints": ["chat"]} + ) is None + + +def test_native_singleton_and_bare_list_shapes_reach_their_readers(): + google_records = records_from_payload( + { + "name": "models/gemini-embed", + "supportedGenerationMethods": ["embedContent"], + }, + vendor="google", + ) + huggingface_records = records_from_payload( + [{"modelId": "org/model", "pipeline_tag": "text-generation"}], + vendor="huggingface", + ) + llamacpp_records = records_from_payload( + { + "models": [ + { + "id": "served-model", + "capabilities": ["chat", "tools"], + } + ] + }, + vendor="llamacpp", + ) + + assert google_records[0].model_id == "gemini-embed" + assert google_records[0].capability.family == mc.FAMILY_EMBEDDING + assert google_records[0].catalog_shape_id == ( + "google.generative-language.model.v1beta" + ) + assert huggingface_records[0].model_id == "org/model" + assert huggingface_records[0].capability.family == mc.FAMILY_CHAT + assert huggingface_records[0].catalog_shape_id == ( + "huggingface.hub.model-info-list.v1" + ) + assert llamacpp_records[0].model_id == "served-model" + assert llamacpp_records[0].capability.family == mc.FAMILY_CHAT + assert llamacpp_records[0].capability.capabilities == (mc.CAP_TOOL_CALL,) + assert llamacpp_records[0].catalog_shape_id == "llamacpp.models.native.v1" + + +def test_huggingface_openai_serving_envelope_stays_identity_only(): + payload = { + "data": [ + { + "id": "served-model", + "pipeline_tag": "text-to-image", + } + ] + } + + direct = huggingface.records_from_payload(payload) + wrapped = records_from_payload(payload, vendor="huggingface") + + assert direct == () + assert wrapped[0].model_id == "served-model" + assert wrapped[0].capability.family == mc.FAMILY_UNKNOWN + assert wrapped[0].capability.capabilities == () + assert wrapped[0].fallback is True + + +def test_generic_model_resource_fields_do_not_infer_anthropic_identity(): + payload = { + "data": [ + { + "id": "foreign-model", + "type": "model", + "display_name": "Foreign Model", + "created_at": "2026-01-01T00:00:00Z", + "capabilities": {"tools": True}, + } + ] + } + + resolution = pcs.resolve_provider(payload) + records = records_from_payload(payload) + + assert resolution.provider_id == pcs.PROVIDER_UNKNOWN + assert resolution.shape_id == "fallback.models.data.v1" + assert resolution.fallback is True + assert records[0].vendor == pcs.PROVIDER_UNKNOWN + assert records[0].fallback is True + assert records[0].capability.capabilities == () + + def test_mistral_reader_maps_per_model_capabilities_without_provider_inheritance(): records = mistral.records_from_payload( { @@ -686,6 +820,60 @@ def test_copilot_reader_ignores_unverified_support_aliases(): assert record.capability.capabilities == () +def test_copilot_catalog_uses_picker_selection_with_no_picker_fallback(): + def payload(*picker_values): + return { + "data": [ + { + "id": f"model-{index}", + "model_picker_enabled": picker_enabled, + "capabilities": {"supports": {}}, + } + for index, picker_enabled in enumerate(picker_values) + ] + } + + selected = records_from_payload(payload(False, True, False), vendor="copilot") + fallback = records_from_payload(payload(False, False), vendor="copilot") + + assert [record.model_id for record in selected] == ["model-1"] + assert [record.model_id for record in fallback] == ["model-0", "model-1"] + + +def test_chatgpt_catalog_applies_visibility_priority_and_slug_deduplication(): + records = records_from_payload( + { + "models": [ + {"slug": "hidden", "visibility": "hidden", "priority": 0}, + {"slug": "later", "visibility": "list", "priority": 20}, + { + "slug": "duplicate", + "visibility": "list", + "priority": 30, + "title": "lower-precedence", + }, + {"slug": "first", "visibility": "list", "priority": 1}, + { + "slug": "duplicate", + "visibility": "list", + "priority": 5, + "title": "selected", + }, + {"slug": "unranked", "visibility": "list", "priority": float("inf")}, + ] + }, + vendor="chatgpt_subscription", + ) + + assert [record.model_id for record in records] == [ + "first", + "duplicate", + "later", + "unranked", + ] + assert records[1].display_name == "selected" + + def test_sglang_model_info_maps_native_generation_flags_only(): generation = sglang.records_from_payload( { @@ -746,7 +934,7 @@ def test_sglang_openai_catalog_preserves_only_valid_native_context_limit(): ] } )[0] - for value in (0, True, float("inf")) + for value in (0, True, 1.5, float("inf")) ] assert valid.vendor == "sglang" From bf257c133b17bf18168b7462ca55fd22cb8c4b69 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sat, 18 Jul 2026 12:43:11 +0000 Subject: [PATCH 10/14] fix(logging): apply log level to uvicorn entrypoints --- app.py | 2 ++ core/log_safety.py | 12 ++++++++++++ tests/test_log_safety.py | 20 ++++++++++++++++++++ tests/test_model_capability_diagnostics.py | 1 + 4 files changed, 35 insertions(+) diff --git a/app.py b/app.py index bae972452..da511384d 100644 --- a/app.py +++ b/app.py @@ -87,12 +87,14 @@ from core.log_safety import ( CAPABILITY_DIAGNOSTICS_LOGGER, ScopedDiagnosticsFilter, application_log_settings, + configure_uvicorn_log_levels, ) _root_logger = logging.getLogger() _log_level_name = os.getenv("LOG_LEVEL", "INFO").strip().upper() _application_log_level, _capability_debug = application_log_settings(_log_level_name) _root_logger.setLevel(_application_log_level) +configure_uvicorn_log_levels(_application_log_level) logging.getLogger(CAPABILITY_DIAGNOSTICS_LOGGER).setLevel( logging.DEBUG if _capability_debug else logging.NOTSET ) diff --git a/core/log_safety.py b/core/log_safety.py index a314e1e98..b0bab625d 100644 --- a/core/log_safety.py +++ b/core/log_safety.py @@ -15,6 +15,7 @@ from urllib.parse import urlparse, urlunparse CAPABILITY_DIAGNOSTICS_LOGGER = "src.model_capability_readers" +UVICORN_LOGGER_NAMES = ("uvicorn", "uvicorn.error", "uvicorn.access") _LOG_LEVELS = { "DEBUG": logging.DEBUG, @@ -41,6 +42,17 @@ def application_log_settings(value: object) -> tuple[int, bool]: return max(requested, logging.INFO), requested == logging.DEBUG +def configure_uvicorn_log_levels(application_level: int) -> None: + """Apply the mapped app level to Uvicorn's non-propagating loggers. + + External entrypoints configure these loggers before importing ``app`` and + otherwise bypass the root logger's level and scoped diagnostics filter. + """ + + for logger_name in UVICORN_LOGGER_NAMES: + logging.getLogger(logger_name).setLevel(application_level) + + class ScopedDiagnosticsFilter(logging.Filter): """Allow normal application records plus one explicitly scoped DEBUG log.""" diff --git a/tests/test_log_safety.py b/tests/test_log_safety.py index a4de26666..47f5611ab 100644 --- a/tests/test_log_safety.py +++ b/tests/test_log_safety.py @@ -6,6 +6,7 @@ from core.log_safety import ( CAPABILITY_DIAGNOSTICS_LOGGER, ScopedDiagnosticsFilter, application_log_settings, + configure_uvicorn_log_levels, redact_url, ) @@ -65,6 +66,25 @@ def test_application_log_settings_scope_debug_and_fail_closed( ) +def test_configure_uvicorn_log_levels_clamps_non_propagating_loggers(): + logger_names = ("uvicorn", "uvicorn.error", "uvicorn.access") + previous_levels = { + name: logging.getLogger(name).level for name in logger_names + } + try: + for name in logger_names: + logging.getLogger(name).setLevel(logging.DEBUG) + + configure_uvicorn_log_levels(logging.ERROR) + + assert all( + logging.getLogger(name).level == logging.ERROR for name in logger_names + ) + finally: + for name, level in previous_levels.items(): + logging.getLogger(name).setLevel(level) + + def _record(name: str, level: int) -> logging.LogRecord: return logging.LogRecord(name, level, __file__, 1, "message", (), None) diff --git a/tests/test_model_capability_diagnostics.py b/tests/test_model_capability_diagnostics.py index 7b1969430..e6b2c7487 100644 --- a/tests/test_model_capability_diagnostics.py +++ b/tests/test_model_capability_diagnostics.py @@ -62,6 +62,7 @@ def test_web_app_logging_uses_existing_log_level_environment_toggle(): assert 'os.getenv("LOG_LEVEL", "INFO")' in source assert "application_log_settings(_log_level_name)" in source assert "_root_logger.setLevel(_application_log_level)" in source + assert "configure_uvicorn_log_levels(_application_log_level)" in source assert "_console_h.addFilter(_diagnostics_filter)" in source assert "_file_h.addFilter(_diagnostics_filter)" in source assert "log_level=_application_log_level" in source From ad8bb091816f9de8bfc223194304b375497ab86a Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sat, 18 Jul 2026 13:08:24 +0000 Subject: [PATCH 11/14] fix(models): align Google singleton identity --- src/model_capability_readers/google.py | 6 +++++- tests/test_provider_capability_schemas.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/model_capability_readers/google.py b/src/model_capability_readers/google.py index 9edb57bdb..498baddd3 100644 --- a/src/model_capability_readers/google.py +++ b/src/model_capability_readers/google.py @@ -20,7 +20,11 @@ vendor = VENDOR_GOOGLE def _model_items(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]: models = payload.get("models") if isinstance(payload, Mapping) else None - if models is None and isinstance(payload, Mapping) and payload.get("name"): + if ( + models is None + and isinstance(payload, Mapping) + and ai_studio.google_model_id(payload) + ): models = [payload] return tuple(item for item in as_list(models) if isinstance(item, Mapping)) diff --git a/tests/test_provider_capability_schemas.py b/tests/test_provider_capability_schemas.py index f83c104cd..bf814d678 100644 --- a/tests/test_provider_capability_schemas.py +++ b/tests/test_provider_capability_schemas.py @@ -666,6 +666,13 @@ def test_native_singleton_and_bare_list_shapes_reach_their_readers(): }, vendor="google", ) + google_base_id_records = records_from_payload( + { + "baseModelId": "gemini-base-only", + "supportedGenerationMethods": ["embedContent"], + }, + vendor="google", + ) huggingface_records = records_from_payload( [{"modelId": "org/model", "pipeline_tag": "text-generation"}], vendor="huggingface", @@ -687,6 +694,8 @@ def test_native_singleton_and_bare_list_shapes_reach_their_readers(): assert google_records[0].catalog_shape_id == ( "google.generative-language.model.v1beta" ) + assert google_base_id_records[0].model_id == "gemini-base-only" + assert google_base_id_records[0].capability.family == mc.FAMILY_EMBEDDING assert huggingface_records[0].model_id == "org/model" assert huggingface_records[0].capability.family == mc.FAMILY_CHAT assert huggingface_records[0].catalog_shape_id == ( From 2b3ee58af36abb82e5e167eb877adefaa30e1b45 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sat, 18 Jul 2026 13:08:29 +0000 Subject: [PATCH 12/14] fix(logging): preserve level across uvicorn startup --- app.py | 9 +++- core/log_safety.py | 21 +++++++- launcher.py | 12 ++++- tests/test_log_safety.py | 62 +++++++++++++++++++++- tests/test_model_capability_diagnostics.py | 8 ++- 5 files changed, 106 insertions(+), 6 deletions(-) diff --git a/app.py b/app.py index da511384d..a1856da65 100644 --- a/app.py +++ b/app.py @@ -88,6 +88,7 @@ from core.log_safety import ( ScopedDiagnosticsFilter, application_log_settings, configure_uvicorn_log_levels, + uvicorn_log_config, ) _root_logger = logging.getLogger() @@ -1298,4 +1299,10 @@ if __name__ == "__main__": bind_host = os.getenv("APP_BIND", "127.0.0.1") bind_port = int(os.getenv("APP_PORT", "7000")) - uvicorn.run(app, host=bind_host, port=bind_port, log_level=_application_log_level) + uvicorn.run( + app, + host=bind_host, + port=bind_port, + log_level=_application_log_level, + log_config=uvicorn_log_config(_application_log_level), + ) diff --git a/core/log_safety.py b/core/log_safety.py index b0bab625d..4b86b9091 100644 --- a/core/log_safety.py +++ b/core/log_safety.py @@ -9,13 +9,18 @@ also doubles as a sanitizer barrier for CodeQL's clear-text-logging query. from __future__ import annotations +from copy import deepcopy import logging - from urllib.parse import urlparse, urlunparse CAPABILITY_DIAGNOSTICS_LOGGER = "src.model_capability_readers" -UVICORN_LOGGER_NAMES = ("uvicorn", "uvicorn.error", "uvicorn.access") +UVICORN_LOGGER_NAMES = ( + "uvicorn", + "uvicorn.error", + "uvicorn.access", + "uvicorn.asgi", +) _LOG_LEVELS = { "DEBUG": logging.DEBUG, @@ -53,6 +58,18 @@ def configure_uvicorn_log_levels(application_level: int) -> None: logging.getLogger(logger_name).setLevel(application_level) +def uvicorn_log_config(application_level: int) -> dict: + """Return a Uvicorn config that preserves the mapped level on direct runs.""" + + from uvicorn.config import LOGGING_CONFIG + + config = deepcopy(LOGGING_CONFIG) + loggers = config.setdefault("loggers", {}) + for logger_name in UVICORN_LOGGER_NAMES: + loggers.setdefault(logger_name, {})["level"] = application_level + return config + + class ScopedDiagnosticsFilter(logging.Filter): """Allow normal application records plus one explicitly scoped DEBUG log.""" diff --git a/launcher.py b/launcher.py index ba158444f..2bbcc2f92 100644 --- a/launcher.py +++ b/launcher.py @@ -128,9 +128,13 @@ if __name__ == "__main__": import uvicorn # Import the FastAPI app from app.py from app import app + from core.log_safety import application_log_settings, uvicorn_log_config bind_host = os.getenv("APP_BIND", "127.0.0.1") bind_port = int(os.getenv("APP_PORT", "7000")) + application_log_level, _ = application_log_settings( + os.getenv("LOG_LEVEL", "INFO") + ) url = f"http://{bind_host}:{bind_port}" if getattr(sys, 'frozen', False): @@ -139,4 +143,10 @@ if __name__ == "__main__": # Start system tray manager thread threading.Thread(target=setup_system_tray, args=(url,), daemon=True).start() - uvicorn.run(app, host=bind_host, port=bind_port, log_level="info") + uvicorn.run( + app, + host=bind_host, + port=bind_port, + log_level=application_log_level, + log_config=uvicorn_log_config(application_log_level), + ) diff --git a/tests/test_log_safety.py b/tests/test_log_safety.py index 47f5611ab..584afe75a 100644 --- a/tests/test_log_safety.py +++ b/tests/test_log_safety.py @@ -1,13 +1,18 @@ import logging +import subprocess +import sys +from textwrap import dedent import pytest from core.log_safety import ( CAPABILITY_DIAGNOSTICS_LOGGER, ScopedDiagnosticsFilter, + UVICORN_LOGGER_NAMES, application_log_settings, configure_uvicorn_log_levels, redact_url, + uvicorn_log_config, ) @@ -67,7 +72,7 @@ def test_application_log_settings_scope_debug_and_fail_closed( def test_configure_uvicorn_log_levels_clamps_non_propagating_loggers(): - logger_names = ("uvicorn", "uvicorn.error", "uvicorn.access") + logger_names = UVICORN_LOGGER_NAMES previous_levels = { name: logging.getLogger(name).level for name in logger_names } @@ -85,6 +90,61 @@ def test_configure_uvicorn_log_levels_clamps_non_propagating_loggers(): logging.getLogger(name).setLevel(level) +def test_uvicorn_log_config_sets_all_named_loggers_without_mutating_default(): + from uvicorn.config import LOGGING_CONFIG + + configured = uvicorn_log_config(logging.ERROR) + + assert all( + configured["loggers"][name]["level"] == logging.ERROR + for name in UVICORN_LOGGER_NAMES + ) + assert LOGGING_CONFIG["loggers"]["uvicorn"]["level"] == "INFO" + + +def test_uvicorn_levels_hold_across_external_and_direct_config_order(): + result = subprocess.run( + [ + sys.executable, + "-c", + dedent( + """ + import logging + import uvicorn + + from core.log_safety import ( + UVICORN_LOGGER_NAMES, + configure_uvicorn_log_levels, + uvicorn_log_config, + ) + + uvicorn.Config("app:app", log_level="debug") + configure_uvicorn_log_levels(logging.INFO) + assert all( + logging.getLogger(name).level == logging.INFO + for name in UVICORN_LOGGER_NAMES + ) + + uvicorn.Config( + "app:app", + log_level=logging.ERROR, + log_config=uvicorn_log_config(logging.ERROR), + ) + assert all( + logging.getLogger(name).level == logging.ERROR + for name in UVICORN_LOGGER_NAMES + ) + """ + ), + ], + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + + def _record(name: str, level: int) -> logging.LogRecord: return logging.LogRecord(name, level, __file__, 1, "message", (), None) diff --git a/tests/test_model_capability_diagnostics.py b/tests/test_model_capability_diagnostics.py index e6b2c7487..efa086775 100644 --- a/tests/test_model_capability_diagnostics.py +++ b/tests/test_model_capability_diagnostics.py @@ -57,7 +57,9 @@ def test_fallback_debug_log_is_explicit_and_has_no_capability_claims(caplog): def test_web_app_logging_uses_existing_log_level_environment_toggle(): - source = (Path(__file__).resolve().parents[1] / "app.py").read_text(encoding="utf-8") + root = Path(__file__).resolve().parents[1] + source = (root / "app.py").read_text(encoding="utf-8") + launcher_source = (root / "launcher.py").read_text(encoding="utf-8") assert 'os.getenv("LOG_LEVEL", "INFO")' in source assert "application_log_settings(_log_level_name)" in source @@ -66,3 +68,7 @@ def test_web_app_logging_uses_existing_log_level_environment_toggle(): assert "_console_h.addFilter(_diagnostics_filter)" in source assert "_file_h.addFilter(_diagnostics_filter)" in source assert "log_level=_application_log_level" in source + assert "log_config=uvicorn_log_config(_application_log_level)" in source + assert "application_log_settings(" in launcher_source + assert "log_level=application_log_level" in launcher_source + assert "log_config=uvicorn_log_config(application_log_level)" in launcher_source From b91b0b3c3a018dfd5c8306a3a0380326f4b76f0b Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sat, 18 Jul 2026 13:41:07 +0000 Subject: [PATCH 13/14] fix(models): preserve Hugging Face identity-only cards --- src/model_capability_readers/huggingface.py | 2 +- src/provider_capability_schemas.py | 8 ++-- tests/test_provider_capability_schemas.py | 42 +++++++++++++++++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/model_capability_readers/huggingface.py b/src/model_capability_readers/huggingface.py index f6f5ec6fc..bdbfb45ad 100644 --- a/src/model_capability_readers/huggingface.py +++ b/src/model_capability_readers/huggingface.py @@ -136,7 +136,7 @@ def records_from_payload( endpoint_id: Any = "", base_url: Any = "", ) -> tuple[ModelCapabilityRecord, ...]: - if isinstance(payload, Mapping) and "pipeline_tag" in payload: + if isinstance(payload, Mapping): record = record_from_model(payload, endpoint_id=endpoint_id, base_url=base_url) return (record,) if record else () if not isinstance(payload, (list, tuple)): diff --git a/src/provider_capability_schemas.py b/src/provider_capability_schemas.py index 71b86162c..faffb917e 100644 --- a/src/provider_capability_schemas.py +++ b/src/provider_capability_schemas.py @@ -394,8 +394,10 @@ HUGGINGFACE_MODEL_SHAPE = ProviderCatalogShape( provider_id="huggingface", envelope=ENVELOPE_SINGLE, identity_paths=("modelId", "id"), - required_item_paths=("pipeline_tag",), - item_types=(("pipeline_tag", (str,)),), + # Hub ModelInfo exposes pipeline_tag as optional metadata. Provider/host + # context is still required because this shape has priority zero, so an + # identity-only card can stay native without making generic ``id`` payloads + # look like Hugging Face catalogs. detection_priority=0, ) HUGGINGFACE_MODELS_LIST_SHAPE = ProviderCatalogShape( @@ -403,8 +405,6 @@ HUGGINGFACE_MODELS_LIST_SHAPE = ProviderCatalogShape( provider_id="huggingface", envelope=ENVELOPE_BARE_LIST, identity_paths=("modelId", "id"), - required_item_paths=("pipeline_tag",), - item_types=(("pipeline_tag", (str,)),), detection_priority=0, ) COHERE_MODELS_SHAPE = ProviderCatalogShape( diff --git a/tests/test_provider_capability_schemas.py b/tests/test_provider_capability_schemas.py index bf814d678..228f5120f 100644 --- a/tests/test_provider_capability_schemas.py +++ b/tests/test_provider_capability_schemas.py @@ -1001,6 +1001,48 @@ def test_huggingface_reader_maps_provider_specific_pipeline_metadata(): assert record.capability.confidence == mc.CONFIDENCE_REGISTRY +def test_huggingface_optional_pipeline_tag_preserves_identity_only_records(): + cases = ( + ( + {"modelId": "org/no-pipeline-tag"}, + "huggingface.hub.model-info.v1", + ), + ( + {"modelId": "org/null-pipeline-tag", "pipeline_tag": None}, + "huggingface.hub.model-info.v1", + ), + ( + [{"modelId": "org/list-no-pipeline-tag"}], + "huggingface.hub.model-info-list.v1", + ), + ( + [{"modelId": "org/list-null-pipeline-tag", "pipeline_tag": None}], + "huggingface.hub.model-info-list.v1", + ), + ) + + for payload, shape_id in cases: + resolution = pcs.resolve_provider(payload, provider="huggingface") + direct = huggingface.records_from_payload(payload) + wrapped = records_from_payload(payload, vendor="huggingface") + + assert resolution.shape_id == shape_id + assert resolution.fallback is False + assert len(direct) == 1 + assert len(wrapped) == 1 + assert wrapped[0].model_id == direct[0].model_id + assert wrapped[0].capability.family == mc.FAMILY_UNKNOWN + assert wrapped[0].capability.capabilities == () + assert wrapped[0].catalog_shape_id == shape_id + assert wrapped[0].fallback is False + + # An identity-only singleton remains insufficient to infer Hugging Face + # without configured provider or host context. + unscoped = {"modelId": "org/unscoped"} + assert pcs.resolve_provider(unscoped).provider_id == pcs.PROVIDER_UNKNOWN + assert records_from_payload(unscoped) == () + + def test_cohere_reader_maps_only_native_endpoint_and_limit_fields(): chat, ambiguous = cohere.records_from_payload( { From 7a7f939461153fdb8ca185c100c1a5f06f9bdf99 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:03:32 +0000 Subject: [PATCH 14/14] feat(models): normalize runtime context allocation --- src/model_capability_readers/__init__.py | 4 ++ src/model_capability_readers/base.py | 48 ++++++++++++++++++ src/model_capability_readers/ollama.py | 63 ++++++++++++++++++++++++ tests/test_model_capability_readers.py | 62 +++++++++++++++++++++++ 4 files changed, 177 insertions(+) diff --git a/src/model_capability_readers/__init__.py b/src/model_capability_readers/__init__.py index df5c28581..587ec6507 100644 --- a/src/model_capability_readers/__init__.py +++ b/src/model_capability_readers/__init__.py @@ -25,7 +25,9 @@ from src.model_capability_readers import ( ) from src.model_capability_readers.base import ( CANONICAL_MODEL_SHAPE_VERSION, + CANONICAL_RUNTIME_CONTEXT_SHAPE_VERSION, ModelCapabilityRecord, + RuntimeContextAllocationRecord, VENDOR_ANTHROPIC, VENDOR_CEREBRAS, VENDOR_CHATGPT_SUBSCRIPTION, @@ -250,7 +252,9 @@ def records_from_payload( __all__ = [ "ModelCapabilityRecord", + "RuntimeContextAllocationRecord", "CANONICAL_MODEL_SHAPE_VERSION", + "CANONICAL_RUNTIME_CONTEXT_SHAPE_VERSION", "PLACEHOLDER_VENDOR_IDS", "READER_MODULES", "VENDOR_ANTHROPIC", diff --git a/src/model_capability_readers/base.py b/src/model_capability_readers/base.py index e07b10a55..c9f0aae18 100644 --- a/src/model_capability_readers/base.py +++ b/src/model_capability_readers/base.py @@ -46,6 +46,7 @@ VENDOR_ZAI = "zai" VENDOR_UNKNOWN = "unknown" CANONICAL_MODEL_SHAPE_VERSION = 1 +CANONICAL_RUNTIME_CONTEXT_SHAPE_VERSION = 1 @dataclass(frozen=True) @@ -109,6 +110,53 @@ class ModelCapabilityRecord: return data +@dataclass(frozen=True) +class RuntimeContextAllocationRecord: + """Provider-reported context allocated to one currently loaded model. + + This runtime state is deliberately separate from ``ModelCapability.limits``: + a model's maximum, configured ``num_ctx``, and the allocation reported for a + loaded process are different facts with different lifetimes. + """ + + vendor: str + model_id: str + allocated_context_tokens: int + stable_model_id: str = "" + source: str = mc.SOURCE_PROVIDER_READER + confidence: str = mc.CONFIDENCE_PROVIDER_REPORTED + runtime_shape_id: str = "" + raw: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + context_tokens = int_limit(self.allocated_context_tokens) + if context_tokens is None: + raise ValueError("allocated_context_tokens must be a positive integer") + object.__setattr__(self, "allocated_context_tokens", context_tokens) + if not self.stable_model_id: + object.__setattr__(self, "stable_model_id", stable_model_id_for(self.vendor, self.model_id)) + + def to_dict(self, *, include_raw: bool = False) -> dict[str, Any]: + data = { + "schema_version": CANONICAL_RUNTIME_CONTEXT_SHAPE_VERSION, + "provider": self.vendor, + "model": self.model_id, + "stable_id": self.stable_model_id, + "runtime": { + "allocated_context_tokens": self.allocated_context_tokens, + }, + "evidence": { + "source": self.source, + "confidence": self.confidence, + "shape": self.runtime_shape_id, + "scope": "loaded_model", + }, + } + if include_raw: + data["raw"] = dict(self.raw) + return data + + class CapabilityReader(Protocol): vendor: str diff --git a/src/model_capability_readers/ollama.py b/src/model_capability_readers/ollama.py index 2333b3574..2a8c341e4 100644 --- a/src/model_capability_readers/ollama.py +++ b/src/model_capability_readers/ollama.py @@ -8,6 +8,7 @@ from typing import Any from src import model_capabilities as mc from src.model_capability_readers.base import ( ModelCapabilityRecord, + RuntimeContextAllocationRecord, VENDOR_OLLAMA, as_list, as_mapping, @@ -24,6 +25,9 @@ from src.model_capability_readers.base import ( vendor = VENDOR_OLLAMA +OLLAMA_PS_SHAPE_ID = "ollama.ps.v1" + + _CAPABILITY_MAP = { "completion": None, "completions": None, @@ -110,6 +114,65 @@ def _limits_from_show(raw: Mapping[str, Any]) -> dict[str, Any]: return limits +def _runtime_model_key(value: Any) -> str: + model_id = identity_str(value).casefold() + return model_id.removesuffix(":latest") + + +def runtime_context_from_ps_payload( + model_id: str, + payload: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> RuntimeContextAllocationRecord | None: + """Normalize one loaded model's allocation from Ollama ``GET /api/ps``. + + Exact identity is preferred, with only Ollama's implicit ``:latest`` alias + normalized. Conflicting matching rows fail closed instead of choosing an + arbitrary allocation. + """ + + requested_id = identity_str(model_id) + requested_key = _runtime_model_key(requested_id) + if not requested_key: + return None + + matches: list[tuple[int, Mapping[str, Any]]] = [] + for item in as_list(as_mapping(payload).get("models")): + if not isinstance(item, Mapping): + continue + identities = tuple( + identity + for identity in (identity_str(item.get("model")), identity_str(item.get("name"))) + if identity + ) + if not any(_runtime_model_key(identity) == requested_key for identity in identities): + continue + context_tokens = int_limit(item.get("context_length")) + if context_tokens is not None: + matches.append((context_tokens, item)) + + allocations = {context_tokens for context_tokens, _item in matches} + if len(allocations) != 1: + return None + + context_tokens, raw = matches[0] + return RuntimeContextAllocationRecord( + vendor=VENDOR_OLLAMA, + model_id=requested_id, + stable_model_id=stable_model_id_for( + VENDOR_OLLAMA, + requested_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + allocated_context_tokens=context_tokens, + runtime_shape_id=OLLAMA_PS_SHAPE_ID, + raw=raw, + ) + + def record_from_show_payload( model_id: str, payload: Mapping[str, Any], diff --git a/tests/test_model_capability_readers.py b/tests/test_model_capability_readers.py index ca547b15a..e69cca761 100644 --- a/tests/test_model_capability_readers.py +++ b/tests/test_model_capability_readers.py @@ -404,6 +404,68 @@ def test_ollama_reader_uses_generic_model_info_context_length_when_no_num_ctx(): assert dict(record.capability.limits) == {"context_tokens": 32768} +def test_ollama_ps_reader_keeps_loaded_allocation_in_runtime_shape(): + record = ollama.runtime_context_from_ps_payload( + "hf.co/example/Qwen3:Q6_K", + { + "models": [ + { + "name": "hf.co/example/Qwen3:Q6_K", + "model": "hf.co/example/Qwen3:Q6_K", + "context_length": 65536, + "details": {"family": "qwen3"}, + } + ] + }, + endpoint_id="7", + ) + + assert record is not None + assert record.allocated_context_tokens == 65536 + assert record.stable_model_id == "ollama|endpoint:7|hf.co/example/qwen3:q6_k" + serialized = record.to_dict() + assert serialized["runtime"] == {"allocated_context_tokens": 65536} + assert serialized["evidence"] == { + "source": mc.SOURCE_PROVIDER_READER, + "confidence": mc.CONFIDENCE_PROVIDER_REPORTED, + "shape": "ollama.ps.v1", + "scope": "loaded_model", + } + assert "limits" not in serialized + assert "raw" not in serialized + + +def test_ollama_ps_reader_matches_only_exact_or_latest_identity(): + payload = { + "models": [ + {"model": "qwen3:latest", "context_length": 32768}, + {"model": "qwen3:14b", "context_length": 65536}, + ] + } + + record = ollama.runtime_context_from_ps_payload("qwen3", payload) + + assert record is not None + assert record.allocated_context_tokens == 32768 + assert ollama.runtime_context_from_ps_payload("qwen3:8b", payload) is None + + +def test_ollama_ps_reader_fails_closed_for_invalid_or_conflicting_allocation(): + assert ollama.runtime_context_from_ps_payload( + "qwen3", + {"models": [{"model": "qwen3", "context_length": True}]}, + ) is None + assert ollama.runtime_context_from_ps_payload( + "qwen3", + { + "models": [ + {"model": "qwen3", "context_length": 32768}, + {"name": "qwen3:latest", "context_length": 65536}, + ] + }, + ) is None + + def test_lmstudio_reader_uses_native_v1_capabilities_when_present(): records = lmstudio.records_from_payload( {