diff --git a/src/model_behavior_quirks.py b/src/model_behavior_quirks.py new file mode 100644 index 000000000..5eb5011ae --- /dev/null +++ b/src/model_behavior_quirks.py @@ -0,0 +1,297 @@ +"""Canonical, exact-match model/provider behavior observations. + +The registry captures behavior that cannot safely be promoted to a provider- +wide capability. Selectors accept already-structured identity (provider, +model ID/family/version, API dialect, and canonical capabilities); they never +extract those facts from a display name with regexes or substring matching. + +This is shape/evidence data only. Runtime request builders can consume it in a +later integration pass after their endpoint has supplied structured identity. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +from src import model_capabilities as mc +from src import provider_capability_schemas as pcs + + +def _identity(value: Any) -> str: + return str(value or "").strip().lower() + + +def _version(value: Any) -> tuple[int, ...]: + if not isinstance(value, (list, tuple)): + return () + out: list[int] = [] + for part in value: + try: + out.append(int(part)) + except (TypeError, ValueError): + return () + return tuple(out) + + +@dataclass(frozen=True) +class ModelBehaviorSelector: + providers: tuple[str, ...] = () + model_ids: tuple[str, ...] = () + model_families: tuple[str, ...] = () + minimum_model_version: tuple[int, ...] = () + minimum_provider_version: tuple[int, ...] = () + api_dialects: tuple[str, ...] = () + required_capabilities: tuple[str, ...] = () + + def matches( + self, + *, + provider: Any, + model_id: Any = "", + model_family: Any = "", + model_version: Any = (), + provider_version: Any = (), + api_dialect: Any = "", + capabilities: Any = (), + ) -> bool: + provider_id = pcs.normalize_provider_id(provider) + if self.providers and provider_id not in self.providers: + return False + + identity_constraints = bool(self.model_ids or self.model_families) + identity_match = ( + _identity(model_id) in self.model_ids + or _identity(model_family) in self.model_families + ) + if identity_constraints and not identity_match: + return False + + actual_model_version = _version(model_version) + if self.minimum_model_version and ( + not actual_model_version or actual_model_version < self.minimum_model_version + ): + return False + actual_provider_version = _version(provider_version) + if self.minimum_provider_version and ( + not actual_provider_version or actual_provider_version < self.minimum_provider_version + ): + return False + if self.api_dialects and str(api_dialect or "").strip() not in self.api_dialects: + return False + + if isinstance(capabilities, Mapping): + capability_values: Iterable[Any] = ( + key for key, enabled in capabilities.items() if enabled is True + ) + elif isinstance(capabilities, str): + capability_values = (capabilities,) + elif isinstance(capabilities, Iterable): + capability_values = capabilities + else: + capability_values = () + normalized_caps = { + normalized + for value in capability_values + if (normalized := mc.normalize_capability(value)) + } + return set(self.required_capabilities).issubset(normalized_caps) + + +@dataclass(frozen=True) +class ModelBehaviorQuirk: + quirk_id: str + selector: ModelBehaviorSelector + request_omit_paths: tuple[str, ...] = () + request_fixed_values: tuple[tuple[str, Any], ...] = () + required_history_paths: tuple[str, ...] = () + response_reasoning_paths: tuple[str, ...] = () + reasoning_controls: tuple[mc.ReasoningControl, ...] = () + status: str = mc.ASSERTION_CLAIMED + source: str = mc.SOURCE_PROVIDER_DOCS_REGISTRY + confidence: str = mc.CONFIDENCE_REGISTRY + evidence_refs: tuple[str, ...] = () + + def to_dict(self) -> dict[str, Any]: + return { + "quirk_id": self.quirk_id, + "selector": { + "providers": list(self.selector.providers), + "model_ids": list(self.selector.model_ids), + "model_families": list(self.selector.model_families), + "minimum_model_version": list(self.selector.minimum_model_version), + "minimum_provider_version": list(self.selector.minimum_provider_version), + "api_dialects": list(self.selector.api_dialects), + "required_capabilities": list(self.selector.required_capabilities), + }, + "request_omit_paths": list(self.request_omit_paths), + "request_fixed_values": dict(self.request_fixed_values), + "required_history_paths": list(self.required_history_paths), + "response_reasoning_paths": list(self.response_reasoning_paths), + "reasoning_controls": [control.to_dict() for control in self.reasoning_controls], + "status": self.status, + "source": self.source, + "confidence": self.confidence, + "evidence_refs": list(self.evidence_refs), + } + + +MODEL_BEHAVIOR_QUIRKS = ( + ModelBehaviorQuirk( + quirk_id="moonshot.kimi-k2.5-k2.6.provider-fixed-temperature", + selector=ModelBehaviorSelector( + providers=("moonshot",), + model_ids=("kimi-k2.5", "kimi-k2.6"), + model_families=("kimi-k2.5", "kimi-k2.6"), + api_dialects=(pcs.DIALECT_OPENAI_CHAT,), + ), + request_omit_paths=("temperature",), + evidence_refs=( + "github:odysseus-dev/odysseus#3960", + "commit:f5d3e509", + ), + ), + ModelBehaviorQuirk( + quirk_id="moonshot.kimi-k2.5-k2.6.tool-history-reasoning-content", + selector=ModelBehaviorSelector( + providers=("moonshot",), + model_ids=("kimi-k2.5", "kimi-k2.6"), + model_families=("kimi-k2.5", "kimi-k2.6"), + api_dialects=(pcs.DIALECT_OPENAI_CHAT,), + ), + required_history_paths=("messages[assistant+tool_calls].reasoning_content",), + response_reasoning_paths=( + "choices[].message.reasoning_content", + "choices[].delta.reasoning_content", + ), + evidence_refs=( + "github:odysseus-dev/odysseus#3118", + "commit:2e6fff22", + ), + ), + ModelBehaviorQuirk( + quirk_id="anthropic.claude-opus-4.7-plus.omit-sampling-controls", + selector=ModelBehaviorSelector( + providers=("anthropic",), + model_families=("claude-opus",), + minimum_model_version=(4, 7), + api_dialects=(pcs.DIALECT_ANTHROPIC_MESSAGES,), + ), + request_omit_paths=("temperature", "top_p", "top_k"), + evidence_refs=( + "github:odysseus-dev/odysseus#3117", + "commit:4f48cfa9", + ), + ), + ModelBehaviorQuirk( + quirk_id="mistral.reasoning.structured-content", + selector=ModelBehaviorSelector( + providers=("mistral",), + model_families=("magistral", "mistral-small", "mistral-medium"), + api_dialects=(pcs.DIALECT_OPENAI_CHAT,), + required_capabilities=(mc.CAP_REASONING,), + ), + response_reasoning_paths=( + "choices[].message.content[type=thinking].thinking[].text", + "choices[].delta.content[type=thinking].thinking[].text", + ), + reasoning_controls=( + mc.ReasoningControl.build( + mechanism=mc.REASONING_CONTROL_EFFORT, + values=(mc.REASONING_CONTROL_VALUE_ON, mc.REASONING_CONTROL_VALUE_OFF), + native_values=("high", "medium", "low", "none"), + request_path="reasoning_effort", + response_paths=("choices[].message.content[type=thinking]",), + status=mc.ASSERTION_CLAIMED, + source=mc.SOURCE_PROVIDER_DOCS_REGISTRY, + confidence=mc.CONFIDENCE_REGISTRY, + ), + ), + evidence_refs=( + "github:odysseus-dev/odysseus#4698", + "commit:bd9149f7", + "https://docs.mistral.ai/capabilities/reasoning/", + ), + ), + ModelBehaviorQuirk( + quirk_id="ollama.native.reasoning-control", + selector=ModelBehaviorSelector( + providers=("ollama",), + model_families=("qwen3", "deepseek-v3.1", "deepseek-r1"), + api_dialects=(pcs.DIALECT_OLLAMA_NATIVE,), + required_capabilities=(mc.CAP_REASONING,), + ), + response_reasoning_paths=("message.thinking", "thinking"), + reasoning_controls=( + mc.ReasoningControl.build( + mechanism=mc.REASONING_CONTROL_NATIVE_BOOL, + values=(mc.REASONING_CONTROL_VALUE_ON, mc.REASONING_CONTROL_VALUE_OFF), + native_values=(True, False), + request_path="think", + response_paths=("message.thinking", "thinking"), + status=mc.ASSERTION_CLAIMED, + source=mc.SOURCE_PROVIDER_DOCS_REGISTRY, + confidence=mc.CONFIDENCE_REGISTRY, + ), + ), + evidence_refs=( + "https://docs.ollama.com/capabilities/thinking", + "github:odysseus-dev/odysseus#3031", + ), + ), + ModelBehaviorQuirk( + quirk_id="ollama.native.gpt-oss-reasoning-level", + selector=ModelBehaviorSelector( + providers=("ollama",), + model_families=("gpt-oss", "gptoss"), + api_dialects=(pcs.DIALECT_OLLAMA_NATIVE,), + required_capabilities=(mc.CAP_REASONING,), + ), + response_reasoning_paths=("message.thinking", "thinking"), + reasoning_controls=( + mc.ReasoningControl.build( + mechanism=mc.REASONING_CONTROL_EFFORT, + values=(mc.REASONING_CONTROL_VALUE_ON,), + native_values=("low", "medium", "high"), + request_path="think", + response_paths=("message.thinking", "thinking"), + status=mc.ASSERTION_CLAIMED, + source=mc.SOURCE_PROVIDER_DOCS_REGISTRY, + confidence=mc.CONFIDENCE_REGISTRY, + ), + ), + evidence_refs=("https://docs.ollama.com/capabilities/thinking",), + ), + ModelBehaviorQuirk( + quirk_id="ollama.openai-compat.0.20.6-reasoning-disable", + selector=ModelBehaviorSelector( + providers=("ollama",), + model_families=("qwen3.5",), + minimum_provider_version=(0, 20, 6), + api_dialects=(pcs.DIALECT_OPENAI_CHAT,), + required_capabilities=(mc.CAP_REASONING,), + ), + request_fixed_values=(("reasoning_effort", "none"),), + status=mc.ASSERTION_CLAIMED, + source=mc.SOURCE_HEURISTIC, + confidence=mc.CONFIDENCE_HEURISTIC, + evidence_refs=("github:odysseus-dev/odysseus#5503",), + ), +) + + +def matching_quirks(**identity: Any) -> tuple[ModelBehaviorQuirk, ...]: + return tuple( + quirk + for quirk in MODEL_BEHAVIOR_QUIRKS + if quirk.selector.matches(**identity) + ) + + +__all__ = [ + "MODEL_BEHAVIOR_QUIRKS", + "ModelBehaviorQuirk", + "ModelBehaviorSelector", + "matching_quirks", +] diff --git a/src/model_capabilities.py b/src/model_capabilities.py index d5b3d8507..2929d68ad 100644 --- a/src/model_capabilities.py +++ b/src/model_capabilities.py @@ -657,6 +657,106 @@ class DeterministicControl: } +@dataclass(frozen=True) +class ReasoningControl: + """A provider/model-supported request mechanism for reasoning. + + This is intentionally separate from :class:`DeterministicControl` and + from the user's on/off/auto preference. The mechanism records the native + request shape that a later resolver may choose after provider and model + evidence have been reconciled. + """ + + mechanism: str = "" + values: tuple[str, ...] = () + native_values: tuple[Any, ...] = () + request_path: str = "" + response_paths: tuple[str, ...] = () + status: str = ASSERTION_UNKNOWN + source: str = SOURCE_UNKNOWN + confidence: str = CONFIDENCE_UNKNOWN + evidence: tuple[tuple[str, Any], ...] = () + tested_at: str = "" + + @classmethod + def build( + cls, + *, + mechanism: Any, + values: Any = None, + native_values: Any = None, + request_path: Any = "", + response_paths: Any = None, + status: Any = ASSERTION_UNKNOWN, + source: Any = SOURCE_UNKNOWN, + confidence: Any = CONFIDENCE_UNKNOWN, + evidence: Mapping[str, Any] | None = None, + tested_at: Any = "", + ) -> "ReasoningControl": + normalized_mechanism = normalize_reasoning_control_mechanism(mechanism) + normalized_status = normalize_assertion_status(status) + if not normalized_mechanism: + normalized_status = ASSERTION_UNKNOWN + if isinstance(response_paths, str): + response_paths = (response_paths,) + elif isinstance(response_paths, Mapping) or not isinstance(response_paths, Iterable): + response_paths = () + if native_values is None: + native_values = () + elif ( + isinstance(native_values, (str, Mapping)) + or not isinstance(native_values, Iterable) + ): + native_values = (native_values,) + return cls( + mechanism=normalized_mechanism, + values=_normalize_tokens(values, normalize_reasoning_control_value), + native_values=tuple(native_values), + request_path=str(request_path or "").strip(), + response_paths=tuple( + str(path or "").strip() + for path in response_paths + if str(path or "").strip() + ), + status=normalized_status, + source=normalize_source(source), + confidence=normalize_confidence(confidence), + evidence=_normalize_limits(evidence), + tested_at=str(tested_at or "").strip(), + ) + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "ReasoningControl": + if not isinstance(value, Mapping): + return cls.build(mechanism="") + return cls.build( + mechanism=value.get("mechanism"), + values=value.get("values"), + native_values=value.get("native_values"), + request_path=value.get("request_path"), + response_paths=value.get("response_paths"), + status=value.get("status"), + source=value.get("source"), + confidence=value.get("confidence"), + evidence=value.get("evidence"), + tested_at=value.get("tested_at"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "mechanism": self.mechanism, + "values": list(self.values), + "native_values": list(self.native_values), + "request_path": self.request_path, + "response_paths": list(self.response_paths), + "status": self.status, + "source": self.source, + "confidence": self.confidence, + "evidence": dict(self.evidence), + "tested_at": self.tested_at, + } + + @dataclass(frozen=True) class CapabilityProbeResult: provider: str diff --git a/src/model_capability_readers/__init__.py b/src/model_capability_readers/__init__.py index 74448281f..a9425c524 100644 --- a/src/model_capability_readers/__init__.py +++ b/src/model_capability_readers/__init__.py @@ -2,24 +2,54 @@ from __future__ import annotations -from collections.abc import Mapping +from dataclasses import replace from typing import Any -from src.model_capability_readers import generic_openai, google, llamacpp, lmstudio, ollama, openai, openrouter +from src import provider_capability_schemas as pcs +from src.model_capability_readers import ( + anthropic, + chatgpt_subscription, + cohere, + copilot, + generic_openai, + google, + huggingface, + llamacpp, + lmstudio, + mistral, + ollama, + openai, + openrouter, + sglang, +) from src.model_capability_readers.base import ( ModelCapabilityRecord, VENDOR_ANTHROPIC, + VENDOR_CEREBRAS, + VENDOR_CHATGPT_SUBSCRIPTION, + VENDOR_COHERE, + VENDOR_COPILOT, + VENDOR_DEEPSEEK, + VENDOR_FIREWORKS, VENDOR_GENERIC_OPENAI, VENDOR_GOOGLE, + VENDOR_GROQ, VENDOR_HUGGINGFACE, VENDOR_LLAMACPP, VENDOR_LMSTUDIO, + VENDOR_MINIMAX, + VENDOR_MISTRAL, + VENDOR_MOONSHOT, + VENDOR_NVIDIA, VENDOR_OLLAMA, VENDOR_OPENAI, VENDOR_OPENROUTER, VENDOR_SGLANG, + VENDOR_TOGETHER, VENDOR_UNKNOWN, VENDOR_VLLM, + VENDOR_XAI, + VENDOR_ZAI, detect_vendor, stable_model_id_for, ) @@ -30,46 +60,69 @@ READER_MODULES = { VENDOR_OPENAI: openai, VENDOR_OPENROUTER: openrouter, VENDOR_GOOGLE: google, + VENDOR_ANTHROPIC: anthropic, VENDOR_LLAMACPP: llamacpp, VENDOR_OLLAMA: ollama, VENDOR_LMSTUDIO: lmstudio, + VENDOR_MISTRAL: mistral, + VENDOR_COPILOT: copilot, + VENDOR_CHATGPT_SUBSCRIPTION: chatgpt_subscription, + VENDOR_COHERE: cohere, + VENDOR_SGLANG: sglang, + VENDOR_HUGGINGFACE: huggingface, } PLACEHOLDER_VENDOR_IDS = frozenset( { - VENDOR_ANTHROPIC, - VENDOR_HUGGINGFACE, - VENDOR_SGLANG, VENDOR_VLLM, } ) def reader_for_vendor(vendor: Any): - vendor_id = str(vendor or "").strip().lower().replace("-", "_") + vendor_id = pcs.normalize_provider_id(vendor) return READER_MODULES.get(vendor_id, generic_openai) def records_from_payload( - payload: Mapping[str, Any], + payload: Any, *, vendor: str | None = None, base_url: str = "", endpoint_kind: str = "", endpoint_id: str = "", ) -> tuple[ModelCapabilityRecord, ...]: - vendor_id = vendor or detect_vendor(base_url, endpoint_kind) + resolution = pcs.resolve_provider( + payload, + provider=vendor, + base_url=base_url, + endpoint_kind=endpoint_kind, + ) + vendor_id = resolution.provider_id + if vendor_id == pcs.PROVIDER_UNKNOWN: + vendor_id = detect_vendor(base_url, endpoint_kind) reader = reader_for_vendor(vendor_id) if reader is generic_openai: record_vendor = vendor_id if vendor_id not in {VENDOR_UNKNOWN, ""} else VENDOR_GENERIC_OPENAI - return reader.records_from_payload( + records = reader.records_from_payload( payload, vendor_id=record_vendor, endpoint_id=endpoint_id, base_url=base_url, ) - return reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url) + else: + records = reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url) + shape_id = resolution.catalog_shape.shape_id if resolution.catalog_shape else "" + return tuple( + replace( + record, + provider_schema_id=resolution.schema.provider_id, + catalog_shape_id=shape_id, + provider_resolution=resolution.stage, + ) + for record in records + ) __all__ = [ @@ -77,17 +130,31 @@ __all__ = [ "PLACEHOLDER_VENDOR_IDS", "READER_MODULES", "VENDOR_ANTHROPIC", + "VENDOR_CEREBRAS", + "VENDOR_CHATGPT_SUBSCRIPTION", + "VENDOR_COHERE", + "VENDOR_COPILOT", + "VENDOR_DEEPSEEK", + "VENDOR_FIREWORKS", "VENDOR_GENERIC_OPENAI", "VENDOR_GOOGLE", + "VENDOR_GROQ", "VENDOR_HUGGINGFACE", "VENDOR_LLAMACPP", "VENDOR_LMSTUDIO", + "VENDOR_MINIMAX", + "VENDOR_MISTRAL", + "VENDOR_MOONSHOT", + "VENDOR_NVIDIA", "VENDOR_OLLAMA", "VENDOR_OPENAI", "VENDOR_OPENROUTER", "VENDOR_SGLANG", + "VENDOR_TOGETHER", "VENDOR_UNKNOWN", "VENDOR_VLLM", + "VENDOR_XAI", + "VENDOR_ZAI", "detect_vendor", "reader_for_vendor", "records_from_payload", diff --git a/src/model_capability_readers/anthropic.py b/src/model_capability_readers/anthropic.py new file mode 100644 index 000000000..4c2376d52 --- /dev/null +++ b/src/model_capability_readers/anthropic.py @@ -0,0 +1,63 @@ +"""Anthropic Models API identity reader. + +The current Model resource is availability/identity metadata, not an explicit +per-model capability card, so records stay unknown. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_ANTHROPIC, + compact_str, + model_id_from, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_ANTHROPIC + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = model_id_from(raw, "id") + if not model_id: + return None + return ModelCapabilityRecord( + vendor=VENDOR_ANTHROPIC, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_ANTHROPIC, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=compact_str(raw.get("display_name")) or model_id, + capability=mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ), + raw=raw, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + return tuple( + record + for item in openai_model_items(payload) + if (record := record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)) + ) diff --git a/src/model_capability_readers/base.py b/src/model_capability_readers/base.py index ee17650a6..1aa2e6adc 100644 --- a/src/model_capability_readers/base.py +++ b/src/model_capability_readers/base.py @@ -28,6 +28,20 @@ VENDOR_LLAMACPP = "llamacpp" VENDOR_VLLM = "vllm" VENDOR_SGLANG = "sglang" VENDOR_HUGGINGFACE = "huggingface" +VENDOR_MISTRAL = "mistral" +VENDOR_COPILOT = "copilot" +VENDOR_CHATGPT_SUBSCRIPTION = "chatgpt_subscription" +VENDOR_COHERE = "cohere" +VENDOR_MINIMAX = "minimax" +VENDOR_MOONSHOT = "moonshot" +VENDOR_GROQ = "groq" +VENDOR_NVIDIA = "nvidia" +VENDOR_CEREBRAS = "cerebras" +VENDOR_DEEPSEEK = "deepseek" +VENDOR_TOGETHER = "together" +VENDOR_FIREWORKS = "fireworks" +VENDOR_XAI = "xai" +VENDOR_ZAI = "zai" VENDOR_UNKNOWN = "unknown" @@ -40,6 +54,14 @@ class ModelCapabilityRecord: stable_model_id: str = "" capability_assertions: tuple[mc.CapabilityAssertion, ...] = () deterministic_controls: tuple[mc.DeterministicControl, ...] = () + reasoning_controls: tuple[mc.ReasoningControl, ...] = () + model_family: str = "" + model_version: tuple[int, ...] = () + provider_version: tuple[int, ...] = () + api_dialect: str = "" + provider_schema_id: str = "" + catalog_shape_id: str = "" + provider_resolution: str = "" raw: Mapping[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: @@ -66,6 +88,14 @@ class ModelCapabilityRecord: "capability": self.capability.to_dict(), "capability_assertions": [assertion.to_dict() for assertion in self.capability_assertions], "deterministic_controls": [control.to_dict() for control in self.deterministic_controls], + "reasoning_controls": [control.to_dict() for control in self.reasoning_controls], + "model_family": self.model_family, + "model_version": list(self.model_version), + "provider_version": list(self.provider_version), + "api_dialect": self.api_dialect, + "provider_schema_id": self.provider_schema_id, + "catalog_shape_id": self.catalog_shape_id, + "provider_resolution": self.provider_resolution, } if include_raw: data["raw"] = dict(self.raw) @@ -77,7 +107,7 @@ class CapabilityReader(Protocol): def records_from_payload( self, - payload: Mapping[str, Any], + payload: Any, *, endpoint_id: Any = "", base_url: Any = "", @@ -168,11 +198,14 @@ def deterministic_controls_from_supported_parameters(values: Any) -> tuple[mc.De ) -def openai_model_items(payload: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]: - payload = as_mapping(payload) - data = payload.get("data") - if data is None: - data = payload.get("models") +def openai_model_items(payload: Any) -> tuple[Mapping[str, Any], ...]: + if isinstance(payload, (list, tuple)): + data = payload + else: + payload = as_mapping(payload) + data = payload.get("data") + if data is None: + data = payload.get("models") return tuple(item for item in as_list(data) if isinstance(item, Mapping)) @@ -254,6 +287,7 @@ def build_capability( output_modalities: Iterable[str] = (), capabilities: Iterable[str] = (), limits: Mapping[str, Any] | None = None, + source: str = mc.SOURCE_PROVIDER_READER, confidence: str = mc.CONFIDENCE_PROVIDER_REPORTED, ) -> mc.ModelCapability: return mc.ModelCapability.build( @@ -263,49 +297,22 @@ def build_capability( output_modalities=tuple(output_modalities), capabilities=tuple(capabilities), limits=limits, - source=mc.SOURCE_PROVIDER_READER, + source=source, confidence=confidence, ) def detect_vendor(base_url: Any = "", endpoint_kind: Any = "") -> str: - kind = compact_str(endpoint_kind).lower().replace("-", "_") - kind_map = { - "openai": VENDOR_OPENAI, - "openrouter": VENDOR_OPENROUTER, - "google": VENDOR_GOOGLE, - "gemini": VENDOR_GOOGLE, - "anthropic": VENDOR_ANTHROPIC, - "ollama": VENDOR_OLLAMA, - "lmstudio": VENDOR_LMSTUDIO, - "lm_studio": VENDOR_LMSTUDIO, - "llamacpp": VENDOR_LLAMACPP, - "llama_cpp": VENDOR_LLAMACPP, - "vllm": VENDOR_VLLM, - "sglang": VENDOR_SGLANG, - "huggingface": VENDOR_HUGGINGFACE, - "hf": VENDOR_HUGGINGFACE, - } - if kind in kind_map: - return kind_map[kind] + # Import lazily to keep the reader primitives independent of registry load + # order. Exact endpoint kind and host identity are authoritative enough + # for provider selection; default ports are not. + from src import provider_capability_schemas as pcs + resolution = pcs.resolve_provider( + endpoint_kind=endpoint_kind, + base_url=base_url, + ) + if resolution.provider_id != pcs.PROVIDER_UNKNOWN: + return resolution.provider_id parsed = urlparse(compact_str(base_url)) - host = (parsed.hostname or "").lower() - port = parsed.port - if host.endswith("openrouter.ai"): - return VENDOR_OPENROUTER - if host.endswith("openai.com"): - return VENDOR_OPENAI - if host.endswith("anthropic.com"): - return VENDOR_ANTHROPIC - if host.endswith("googleapis.com"): - return VENDOR_GOOGLE - if host.endswith("ollama.com") or port == 11434: - return VENDOR_OLLAMA - if port == 1234: - return VENDOR_LMSTUDIO - if port == 8000: - return VENDOR_VLLM - if port == 30000: - return VENDOR_SGLANG - return VENDOR_GENERIC_OPENAI if host else VENDOR_UNKNOWN + return VENDOR_GENERIC_OPENAI if parsed.hostname else VENDOR_UNKNOWN diff --git a/src/model_capability_readers/chatgpt_subscription.py b/src/model_capability_readers/chatgpt_subscription.py new file mode 100644 index 000000000..81400f1b9 --- /dev/null +++ b/src/model_capability_readers/chatgpt_subscription.py @@ -0,0 +1,62 @@ +"""ChatGPT Subscription Codex model-list identity reader.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_CHATGPT_SUBSCRIPTION, + as_list, + as_mapping, + compact_str, + stable_model_id_for, +) + + +vendor = VENDOR_CHATGPT_SUBSCRIPTION + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = compact_str(raw.get("slug")) + if not model_id: + return None + return ModelCapabilityRecord( + vendor=VENDOR_CHATGPT_SUBSCRIPTION, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_CHATGPT_SUBSCRIPTION, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=compact_str(raw.get("display_name") or raw.get("title")) or model_id, + capability=mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ), + model_family=compact_str(raw.get("family")), + raw=raw, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + values = as_mapping(payload).get("models") + return tuple( + record + for item in as_list(values) + if isinstance(item, Mapping) + if (record := record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)) + ) diff --git a/src/model_capability_readers/cohere.py b/src/model_capability_readers/cohere.py new file mode 100644 index 000000000..e4108fc04 --- /dev/null +++ b/src/model_capability_readers/cohere.py @@ -0,0 +1,110 @@ +"""Cohere native model-catalog capability reader. + +The `/v1/models` resource reports endpoint compatibility and context size per +model. It does not prove provider-wide chat/tool support for every model, so +the reader maps only those exact model-card fields. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_COHERE, + as_list, + as_mapping, + build_capability, + compact_str, + deterministic_controls_from_supported_parameters, + int_limit, + stable_model_id_for, +) + + +vendor = VENDOR_COHERE + +_ENDPOINT_FAMILIES = { + "chat": mc.FAMILY_CHAT, + "generate": mc.FAMILY_CHAT, + "embed": mc.FAMILY_EMBEDDING, + "rerank": mc.FAMILY_RERANK, + "classify": mc.FAMILY_CLASSIFICATION, +} + + +def _family(raw: Mapping[str, Any]) -> str: + families = { + family + for value in as_list(raw.get("endpoints")) + if (family := _ENDPOINT_FAMILIES.get(compact_str(value).lower())) + } + return next(iter(families)) if len(families) == 1 else mc.FAMILY_UNKNOWN + + +def _modalities(family: str) -> tuple[tuple[str, ...], tuple[str, ...]]: + if family == mc.FAMILY_CHAT: + return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,) + if family == mc.FAMILY_EMBEDDING: + return (mc.MODALITY_TEXT,), (mc.MODALITY_EMBEDDING,) + if family in {mc.FAMILY_RERANK, mc.FAMILY_CLASSIFICATION}: + return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,) + return (), () + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = compact_str(raw.get("name")) + if not model_id: + return None + family = _family(raw) + inputs, outputs = _modalities(family) + context_tokens = int_limit(raw.get("context_length")) + limits = {"context_tokens": context_tokens} if context_tokens else {} + sampling_defaults = as_mapping(raw.get("sampling_defaults")) + sampling_controls = ( + "top_p" if key == "p" else "top_k" if key == "k" else key + for key in sampling_defaults + ) + return ModelCapabilityRecord( + vendor=VENDOR_COHERE, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_COHERE, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=model_id, + capability=build_capability( + family=family, + input_modalities=inputs, + output_modalities=outputs, + limits=limits, + ), + deterministic_controls=deterministic_controls_from_supported_parameters( + sampling_controls + ), + raw=raw, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + values = as_mapping(payload).get("models") + return tuple( + record + for item in as_list(values) + if isinstance(item, Mapping) + if (record := record_from_model(item, endpoint_id=endpoint_id, base_url=base_url)) + ) diff --git a/src/model_capability_readers/copilot.py b/src/model_capability_readers/copilot.py new file mode 100644 index 000000000..99920d0a0 --- /dev/null +++ b/src/model_capability_readers/copilot.py @@ -0,0 +1,113 @@ +"""GitHub Copilot model-catalog capability reader.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_COPILOT, + as_mapping, + build_capability, + compact_str, + int_limit, + merge_unique, + model_id_from, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_COPILOT + +_SUPPORT_CAPABILITIES = { + "tool_calls": mc.CAP_TOOL_CALL, + "tools": mc.CAP_TOOL_CALL, + "vision": mc.CAP_VISION, + "reasoning": mc.CAP_REASONING, + "structured_outputs": mc.CAP_STRUCTURED_OUTPUT, +} + + +def _supports(raw: Mapping[str, Any]) -> Mapping[str, Any]: + return as_mapping(as_mapping(raw.get("capabilities")).get("supports")) + + +def _limits(raw: Mapping[str, Any]) -> dict[str, int]: + payload = as_mapping(raw.get("limits")) + out: dict[str, int] = {} + for keys, target in ( + (("max_prompt_tokens", "input_tokens"), "input_tokens"), + (("max_output_tokens", "output_tokens"), "output_tokens"), + (("max_context_tokens", "context_window"), "context_tokens"), + ): + for key in keys: + value = int_limit(payload.get(key)) or int_limit(raw.get(key)) + if value: + out[target] = value + break + return out + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = model_id_from(raw, "id") + if not model_id: + return None + supports = _supports(raw) + capabilities = merge_unique( + _SUPPORT_CAPABILITIES[key] + for key, enabled in supports.items() + if enabled is True and key in _SUPPORT_CAPABILITIES + ) + picker_enabled = raw.get("model_picker_enabled") is True + if picker_enabled or capabilities: + inputs = [mc.MODALITY_TEXT] + if mc.CAP_VISION in capabilities: + inputs.append(mc.MODALITY_IMAGE) + capability = build_capability( + family=mc.FAMILY_CHAT, + input_modalities=inputs, + output_modalities=(mc.MODALITY_TEXT,), + capabilities=capabilities, + limits=_limits(raw), + ) + else: + capability = mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ) + return ModelCapabilityRecord( + vendor=VENDOR_COPILOT, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_COPILOT, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=compact_str(raw.get("name")) or model_id, + capability=capability, + model_family=compact_str(raw.get("family")), + raw=raw, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + records: list[ModelCapabilityRecord] = [] + for item in openai_model_items(payload): + record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url) + if record: + records.append(record) + return tuple(records) diff --git a/src/model_capability_readers/generic_openai.py b/src/model_capability_readers/generic_openai.py index edff3ad33..5759405b2 100644 --- a/src/model_capability_readers/generic_openai.py +++ b/src/model_capability_readers/generic_openai.py @@ -1,4 +1,10 @@ -"""Reader for bare OpenAI-compatible model-list payloads.""" +"""General structural reader for OpenAI-compatible model-list payloads. + +Identity-only model cards remain unknown. Rich records are promoted only from +recognized explicit fields (modalities, task/type, capability booleans, +supported parameters, and numeric limits). Names and descriptions are never +parsed for capability hints. +""" from __future__ import annotations @@ -9,9 +15,18 @@ from src import model_capabilities as mc from src.model_capability_readers.base import ( ModelCapabilityRecord, VENDOR_GENERIC_OPENAI, + as_list, + as_mapping, + build_capability, compact_str, + deterministic_controls_from_supported_parameters, + family_from_modalities, + int_limit, + merge_unique, model_id_from, + modalities_from_value, openai_model_items, + split_modality_arrow, stable_model_id_for, ) @@ -19,6 +34,191 @@ from src.model_capability_readers.base import ( vendor = VENDOR_GENERIC_OPENAI +_TYPE_FAMILIES = { + "llm": mc.FAMILY_CHAT, + "chat": mc.FAMILY_CHAT, + "chat_completion": mc.FAMILY_CHAT, + "text_generation": mc.FAMILY_CHAT, + "causal_lm": mc.FAMILY_CHAT, + "image_text_to_text": mc.FAMILY_CHAT, + "image_question_answering": mc.FAMILY_CHAT, + "embedding": mc.FAMILY_EMBEDDING, + "embeddings": mc.FAMILY_EMBEDDING, + "text_embedding": mc.FAMILY_EMBEDDING, + "feature_extraction": mc.FAMILY_EMBEDDING, + "text_to_image": mc.FAMILY_IMAGE, + "image_to_image": mc.FAMILY_IMAGE, + "text_to_video": mc.FAMILY_VIDEO, + "automatic_speech_recognition": mc.FAMILY_AUDIO, + "text_to_speech": mc.FAMILY_AUDIO, + "rerank": mc.FAMILY_RERANK, + "reranking": mc.FAMILY_RERANK, + "classification": mc.FAMILY_CLASSIFICATION, + "text_classification": mc.FAMILY_CLASSIFICATION, + "moderation": mc.FAMILY_MODERATION, +} + +_PARAMETER_CAPABILITIES = { + "tools": mc.CAP_TOOL_CALL, + "tool_choice": mc.CAP_TOOL_CALL, + "parallel_tool_calls": mc.CAP_TOOL_CALL, + "function_calling": mc.CAP_TOOL_CALL, + "response_format": mc.CAP_JSON_MODE, + "structured_output": mc.CAP_STRUCTURED_OUTPUT, + "structured_outputs": mc.CAP_STRUCTURED_OUTPUT, + "json_schema": mc.CAP_STRUCTURED_OUTPUT, + "reasoning": mc.CAP_REASONING, + "reasoning_effort": mc.CAP_REASONING, + "include_reasoning": mc.CAP_REASONING, + "web_search": mc.CAP_WEB_SEARCH, + "web_search_options": mc.CAP_WEB_SEARCH, +} + + +def _shape_token(value: Any) -> str: + return compact_str(value).lower().replace("-", "_").replace(" ", "_") + + +def _family_from_explicit_fields(raw: Mapping[str, Any]) -> str: + for key in ("type", "model_type", "task", "pipeline_tag"): + family = _TYPE_FAMILIES.get(_shape_token(raw.get(key))) + if family: + return family + return mc.FAMILY_UNKNOWN + + +def _modalities(raw: Mapping[str, Any]) -> tuple[tuple[str, ...], tuple[str, ...]]: + architecture = as_mapping(raw.get("architecture")) + input_modalities = modalities_from_value( + raw.get("input_modalities") or architecture.get("input_modalities") + ) + output_modalities = modalities_from_value( + raw.get("output_modalities") or architecture.get("output_modalities") + ) + if not input_modalities or not output_modalities: + arrow_input, arrow_output = split_modality_arrow( + raw.get("modality") or architecture.get("modality") + ) + input_modalities = input_modalities or arrow_input + output_modalities = output_modalities or arrow_output + return input_modalities, output_modalities + + +def _capabilities_from_modalities( + input_modalities: tuple[str, ...], + output_modalities: tuple[str, ...], +) -> tuple[str, ...]: + input_set = set(input_modalities) + output_set = set(output_modalities) + out: list[str] = [] + if mc.MODALITY_IMAGE in input_set and mc.MODALITY_TEXT in output_set: + out.append(mc.CAP_VISION) + if mc.MODALITY_FILE in input_set: + out.append(mc.CAP_FILES) + if mc.MODALITY_PDF in input_set: + out.append(mc.CAP_PDF) + if mc.MODALITY_AUDIO in input_set: + out.append(mc.CAP_AUDIO_INPUT) + if mc.MODALITY_AUDIO in output_set: + out.append(mc.CAP_AUDIO_OUTPUT) + if mc.MODALITY_IMAGE in output_set: + out.append(mc.CAP_IMAGE_GENERATION) + if mc.MODALITY_IMAGE in input_set: + out.append(mc.CAP_IMAGE_EDITING) + if mc.MODALITY_VIDEO in output_set: + out.append(mc.CAP_VIDEO_GENERATION) + return tuple(out) + + +def _explicit_capabilities(raw: Mapping[str, Any]) -> tuple[str, ...]: + values: list[Any] = [] + payload = raw.get("capabilities") + if isinstance(payload, Mapping): + supports = payload.get("supports") + if isinstance(supports, Mapping): + values.extend(key for key, enabled in supports.items() if enabled is True) + values.extend(key for key, enabled in payload.items() if enabled is True) + elif isinstance(payload, (list, tuple)): + values.extend(payload) + + out: list[str] = [] + for value in values: + cap = mc.normalize_capability(value) + if cap and cap not in out: + out.append(cap) + for value in as_list(raw.get("supported_parameters")): + cap = _PARAMETER_CAPABILITIES.get(_shape_token(value)) + if cap and cap not in out: + out.append(cap) + task = next( + (_shape_token(raw.get(key)) for key in ("type", "model_type", "task", "pipeline_tag") if raw.get(key)), + "", + ) + task_capability = { + "automatic_speech_recognition": mc.CAP_TRANSCRIPTION, + "text_to_speech": mc.CAP_TTS, + "text_to_image": mc.CAP_IMAGE_GENERATION, + "image_to_image": mc.CAP_IMAGE_EDITING, + "text_to_video": mc.CAP_VIDEO_GENERATION, + "image_text_to_text": mc.CAP_VISION, + "image_question_answering": mc.CAP_VISION, + }.get(task) + if task_capability and task_capability not in out: + out.append(task_capability) + return tuple(out) + + +def _limits(raw: Mapping[str, Any]) -> dict[str, int]: + architecture = as_mapping(raw.get("architecture")) + top_provider = as_mapping(raw.get("top_provider")) + limits: dict[str, int] = {} + for keys, target in ( + (("context_length", "max_context_length", "max_model_len"), "context_tokens"), + (("input_token_limit", "inputTokenLimit"), "input_tokens"), + (("output_token_limit", "outputTokenLimit", "max_completion_tokens"), "output_tokens"), + ): + for key in keys: + value = int_limit(raw.get(key)) or int_limit(architecture.get(key)) or int_limit(top_provider.get(key)) + if value: + limits[target] = value + break + return limits + + +def _default_modalities( + family: str, + raw: Mapping[str, Any], +) -> tuple[tuple[str, ...], tuple[str, ...]]: + task = next( + (_shape_token(raw.get(key)) for key in ("type", "model_type", "task", "pipeline_tag") if raw.get(key)), + "", + ) + task_modalities = { + "automatic_speech_recognition": ((mc.MODALITY_AUDIO,), (mc.MODALITY_TEXT,)), + "text_to_speech": ((mc.MODALITY_TEXT,), (mc.MODALITY_AUDIO,)), + "text_to_image": ((mc.MODALITY_TEXT,), (mc.MODALITY_IMAGE,)), + "image_to_image": ((mc.MODALITY_IMAGE,), (mc.MODALITY_IMAGE,)), + "text_to_video": ((mc.MODALITY_TEXT,), (mc.MODALITY_VIDEO,)), + "image_text_to_text": ((mc.MODALITY_TEXT, mc.MODALITY_IMAGE), (mc.MODALITY_TEXT,)), + "image_question_answering": ((mc.MODALITY_TEXT, mc.MODALITY_IMAGE), (mc.MODALITY_TEXT,)), + }.get(task) + if task_modalities: + return task_modalities + if family == mc.FAMILY_CHAT: + return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,) + if family == mc.FAMILY_EMBEDDING: + return (mc.MODALITY_TEXT,), (mc.MODALITY_EMBEDDING,) + if family == mc.FAMILY_IMAGE: + return (mc.MODALITY_TEXT,), (mc.MODALITY_IMAGE,) + if family == mc.FAMILY_VIDEO: + return (mc.MODALITY_TEXT,), (mc.MODALITY_VIDEO,) + if family == mc.FAMILY_AUDIO: + return (), () + if family in {mc.FAMILY_RERANK, mc.FAMILY_CLASSIFICATION, mc.FAMILY_MODERATION}: + return (mc.MODALITY_TEXT,), (mc.MODALITY_TEXT,) + return (), () + + def record_from_model( raw: Mapping[str, Any], *, @@ -29,22 +229,47 @@ def record_from_model( model_id = model_id_from(raw, "id", "name", "model") if not model_id: return None - capability = mc.unknown_capability( - source=mc.SOURCE_PROVIDER_READER, - confidence=mc.CONFIDENCE_UNKNOWN, + + family = _family_from_explicit_fields(raw) + input_modalities, output_modalities = _modalities(raw) + if family == mc.FAMILY_UNKNOWN: + family = family_from_modalities(input_modalities, output_modalities) + if family != mc.FAMILY_UNKNOWN and not input_modalities and not output_modalities: + input_modalities, output_modalities = _default_modalities(family, raw) + capabilities = merge_unique( + _explicit_capabilities(raw), + _capabilities_from_modalities(input_modalities, output_modalities), ) + limits = _limits(raw) + if family == mc.FAMILY_UNKNOWN and not capabilities and not limits: + capability = mc.unknown_capability( + source=mc.SOURCE_PROVIDER_READER, + confidence=mc.CONFIDENCE_UNKNOWN, + ) + else: + capability = build_capability( + family=family, + input_modalities=input_modalities, + output_modalities=output_modalities, + capabilities=capabilities, + limits=limits, + ) return ModelCapabilityRecord( vendor=vendor_id, model_id=model_id, stable_model_id=stable_model_id_for(vendor_id, model_id, endpoint_id=endpoint_id, base_url=base_url), display_name=compact_str(raw.get("display_name") or raw.get("name")), capability=capability, + deterministic_controls=deterministic_controls_from_supported_parameters( + raw.get("supported_parameters") + ), + model_family=compact_str(raw.get("root") or raw.get("model_family")), raw=raw, ) def records_from_payload( - payload: Mapping[str, Any], + payload: Any, *, vendor_id: str = VENDOR_GENERIC_OPENAI, endpoint_id: Any = "", diff --git a/src/model_capability_readers/google.py b/src/model_capability_readers/google.py index 9edb57bdb..f6d0bfe28 100644 --- a/src/model_capability_readers/google.py +++ b/src/model_capability_readers/google.py @@ -42,6 +42,7 @@ def record_from_model( display_name=compact_str(raw.get("displayName")) or model_id, capability=ai_studio.capability_from_model(raw), deterministic_controls=ai_studio.deterministic_controls_from_model(raw), + model_family=compact_str(raw.get("baseModelId")), raw=raw, ) diff --git a/src/model_capability_readers/huggingface.py b/src/model_capability_readers/huggingface.py new file mode 100644 index 000000000..edb34d471 --- /dev/null +++ b/src/model_capability_readers/huggingface.py @@ -0,0 +1,89 @@ +"""Hugging Face Hub model-info reader using explicit pipeline metadata.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers import generic_openai +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_HUGGINGFACE, + as_mapping, + compact_str, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_HUGGINGFACE + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = compact_str(raw.get("modelId") or raw.get("id")) + if not model_id: + return None + structural = generic_openai.record_from_model( + {**raw, "id": model_id}, + vendor_id=VENDOR_HUGGINGFACE, + endpoint_id=endpoint_id, + base_url=base_url, + ) + if not structural: + return None + capability = mc.ModelCapability.build( + family=structural.capability.family, + primary_task=structural.capability.primary_task, + input_modalities=structural.capability.modalities.input, + output_modalities=structural.capability.modalities.output, + capabilities=structural.capability.capabilities, + limits=dict(structural.capability.limits), + source=mc.SOURCE_COOKBOOK_HF, + confidence=mc.CONFIDENCE_REGISTRY, + ) + config = as_mapping(raw.get("config")) + return ModelCapabilityRecord( + vendor=VENDOR_HUGGINGFACE, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_HUGGINGFACE, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=( + compact_str( + raw.get("cardData", {}).get("pretty_name") + if isinstance(raw.get("cardData"), Mapping) + else "" + ) + or model_id + ), + capability=capability, + deterministic_controls=structural.deterministic_controls, + model_family=compact_str(config.get("model_type")), + raw=raw, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + if isinstance(payload, Mapping) and (payload.get("modelId") or payload.get("pipeline_tag")): + record = record_from_model(payload, endpoint_id=endpoint_id, base_url=base_url) + return (record,) if record else () + records: list[ModelCapabilityRecord] = [] + for item in openai_model_items(payload): + record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url) + if record: + records.append(record) + return tuple(records) diff --git a/src/model_capability_readers/lmstudio.py b/src/model_capability_readers/lmstudio.py index 960649959..919055936 100644 --- a/src/model_capability_readers/lmstudio.py +++ b/src/model_capability_readers/lmstudio.py @@ -160,6 +160,7 @@ def record_from_native_model( ), display_name=compact_str(raw.get("display_name") or raw.get("name")) or model_id, capability=capability, + model_family=compact_str(raw.get("architecture") or raw.get("arch")), raw=raw, ) diff --git a/src/model_capability_readers/mistral.py b/src/model_capability_readers/mistral.py new file mode 100644 index 000000000..6a316169a --- /dev/null +++ b/src/model_capability_readers/mistral.py @@ -0,0 +1,112 @@ +"""Mistral native model-catalog capability reader.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_MISTRAL, + as_mapping, + build_capability, + compact_str, + int_limit, + merge_unique, + model_id_from, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_MISTRAL + + +def _family(raw: Mapping[str, Any]) -> str: + capabilities = as_mapping(raw.get("capabilities")) + if capabilities.get("classification") is True and not ( + capabilities.get("completion_chat") is True + or capabilities.get("completion_fim") is True + ): + return mc.FAMILY_CLASSIFICATION + if capabilities.get("completion_chat") is True or capabilities.get("completion_fim") is True: + return mc.FAMILY_CHAT + return mc.FAMILY_UNKNOWN + + +def _capabilities(raw: Mapping[str, Any]) -> tuple[str, ...]: + payload = as_mapping(raw.get("capabilities")) + values: list[str] = [] + for key, capability in ( + ("vision", mc.CAP_VISION), + ("function_calling", mc.CAP_TOOL_CALL), + ("reasoning", mc.CAP_REASONING), + ("structured_output", mc.CAP_STRUCTURED_OUTPUT), + ("structured_outputs", mc.CAP_STRUCTURED_OUTPUT), + ): + if payload.get(key) is True: + values.append(capability) + return merge_unique(values) + + +def record_from_model( + raw: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = model_id_from(raw, "id") + if not model_id: + return None + family = _family(raw) + capabilities = _capabilities(raw) + if family == mc.FAMILY_CHAT: + inputs = [mc.MODALITY_TEXT] + if mc.CAP_VISION in capabilities: + inputs.append(mc.MODALITY_IMAGE) + input_modalities = tuple(inputs) + output_modalities = (mc.MODALITY_TEXT,) + elif family == mc.FAMILY_CLASSIFICATION: + input_modalities = (mc.MODALITY_TEXT,) + output_modalities = (mc.MODALITY_TEXT,) + else: + input_modalities = () + output_modalities = () + context_tokens = int_limit(raw.get("max_context_length")) + limits = {"context_tokens": context_tokens} if context_tokens else {} + capability = build_capability( + family=family, + input_modalities=input_modalities, + output_modalities=output_modalities, + capabilities=capabilities, + limits=limits, + ) + return ModelCapabilityRecord( + vendor=VENDOR_MISTRAL, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_MISTRAL, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=compact_str(raw.get("name")) or model_id, + capability=capability, + model_family=compact_str(raw.get("root")), + raw=raw, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + records: list[ModelCapabilityRecord] = [] + for item in openai_model_items(payload): + record = record_from_model(item, endpoint_id=endpoint_id, base_url=base_url) + if record: + records.append(record) + return tuple(records) diff --git a/src/model_capability_readers/ollama.py b/src/model_capability_readers/ollama.py index 8ad7a5e54..cd21f6c0e 100644 --- a/src/model_capability_readers/ollama.py +++ b/src/model_capability_readers/ollama.py @@ -59,17 +59,11 @@ def _family_from_ollama_capabilities(values: Any) -> str: def _parameters_mapping(value: Any) -> Mapping[str, Any]: - if isinstance(value, Mapping): - return value - text = compact_str(value) - if not text: - return {} - parsed: dict[str, str] = {} - for line in text.splitlines(): - parts = line.strip().split(None, 1) - if len(parts) == 2: - parsed[parts[0]] = parts[1] - return parsed + # `/api/show` currently serializes this field as Modelfile text. Do not + # recover capability truth by reparsing that late text; prefer the native + # structured `model_info.*.context_length` shape. Mapping support remains + # for compatible servers that already return structured parameters. + return value if isinstance(value, Mapping) else {} def _modalities_for_family(family: str, capabilities: tuple[str, ...]) -> tuple[tuple[str, ...], tuple[str, ...]]: @@ -148,6 +142,7 @@ def record_from_show_payload( stable_model_id=stable_model_id_for(VENDOR_OLLAMA, model_id, endpoint_id=endpoint_id, base_url=base_url), display_name=model_id, capability=capability, + model_family=compact_str(as_mapping(payload.get("details")).get("family")), raw=payload, ) @@ -180,6 +175,7 @@ def records_from_tags_payload( source=mc.SOURCE_PROVIDER_READER, confidence=mc.CONFIDENCE_UNKNOWN, ), + model_family=compact_str(as_mapping(item.get("details")).get("family")), raw=item, ) ) diff --git a/src/model_capability_readers/sglang.py b/src/model_capability_readers/sglang.py new file mode 100644 index 000000000..5b6b96c63 --- /dev/null +++ b/src/model_capability_readers/sglang.py @@ -0,0 +1,100 @@ +"""SGLang `/model_info` and OpenAI model-card reader.""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import PurePosixPath +from typing import Any + +from src import model_capabilities as mc +from src.model_capability_readers import generic_openai +from src.model_capability_readers.base import ( + ModelCapabilityRecord, + VENDOR_SGLANG, + as_mapping, + build_capability, + compact_str, + deterministic_controls_from_supported_parameters, + openai_model_items, + stable_model_id_for, +) + + +vendor = VENDOR_SGLANG + + +def _model_id(payload: Mapping[str, Any]) -> str: + value = compact_str(payload.get("served_model_name") or payload.get("model_path")) + if not value: + return "" + return PurePosixPath(value).name if value.startswith("/") else value + + +def record_from_model_info( + payload: Mapping[str, Any], + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> ModelCapabilityRecord | None: + model_id = _model_id(payload) + if not model_id: + return None + capabilities: list[str] = [] + inputs: list[str] = [] + outputs: list[str] = [] + family = mc.FAMILY_UNKNOWN + if payload.get("is_generation") is True: + family = mc.FAMILY_CHAT + inputs.append(mc.MODALITY_TEXT) + outputs.append(mc.MODALITY_TEXT) + if payload.get("has_image_understanding") is True: + inputs.append(mc.MODALITY_IMAGE) + capabilities.append(mc.CAP_VISION) + if payload.get("has_audio_understanding") is True: + inputs.append(mc.MODALITY_AUDIO) + capabilities.append(mc.CAP_AUDIO_INPUT) + capability = build_capability( + family=family, + input_modalities=inputs, + output_modalities=outputs, + capabilities=capabilities, + ) + sampling = as_mapping(payload.get("preferred_sampling_params")) + return ModelCapabilityRecord( + vendor=VENDOR_SGLANG, + model_id=model_id, + stable_model_id=stable_model_id_for( + VENDOR_SGLANG, + model_id, + endpoint_id=endpoint_id, + base_url=base_url, + ), + display_name=model_id, + capability=capability, + deterministic_controls=deterministic_controls_from_supported_parameters(sampling.keys()), + model_family=compact_str(payload.get("model_type")), + raw=payload, + ) + + +def records_from_payload( + payload: Any, + *, + endpoint_id: Any = "", + base_url: Any = "", +) -> tuple[ModelCapabilityRecord, ...]: + mapping = as_mapping(payload) + if "is_generation" in mapping and "model_path" in mapping: + record = record_from_model_info(mapping, endpoint_id=endpoint_id, base_url=base_url) + return (record,) if record else () + records: list[ModelCapabilityRecord] = [] + for item in openai_model_items(payload): + record = generic_openai.record_from_model( + item, + vendor_id=VENDOR_SGLANG, + endpoint_id=endpoint_id, + base_url=base_url, + ) + if record: + records.append(record) + return tuple(records) diff --git a/src/provider_capability_schemas.py b/src/provider_capability_schemas.py new file mode 100644 index 000000000..531cbd19b --- /dev/null +++ b/src/provider_capability_schemas.py @@ -0,0 +1,933 @@ +"""Canonical serving-provider and payload-shape metadata. + +This module describes provider/API dialects and model-catalog JSON shapes. It +does not perform network I/O, parse model names, or claim that every model on a +provider supports every feature exposed by that provider. Model capability is +still read from each model record (or remains unknown). + +Resolution is deliberately stepped and deterministic: + +1. an explicit provider/endpoint kind; +2. an exact known provider host; +3. a discriminating native payload shape; +4. a general structural model-list shape; +5. unknown. + +Unknown keys are left to the reader's ``raw`` evidence. They never become a +capability merely because a future provider happens to add them. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlparse + + +PROVIDER_UNKNOWN = "unknown" +PROVIDER_GENERIC_OPENAI = "generic_openai" + +DIALECT_OPENAI_CHAT = "openai_chat_completions" +DIALECT_OPENAI_RESPONSES = "openai_responses" +DIALECT_ANTHROPIC_MESSAGES = "anthropic_messages" +DIALECT_COHERE_V2 = "cohere_v2" +DIALECT_GOOGLE_GENERATE_CONTENT = "google_generate_content" +DIALECT_OLLAMA_NATIVE = "ollama_native" +DIALECT_LMSTUDIO_NATIVE_V1 = "lmstudio_native_v1" +DIALECT_LLAMACPP_NATIVE = "llamacpp_native" +DIALECT_SGLANG_NATIVE = "sglang_native" +DIALECT_HUGGINGFACE_HUB = "huggingface_hub" +DIALECT_CHATGPT_SUBSCRIPTION = "chatgpt_subscription_responses" + +RESOLUTION_EXPLICIT = "explicit" +RESOLUTION_ENDPOINT_KIND = "endpoint_kind" +RESOLUTION_HOST = "host" +RESOLUTION_NATIVE_SHAPE = "native_shape" +RESOLUTION_GENERAL_SHAPE = "general_shape" +RESOLUTION_UNKNOWN = "unknown" + +ENVELOPE_DATA = "data" +ENVELOPE_MODELS = "models" +ENVELOPE_BARE_LIST = "bare_list" +ENVELOPE_SINGLE = "single" + +_MISSING = object() + + +def _token(value: Any) -> str: + return str(value or "").strip().lower().replace("-", "_").replace(" ", "_") + + +def _path_value(value: Any, path: str) -> Any: + current = value + for part in path.split("."): + if not isinstance(current, Mapping) or part not in current: + return _MISSING + current = current[part] + return current + + +def _path_present(value: Any, path: str) -> bool: + return _path_value(value, path) is not _MISSING + + +def _items_for_envelope(payload: Any, envelope: str) -> tuple[Mapping[str, Any], ...]: + if envelope == ENVELOPE_BARE_LIST: + values = payload if isinstance(payload, (list, tuple)) else () + elif envelope == ENVELOPE_SINGLE: + values = (payload,) if isinstance(payload, Mapping) else () + elif isinstance(payload, Mapping): + values = payload.get(envelope) + values = values if isinstance(values, (list, tuple)) else () + else: + values = () + return tuple(item for item in values if isinstance(item, Mapping)) + + +@dataclass(frozen=True) +class ProviderCatalogShape: + """A declarative, versioned provider model-catalog shape.""" + + shape_id: str + provider_id: str + endpoint_path: str + envelope: str + identity_paths: tuple[str, ...] + required_root_paths: tuple[str, ...] = () + required_item_paths: tuple[str, ...] = () + required_item_any_paths: tuple[str, ...] = () + item_types: tuple[tuple[str, tuple[Any, ...]], ...] = () + item_values: tuple[tuple[str, tuple[Any, ...]], ...] = () + capability_paths: tuple[str, ...] = () + api_version: str = "" + priority: int = 0 + latest: bool = True + + def items(self, payload: Any) -> tuple[Mapping[str, Any], ...]: + return _items_for_envelope(payload, self.envelope) + + def matches(self, payload: Any) -> bool: + if self.required_root_paths: + if not isinstance(payload, Mapping): + return False + if not all(_path_present(payload, path) for path in self.required_root_paths): + return False + + items = self.items(payload) + if not items: + return False + for item in items: + if self.identity_paths: + has_identity = False + for path in self.identity_paths: + value = _path_value(item, path) + if value is not _MISSING and value is not None and value != "": + has_identity = True + break + if not has_identity: + continue + if not all(_path_present(item, path) for path in self.required_item_paths): + continue + if self.required_item_any_paths and not any( + _path_present(item, path) for path in self.required_item_any_paths + ): + continue + if any( + not isinstance(_path_value(item, path), expected_types) + for path, expected_types in self.item_types + ): + continue + if any(_path_value(item, path) not in expected for path, expected in self.item_values): + continue + return True + return False + + +@dataclass(frozen=True) +class ProviderApiShape: + """Stable request/response field paths for one API dialect. + + Paths are documentation and validation inputs, not late response parsers. + A model-specific exception can narrow these fields in the model quirk + registry without changing the provider's general transport contract. + """ + + dialect: str + request_path: str + stream_path: str = "" + model_field: str = "model" + message_field: str = "messages" + tool_request_paths: tuple[str, ...] = () + tool_response_paths: tuple[str, ...] = () + text_response_paths: tuple[str, ...] = () + reasoning_response_paths: tuple[str, ...] = () + request_control_paths: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ProviderCapabilitySchema: + provider_id: str + display_name: str + aliases: tuple[str, ...] = () + host_suffixes: tuple[str, ...] = () + api_shapes: tuple[ProviderApiShape, ...] = () + catalog_shapes: tuple[ProviderCatalogShape, ...] = () + fallback_provider_id: str = PROVIDER_GENERIC_OPENAI + model_capabilities_are_per_model: bool = True + + +@dataclass(frozen=True) +class ProviderResolution: + provider_id: str + stage: str + schema: ProviderCapabilitySchema + catalog_shape: ProviderCatalogShape | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "provider_id": self.provider_id, + "stage": self.stage, + "schema_id": self.schema.provider_id, + "catalog_shape_id": self.catalog_shape.shape_id if self.catalog_shape else "", + } + + +OPENAI_CHAT_SHAPE = ProviderApiShape( + dialect=DIALECT_OPENAI_CHAT, + request_path="/v1/chat/completions", + stream_path="/v1/chat/completions", + tool_request_paths=("tools[].function", "tool_choice", "parallel_tool_calls"), + tool_response_paths=("choices[].message.tool_calls[].function", "choices[].delta.tool_calls[].function"), + text_response_paths=("choices[].message.content", "choices[].delta.content"), + reasoning_response_paths=( + "choices[].message.reasoning_content", + "choices[].delta.reasoning_content", + "choices[].delta.reasoning", + "choices[].delta.thinking", + ), + request_control_paths=( + "temperature", + "top_p", + "seed", + "response_format", + "reasoning_effort", + ), +) + +OPENAI_RESPONSES_SHAPE = ProviderApiShape( + dialect=DIALECT_OPENAI_RESPONSES, + request_path="/v1/responses", + stream_path="/v1/responses", + message_field="input", + tool_request_paths=("tools[]", "tool_choice", "parallel_tool_calls"), + tool_response_paths=( + "output[type=function_call].arguments", + "response.function_call_arguments.delta", + ), + text_response_paths=( + "output[type=message].content[type=output_text].text", + "response.output_text.delta", + ), + reasoning_response_paths=( + "output[type=reasoning].summary[].text", + "output[type=reasoning].encrypted_content", + "response.reasoning_summary_text.delta", + ), + request_control_paths=("temperature", "top_p", "reasoning", "text.format"), +) + +ANTHROPIC_MESSAGES_SHAPE = ProviderApiShape( + dialect=DIALECT_ANTHROPIC_MESSAGES, + request_path="/v1/messages", + stream_path="/v1/messages", + tool_request_paths=("tools[].input_schema", "tool_choice", "messages[].content[].tool_result"), + tool_response_paths=("content[].tool_use", "content_block_delta.delta.partial_json"), + text_response_paths=("content[].text", "content_block_delta.delta.text"), + reasoning_response_paths=("content[].thinking", "content[].signature"), + request_control_paths=("temperature", "top_p", "top_k", "thinking", "output_config"), +) + +COHERE_V2_SHAPE = ProviderApiShape( + dialect=DIALECT_COHERE_V2, + request_path="/v2/chat", + stream_path="/v2/chat", + tool_request_paths=("tools[].parameters", "messages[].tool_calls", "messages[].tool_results"), + tool_response_paths=("message.tool_calls", "tool-call-start", "tool-call-delta", "tool-call-end"), + text_response_paths=("message.content[].text", "content-delta.delta.message.content.text"), + reasoning_response_paths=("message.content[].thinking",), + request_control_paths=( + "temperature", + "p", + "k", + "seed", + "response_format", + "thinking", + ), +) + +GOOGLE_CONTENT_SHAPE = ProviderApiShape( + dialect=DIALECT_GOOGLE_GENERATE_CONTENT, + request_path="/v1beta/models/{model}:generateContent", + stream_path="/v1beta/models/{model}:streamGenerateContent?alt=sse", + message_field="contents", + tool_request_paths=("tools[].functionDeclarations", "toolConfig"), + tool_response_paths=("candidates[].content.parts[].functionCall", "candidates[].content.parts[].functionResponse"), + text_response_paths=("candidates[].content.parts[].text",), + reasoning_response_paths=("candidates[].content.parts[].thought", "candidates[].content.parts[].thoughtSignature"), + request_control_paths=( + "generationConfig.temperature", + "generationConfig.topP", + "generationConfig.topK", + "generationConfig.thinkingConfig", + "generationConfig.responseJsonSchema", + ), +) + +OLLAMA_NATIVE_SHAPE = ProviderApiShape( + dialect=DIALECT_OLLAMA_NATIVE, + request_path="/api/chat", + stream_path="/api/chat", + tool_request_paths=("tools[].function", "messages[].tool_calls[].function"), + tool_response_paths=("message.tool_calls[].function",), + text_response_paths=("message.content",), + reasoning_response_paths=("message.thinking",), + request_control_paths=("think", "format", "options.temperature", "options.top_p", "options.seed"), +) + +LMSTUDIO_NATIVE_SHAPE = ProviderApiShape( + dialect=DIALECT_LMSTUDIO_NATIVE_V1, + request_path="/api/v1/chat", + stream_path="/api/v1/chat", + message_field="input", + tool_request_paths=("integrations[].mcp",), + text_response_paths=("output[].content",), + reasoning_response_paths=("output[].reasoning",), + request_control_paths=("temperature", "top_p", "reasoning", "response_format"), +) + +SGLANG_NATIVE_SHAPE = ProviderApiShape( + dialect=DIALECT_SGLANG_NATIVE, + request_path="/generate", + stream_path="/generate", + message_field="text", + text_response_paths=("text",), + request_control_paths=("sampling_params.temperature", "sampling_params.top_p", "sampling_params.seed"), +) + +CHATGPT_SUBSCRIPTION_SHAPE = ProviderApiShape( + dialect=DIALECT_CHATGPT_SUBSCRIPTION, + request_path="/backend-api/codex/responses", + stream_path="/backend-api/codex/responses", + message_field="input", + tool_request_paths=("tools[]", "input[].function_call_output"), + tool_response_paths=("response.function_call_arguments.delta", "response.output_item.done"), + text_response_paths=("response.output_text.delta",), + reasoning_response_paths=("response.reasoning_summary_text.delta", "reasoning.encrypted_content"), + request_control_paths=("reasoning", "text", "parallel_tool_calls"), +) + + +GENERAL_DATA_SHAPE = ProviderCatalogShape( + shape_id="openai-compatible.models.data.v1", + provider_id=PROVIDER_GENERIC_OPENAI, + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id", "name", "model"), +) +GENERAL_MODELS_SHAPE = ProviderCatalogShape( + shape_id="general.models-envelope.v1", + provider_id=PROVIDER_GENERIC_OPENAI, + endpoint_path="/models", + envelope=ENVELOPE_MODELS, + identity_paths=("id", "key", "slug", "name", "model"), +) +GENERAL_BARE_SHAPE = ProviderCatalogShape( + shape_id="openai-compatible.models.bare-list.v1", + provider_id=PROVIDER_GENERIC_OPENAI, + endpoint_path="/models", + envelope=ENVELOPE_BARE_LIST, + identity_paths=("id", "name", "model"), +) + +OPENAI_MODELS_SHAPE = ProviderCatalogShape( + shape_id="openai.models.identity.v1", + provider_id="openai", + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("object", "created", "owned_by"), + item_values=(("object", ("model",)),), + api_version="v1", +) +OPENROUTER_MODELS_SHAPE = ProviderCatalogShape( + shape_id="openrouter.models.rich.v1", + provider_id="openrouter", + endpoint_path="/api/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_any_paths=("architecture", "supported_parameters", "top_provider", "canonical_slug"), + capability_paths=( + "architecture.input_modalities", + "architecture.output_modalities", + "supported_parameters", + "context_length", + "top_provider.max_completion_tokens", + ), + api_version="v1", + priority=90, +) +GOOGLE_MODELS_SHAPE = ProviderCatalogShape( + shape_id="google.generative-language.models.v1beta", + provider_id="google", + endpoint_path="/v1beta/models", + envelope=ENVELOPE_MODELS, + identity_paths=("baseModelId", "name"), + required_item_paths=("supportedGenerationMethods",), + item_types=(("supportedGenerationMethods", (list, tuple)),), + capability_paths=( + "supportedGenerationMethods", + "inputTokenLimit", + "outputTokenLimit", + "thinking", + "temperature", + "topP", + "topK", + ), + api_version="v1beta", + priority=100, +) +OLLAMA_TAGS_SHAPE = ProviderCatalogShape( + shape_id="ollama.tags.v1", + provider_id="ollama", + endpoint_path="/api/tags", + envelope=ENVELOPE_MODELS, + identity_paths=("model", "name"), + required_item_any_paths=("digest", "details.family", "details.families"), + capability_paths=("details.family", "details.families"), + priority=90, +) +OLLAMA_SHOW_SHAPE = ProviderCatalogShape( + shape_id="ollama.show.v1", + provider_id="ollama", + endpoint_path="/api/show", + envelope=ENVELOPE_SINGLE, + identity_paths=(), + required_item_paths=("capabilities",), + required_item_any_paths=("model_info", "details", "template", "parameters"), + item_types=(("capabilities", (list, tuple)),), + capability_paths=("capabilities", "model_info.*.context_length"), + priority=100, +) +LMSTUDIO_MODELS_V1_SHAPE = ProviderCatalogShape( + shape_id="lmstudio.models.native.v1", + provider_id="lmstudio", + endpoint_path="/api/v1/models", + envelope=ENVELOPE_MODELS, + identity_paths=("key",), + required_item_paths=("type",), + required_item_any_paths=("capabilities", "loaded_instances", "max_context_length", "architecture", "quantization"), + item_types=(("type", (str,)),), + capability_paths=("type", "capabilities", "max_context_length", "loaded_instances[].config.context_length"), + api_version="1", + priority=100, +) +LMSTUDIO_MODELS_V0_SHAPE = ProviderCatalogShape( + shape_id="lmstudio.models.native.v0", + provider_id="lmstudio", + endpoint_path="/api/v0/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("type",), + required_item_any_paths=("arch", "compatibility_type", "state", "max_context_length"), + item_types=(("type", (str,)),), + capability_paths=("type", "max_context_length"), + api_version="0", + priority=80, + latest=False, +) +LLAMACPP_PROPS_SHAPE = ProviderCatalogShape( + shape_id="llamacpp.props.v1", + provider_id="llamacpp", + endpoint_path="/props", + envelope=ENVELOPE_SINGLE, + identity_paths=("model_alias", "model_path"), + required_item_paths=("default_generation_settings",), + required_item_any_paths=("chat_template_caps", "modalities", "total_slots"), + item_types=(("default_generation_settings", (Mapping,)),), + capability_paths=( + "chat_template_caps", + "modalities", + "default_generation_settings.n_ctx", + "default_generation_settings.params", + ), + priority=100, +) +MISTRAL_MODELS_SHAPE = ProviderCatalogShape( + shape_id="mistral.models.rich.v1", + provider_id="mistral", + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("capabilities",), + required_item_any_paths=( + "capabilities.completion_chat", + "capabilities.completion_fim", + "capabilities.function_calling", + "capabilities.vision", + "capabilities.classification", + ), + item_types=(("capabilities", (Mapping,)),), + capability_paths=("capabilities", "max_context_length"), + api_version="v1", + priority=100, +) +COPILOT_MODELS_SHAPE = ProviderCatalogShape( + shape_id="github-copilot.models.v1", + provider_id="copilot", + endpoint_path="/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("model_picker_enabled", "capabilities.supports"), + item_types=( + ("model_picker_enabled", (bool,)), + ("capabilities.supports", (Mapping,)), + ), + capability_paths=("capabilities.supports.tool_calls", "capabilities.supports.vision"), + priority=100, +) +ANTHROPIC_MODELS_SHAPE = ProviderCatalogShape( + shape_id="anthropic.models.identity.v1", + provider_id="anthropic", + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("type", "display_name", "created_at"), + item_values=(("type", ("model",)),), + api_version="2023-06-01", + priority=70, +) +CHATGPT_MODELS_SHAPE = ProviderCatalogShape( + shape_id="chatgpt-subscription.codex-models.v1", + provider_id="chatgpt_subscription", + endpoint_path="/backend-api/codex/models?client_version=1.0.0", + envelope=ENVELOPE_MODELS, + identity_paths=("slug",), + required_item_any_paths=("visibility", "priority"), + priority=90, +) +SGLANG_MODEL_INFO_SHAPE = ProviderCatalogShape( + shape_id="sglang.model-info.v2", + provider_id="sglang", + endpoint_path="/model_info", + envelope=ENVELOPE_SINGLE, + identity_paths=("model_path",), + required_item_paths=("is_generation",), + required_item_any_paths=("tokenizer_path", "has_image_understanding", "has_audio_understanding"), + item_types=(("is_generation", (bool,)),), + capability_paths=("is_generation", "has_image_understanding", "has_audio_understanding", "preferred_sampling_params"), + api_version="2", + priority=100, +) +SGLANG_MODELS_SHAPE = ProviderCatalogShape( + shape_id="sglang.models.openai.v1", + provider_id="sglang", + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("root", "max_model_len"), + item_values=(("owned_by", ("sglang",)),), + capability_paths=("max_model_len",), + priority=80, +) +VLLM_MODELS_SHAPE = ProviderCatalogShape( + shape_id="vllm.models.openai.v1", + provider_id="vllm", + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("root", "max_model_len", "permission"), + item_values=(("owned_by", ("vllm",)),), + capability_paths=("max_model_len",), + priority=80, +) +HUGGINGFACE_MODEL_SHAPE = ProviderCatalogShape( + shape_id="huggingface.hub.model-info.v1", + provider_id="huggingface", + endpoint_path="/api/models/{model}", + envelope=ENVELOPE_SINGLE, + identity_paths=("modelId", "id"), + required_item_paths=("pipeline_tag",), + item_types=(("pipeline_tag", (str,)),), + capability_paths=("pipeline_tag", "tags", "config"), + priority=80, +) +COHERE_MODELS_SHAPE = ProviderCatalogShape( + shape_id="cohere.models.rich.v1", + provider_id="cohere", + endpoint_path="/v1/models", + envelope=ENVELOPE_MODELS, + identity_paths=("name",), + required_item_paths=("endpoints",), + required_item_any_paths=("context_length", "default_endpoints", "features", "sampling_defaults"), + item_types=(("endpoints", (list, tuple)),), + capability_paths=("endpoints", "context_length", "sampling_defaults"), + api_version="v1", + priority=100, +) +MINIMAX_MODELS_SHAPE = ProviderCatalogShape( + shape_id="minimax.models.identity.v1", + provider_id="minimax", + endpoint_path="/v1/models", + envelope=ENVELOPE_DATA, + identity_paths=("id",), + required_item_paths=("object", "owned_by"), + item_values=(("object", ("model",)), ("owned_by", ("minimax",))), + api_version="v1", + priority=90, +) + + +def _provider( + provider_id: str, + display_name: str, + *, + aliases: tuple[str, ...] = (), + hosts: tuple[str, ...] = (), + api_shapes: tuple[ProviderApiShape, ...] = (OPENAI_CHAT_SHAPE,), + catalog_shapes: tuple[ProviderCatalogShape, ...] = (), +) -> ProviderCapabilitySchema: + return ProviderCapabilitySchema( + provider_id=provider_id, + display_name=display_name, + aliases=aliases, + host_suffixes=hosts, + api_shapes=api_shapes, + catalog_shapes=catalog_shapes, + ) + + +PROVIDER_SCHEMAS = { + PROVIDER_GENERIC_OPENAI: _provider( + PROVIDER_GENERIC_OPENAI, + "General OpenAI-compatible", + aliases=("openai_compatible", "proxy"), + api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE), + catalog_shapes=(GENERAL_DATA_SHAPE, GENERAL_MODELS_SHAPE, GENERAL_BARE_SHAPE), + ), + "openai": _provider( + "openai", + "OpenAI", + hosts=("openai.com",), + api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE), + catalog_shapes=(OPENAI_MODELS_SHAPE,), + ), + "openrouter": _provider( + "openrouter", + "OpenRouter", + hosts=("openrouter.ai",), + catalog_shapes=(OPENROUTER_MODELS_SHAPE,), + ), + "google": _provider( + "google", + "Google Gemini", + aliases=("gemini", "google_ai_studio"), + hosts=("generativelanguage.googleapis.com",), + api_shapes=(GOOGLE_CONTENT_SHAPE, OPENAI_CHAT_SHAPE), + catalog_shapes=(GOOGLE_MODELS_SHAPE,), + ), + "anthropic": _provider( + "anthropic", + "Anthropic", + hosts=("anthropic.com",), + api_shapes=(ANTHROPIC_MESSAGES_SHAPE,), + catalog_shapes=(ANTHROPIC_MODELS_SHAPE,), + ), + "ollama": _provider( + "ollama", + "Ollama", + hosts=("ollama.com",), + api_shapes=(OLLAMA_NATIVE_SHAPE, OPENAI_CHAT_SHAPE), + catalog_shapes=(OLLAMA_SHOW_SHAPE, OLLAMA_TAGS_SHAPE), + ), + "lmstudio": _provider( + "lmstudio", + "LM Studio", + aliases=("lm_studio",), + api_shapes=(LMSTUDIO_NATIVE_SHAPE, OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE), + catalog_shapes=(LMSTUDIO_MODELS_V1_SHAPE, LMSTUDIO_MODELS_V0_SHAPE), + ), + "llamacpp": _provider( + "llamacpp", + "llama.cpp", + aliases=("llama.cpp", "llama_cpp", "llama_server"), + api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE, ANTHROPIC_MESSAGES_SHAPE), + catalog_shapes=(LLAMACPP_PROPS_SHAPE,), + ), + "mistral": _provider( + "mistral", + "Mistral", + hosts=("mistral.ai",), + catalog_shapes=(MISTRAL_MODELS_SHAPE,), + ), + "copilot": _provider( + "copilot", + "GitHub Copilot", + aliases=("github_copilot",), + hosts=("api.githubcopilot.com",), + catalog_shapes=(COPILOT_MODELS_SHAPE,), + ), + "chatgpt_subscription": _provider( + "chatgpt_subscription", + "ChatGPT Subscription", + aliases=("chatgpt-subscription", "chatgpt", "codex_subscription"), + hosts=("chatgpt.com",), + api_shapes=(CHATGPT_SUBSCRIPTION_SHAPE,), + catalog_shapes=(CHATGPT_MODELS_SHAPE,), + ), + "sglang": _provider( + "sglang", + "SGLang", + api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE, SGLANG_NATIVE_SHAPE), + catalog_shapes=(SGLANG_MODEL_INFO_SHAPE, SGLANG_MODELS_SHAPE), + ), + "vllm": _provider( + "vllm", + "vLLM", + api_shapes=(OPENAI_CHAT_SHAPE, OPENAI_RESPONSES_SHAPE), + catalog_shapes=(VLLM_MODELS_SHAPE,), + ), + "huggingface": _provider( + "huggingface", + "Hugging Face", + aliases=("hf", "hugging_face"), + hosts=("huggingface.co",), + api_shapes=(OPENAI_CHAT_SHAPE,), + catalog_shapes=(HUGGINGFACE_MODEL_SHAPE,), + ), + "cohere": _provider( + "cohere", + "Cohere", + hosts=("cohere.ai", "cohere.com"), + api_shapes=(COHERE_V2_SHAPE, OPENAI_CHAT_SHAPE), + catalog_shapes=(COHERE_MODELS_SHAPE,), + ), + "minimax": _provider( + "minimax", + "MiniMax", + hosts=("minimax.io", "minimaxi.com"), + api_shapes=(ANTHROPIC_MESSAGES_SHAPE, OPENAI_CHAT_SHAPE), + catalog_shapes=(MINIMAX_MODELS_SHAPE,), + ), +} + +# These providers are currently OpenAI-compatible in Odysseus and have no +# provider-reported capability catalog shape that is stronger than the general +# structural fallback. Keeping individual identities prevents transport +# quirks from being flattened into "OpenAI" while capability stays per-model. +_GENERAL_PROVIDER_ALIASES = { + "moonshot": ("moonshot_ai",), + "nvidia": ("nvidia_nim", "nim"), + "xai": ("x_ai",), + "zai": ("z.ai", "z_ai"), + "opencode": ("opencode_go", "opencode_zen"), + "together": ("together_ai",), + "fireworks": ("fireworks_ai",), + "atlas_cloud": ("atlas",), + "azure_openai": ("azure",), + "bedrock": ("aws_bedrock",), + "cloudflare_workers_ai": ("workers_ai",), + "mlx_lm": ("mlx",), + "text_generation_inference": ("tgi", "huggingface_tgi", "hugging_face_tgi"), +} +for _provider_id, _display, _hosts in ( + ("moonshot", "Moonshot AI", ("moonshot.ai", "moonshot.cn")), + ("groq", "Groq", ("groq.com",)), + ("nvidia", "NVIDIA NIM", ("nvidia.com",)), + ("cerebras", "Cerebras", ("cerebras.ai",)), + ("deepseek", "DeepSeek", ("deepseek.com",)), + ("together", "Together AI", ("together.xyz", "together.ai")), + ("fireworks", "Fireworks AI", ("fireworks.ai",)), + ("xai", "xAI", ("x.ai",)), + ("zai", "Z.AI", ("z.ai",)), + ("opencode", "OpenCode", ("opencode.ai",)), + ("perplexity", "Perplexity", ("perplexity.ai",)), + ("github_models", "GitHub Models", ("models.inference.ai.azure.com",)), + ("atlas_cloud", "Atlas Cloud", ("atlascloud.ai",)), + ("siliconflow", "SiliconFlow", ("siliconflow.cn", "siliconflow.com",)), + ("kimi_code", "Kimi Code", ("kimi.com",)), + ("venice", "Venice", ("venice.ai",)), + ("azure_openai", "Azure OpenAI", ("openai.azure.com",)), + ("bedrock", "AWS Bedrock", ()), + ("cloudflare_workers_ai", "Cloudflare Workers AI", ()), + ("mlx_lm", "MLX LM", ()), + ("text_generation_inference", "Hugging Face TGI", ()), + ("lmdeploy", "LMDeploy", ()), + ("litellm", "LiteLLM", ()), +): + PROVIDER_SCHEMAS[_provider_id] = _provider( + _provider_id, + _display, + aliases=_GENERAL_PROVIDER_ALIASES.get(_provider_id, ()), + hosts=_hosts, + ) + +UNKNOWN_SCHEMA = ProviderCapabilitySchema( + provider_id=PROVIDER_UNKNOWN, + display_name="Unknown provider", + api_shapes=(), + catalog_shapes=(), +) + +_ALIASES = { + _token(alias): provider_id + for provider_id, schema in PROVIDER_SCHEMAS.items() + for alias in (provider_id, *schema.aliases) +} + + +def normalize_provider_id(value: Any) -> str: + token = _token(value) + return _ALIASES.get(token, token if token in PROVIDER_SCHEMAS else PROVIDER_UNKNOWN) + + +def schema_for_provider(value: Any) -> ProviderCapabilitySchema: + return PROVIDER_SCHEMAS.get(normalize_provider_id(value), UNKNOWN_SCHEMA) + + +def _host_matches(host: str, suffix: str) -> bool: + return host == suffix or host.endswith("." + suffix) + + +def provider_from_host(base_url: Any) -> str: + try: + host = (urlparse(str(base_url or "")).hostname or "").lower().rstrip(".") + except Exception: + return PROVIDER_UNKNOWN + if not host: + return PROVIDER_UNKNOWN + if host.startswith("copilot-api.") and host.endswith(".ghe.com"): + return "copilot" + matches = [ + schema.provider_id + for schema in PROVIDER_SCHEMAS.values() + if any(_host_matches(host, suffix) for suffix in schema.host_suffixes) + ] + return matches[0] if len(set(matches)) == 1 else PROVIDER_UNKNOWN + + +def catalog_shape_for_payload( + payload: Any, + *, + provider_id: Any = None, + include_general: bool = True, +) -> ProviderCatalogShape | None: + normalized = normalize_provider_id(provider_id) + provider_is_explicit = normalized not in {PROVIDER_UNKNOWN, PROVIDER_GENERIC_OPENAI} + if provider_is_explicit: + schemas = (PROVIDER_SCHEMAS[normalized],) + else: + schemas = tuple(PROVIDER_SCHEMAS.values()) + + candidates = [ + shape + for schema in schemas + for shape in schema.catalog_shapes + if (provider_is_explicit or shape.priority > 0) and shape.matches(payload) + ] + if candidates: + best_priority = max(shape.priority for shape in candidates) + best = [shape for shape in candidates if shape.priority == best_priority] + providers = {shape.provider_id for shape in best} + if len(providers) == 1: + return sorted(best, key=lambda shape: shape.shape_id)[0] + + if not include_general: + return None + for shape in PROVIDER_SCHEMAS[PROVIDER_GENERIC_OPENAI].catalog_shapes: + if shape.matches(payload): + return shape + return None + + +def resolve_provider( + payload: Any = None, + *, + provider: Any = None, + endpoint_kind: Any = None, + base_url: Any = None, +) -> ProviderResolution: + explicit = normalize_provider_id(provider) + if explicit != PROVIDER_UNKNOWN: + shape = catalog_shape_for_payload(payload, provider_id=explicit) if payload is not None else None + return ProviderResolution(explicit, RESOLUTION_EXPLICIT, PROVIDER_SCHEMAS[explicit], shape) + + kind = normalize_provider_id(endpoint_kind) + if kind != PROVIDER_UNKNOWN: + shape = catalog_shape_for_payload(payload, provider_id=kind) if payload is not None else None + return ProviderResolution(kind, RESOLUTION_ENDPOINT_KIND, PROVIDER_SCHEMAS[kind], shape) + + host_provider = provider_from_host(base_url) + if host_provider != PROVIDER_UNKNOWN: + shape = catalog_shape_for_payload(payload, provider_id=host_provider) if payload is not None else None + return ProviderResolution( + host_provider, + RESOLUTION_HOST, + PROVIDER_SCHEMAS[host_provider], + shape, + ) + + shape = catalog_shape_for_payload(payload, include_general=False) if payload is not None else None + if shape: + return ProviderResolution( + shape.provider_id, + RESOLUTION_NATIVE_SHAPE, + PROVIDER_SCHEMAS[shape.provider_id], + shape, + ) + + shape = catalog_shape_for_payload(payload) if payload is not None else None + if shape: + return ProviderResolution( + PROVIDER_GENERIC_OPENAI, + RESOLUTION_GENERAL_SHAPE, + PROVIDER_SCHEMAS[PROVIDER_GENERIC_OPENAI], + shape, + ) + + return ProviderResolution(PROVIDER_UNKNOWN, RESOLUTION_UNKNOWN, UNKNOWN_SCHEMA, None) + + +__all__ = [ + "ANTHROPIC_MESSAGES_SHAPE", + "CHATGPT_SUBSCRIPTION_SHAPE", + "COHERE_V2_SHAPE", + "DIALECT_ANTHROPIC_MESSAGES", + "DIALECT_CHATGPT_SUBSCRIPTION", + "DIALECT_COHERE_V2", + "DIALECT_GOOGLE_GENERATE_CONTENT", + "DIALECT_HUGGINGFACE_HUB", + "DIALECT_LLAMACPP_NATIVE", + "DIALECT_LMSTUDIO_NATIVE_V1", + "DIALECT_OLLAMA_NATIVE", + "DIALECT_OPENAI_CHAT", + "DIALECT_OPENAI_RESPONSES", + "DIALECT_SGLANG_NATIVE", + "GOOGLE_CONTENT_SHAPE", + "LMSTUDIO_NATIVE_SHAPE", + "OLLAMA_NATIVE_SHAPE", + "OPENAI_CHAT_SHAPE", + "OPENAI_RESPONSES_SHAPE", + "PROVIDER_GENERIC_OPENAI", + "PROVIDER_SCHEMAS", + "PROVIDER_UNKNOWN", + "ProviderApiShape", + "ProviderCapabilitySchema", + "ProviderCatalogShape", + "ProviderResolution", + "catalog_shape_for_payload", + "normalize_provider_id", + "provider_from_host", + "resolve_provider", + "schema_for_provider", +] diff --git a/tests/test_model_capability_readers.py b/tests/test_model_capability_readers.py index 36ef93ab1..17734d448 100644 --- a/tests/test_model_capability_readers.py +++ b/tests/test_model_capability_readers.py @@ -18,14 +18,14 @@ def surfaces(record): return set(mc.display_surfaces_for(record.capability)) -def test_detect_vendor_uses_endpoint_kind_then_host_and_common_local_ports(): +def test_detect_vendor_uses_endpoint_kind_and_host_but_not_ambiguous_local_ports(): assert detect_vendor("https://example.test/v1", endpoint_kind="ollama") == VENDOR_OLLAMA assert detect_vendor("http://127.0.0.1:8080", endpoint_kind="llama_cpp") == VENDOR_LLAMACPP assert detect_vendor("https://openrouter.ai/api/v1") == VENDOR_OPENROUTER assert detect_vendor("https://api.openai.com/v1") == VENDOR_OPENAI assert detect_vendor("https://generativelanguage.googleapis.com/v1beta/openai") == VENDOR_GOOGLE - assert detect_vendor("http://127.0.0.1:11434") == VENDOR_OLLAMA - assert detect_vendor("http://127.0.0.1:1234") == VENDOR_LMSTUDIO + assert detect_vendor("http://127.0.0.1:11434") == VENDOR_GENERIC_OPENAI + assert detect_vendor("http://127.0.0.1:1234") == VENDOR_GENERIC_OPENAI assert detect_vendor("http://127.0.0.1:8080") == VENDOR_GENERIC_OPENAI assert detect_vendor("http://localhost:7000/v1") == VENDOR_GENERIC_OPENAI @@ -345,7 +345,9 @@ def test_ollama_reader_maps_show_capabilities_and_tags_are_unknown(): "nomic-embed-text:latest", {"capabilities": ["embedding"]}, ) - tags = ollama.records_from_tags_payload({"models": [{"name": "qwen3:latest"}]}) + tags = ollama.records_from_tags_payload( + {"models": [{"name": "qwen3:latest", "details": {"family": "qwen3"}}]} + ) assert vision is not None assert vision.capability.family == mc.FAMILY_CHAT @@ -360,6 +362,7 @@ def test_ollama_reader_maps_show_capabilities_and_tags_are_unknown(): assert len(tags) == 1 assert tags[0].capability.family == mc.FAMILY_UNKNOWN + assert tags[0].model_family == "qwen3" assert surfaces(tags[0]) == set() @@ -381,7 +384,9 @@ def test_ollama_reader_uses_show_shape_without_architecture_name_matching(): assert record.capability.modalities.input == (mc.MODALITY_TEXT,) assert record.capability.modalities.output == (mc.MODALITY_TEXT,) assert record.capability.capabilities == (mc.CAP_REASONING, mc.CAP_TOOL_CALL) - assert dict(record.capability.limits) == {"context_tokens": 8192} + # Serialized Modelfile text is not reparsed for capability truth. The + # structured native `model_info.*.context_length` field wins. + assert dict(record.capability.limits) == {"context_tokens": 32768} assert surfaces(record) == {"chat"} diff --git a/tests/test_provider_capability_schemas.py b/tests/test_provider_capability_schemas.py new file mode 100644 index 000000000..855f85f5c --- /dev/null +++ b/tests/test_provider_capability_schemas.py @@ -0,0 +1,505 @@ +from src import model_behavior_quirks as quirks +from src import model_capabilities as mc +from src import provider_capability_schemas as pcs +from src.model_capability_readers import ( + anthropic, + chatgpt_subscription, + cohere, + copilot, + generic_openai, + huggingface, + mistral, + records_from_payload, + sglang, +) + + +def test_provider_resolution_order_explicit_then_host_then_native_then_general(): + google_payload = { + "models": [ + { + "name": "models/example", + "supportedGenerationMethods": ["generateContent"], + } + ] + } + + explicit = pcs.resolve_provider(google_payload, provider="openrouter") + host = pcs.resolve_provider(google_payload, base_url="https://api.mistral.ai/v1") + native = pcs.resolve_provider(google_payload) + general = pcs.resolve_provider([{"id": "future-model", "future": {"x": True}}]) + unknown = pcs.resolve_provider({"future": [{"not_an_identity": True}]}) + + assert (explicit.provider_id, explicit.stage) == ("openrouter", pcs.RESOLUTION_EXPLICIT) + assert (host.provider_id, host.stage) == ("mistral", pcs.RESOLUTION_HOST) + assert (native.provider_id, native.stage) == ("google", pcs.RESOLUTION_NATIVE_SHAPE) + assert native.catalog_shape.shape_id == "google.generative-language.models.v1beta" + assert (general.provider_id, general.stage) == ( + pcs.PROVIDER_GENERIC_OPENAI, + pcs.RESOLUTION_GENERAL_SHAPE, + ) + assert (unknown.provider_id, unknown.stage) == ( + pcs.PROVIDER_UNKNOWN, + pcs.RESOLUTION_UNKNOWN, + ) + + +def test_provider_host_matching_rejects_lookalikes_and_does_not_use_ports(): + assert pcs.provider_from_host("https://api.openrouter.ai/v1") == "openrouter" + assert pcs.provider_from_host("https://openrouter.ai.evil.test/v1") == pcs.PROVIDER_UNKNOWN + assert pcs.provider_from_host("http://127.0.0.1:11434") == pcs.PROVIDER_UNKNOWN + assert pcs.provider_from_host("http://127.0.0.1:1234") == pcs.PROVIDER_UNKNOWN + assert pcs.provider_from_host("http://127.0.0.1:8000") == pcs.PROVIDER_UNKNOWN + assert pcs.provider_from_host("http://127.0.0.1:30000") == pcs.PROVIDER_UNKNOWN + + +def test_provider_aliases_collapse_runtime_names_without_url_path_guessing(): + assert pcs.normalize_provider_id("opencode-go") == "opencode" + assert pcs.normalize_provider_id("opencode-zen") == "opencode" + assert pcs.normalize_provider_id("nvidia-nim") == "nvidia" + assert pcs.normalize_provider_id("tgi") == "text_generation_inference" + assert pcs.normalize_provider_id("llama.cpp") == "llamacpp" + assert pcs.normalize_provider_id("Z.AI") == "zai" + + +def test_current_native_catalog_shapes_are_discriminating_and_versioned(): + cases = ( + ( + {"models": [{"key": "local/model", "type": "llm", "capabilities": {"vision": True}}]}, + "lmstudio.models.native.v1", + ), + ( + {"data": [{"id": "legacy", "type": "vlm", "arch": "gemma"}]}, + "lmstudio.models.native.v0", + ), + ( + {"models": [{"name": "local", "digest": "abc", "details": {"family": "qwen3"}}]}, + "ollama.tags.v1", + ), + ( + {"capabilities": ["completion", "vision"], "model_info": {"x.context_length": 4096}}, + "ollama.show.v1", + ), + ( + { + "model_alias": "local", + "default_generation_settings": {"n_ctx": 4096}, + "chat_template_caps": {"supports_tools": True}, + }, + "llamacpp.props.v1", + ), + ( + {"data": [{"id": "mistral", "capabilities": {"completion_chat": True, "vision": False}}]}, + "mistral.models.rich.v1", + ), + ( + { + "data": [ + { + "id": "copilot-model", + "model_picker_enabled": True, + "capabilities": {"supports": {"tool_calls": True}}, + } + ] + }, + "github-copilot.models.v1", + ), + ( + { + "model_path": "org/model", + "tokenizer_path": "org/model", + "is_generation": True, + "has_image_understanding": False, + }, + "sglang.model-info.v2", + ), + ( + { + "object": "list", + "data": [ + { + "id": "served-model", + "object": "model", + "owned_by": "vllm", + "root": "org/model", + "max_model_len": 131072, + "permission": [], + } + ], + }, + "vllm.models.openai.v1", + ), + ( + {"models": [{"slug": "gpt-example", "visibility": "list", "priority": 1}]}, + "chatgpt-subscription.codex-models.v1", + ), + ( + { + "models": [ + { + "name": "command-example", + "endpoints": ["chat"], + "context_length": 131072, + } + ] + }, + "cohere.models.rich.v1", + ), + ( + { + "object": "list", + "data": [ + { + "id": "MiniMax-M2-example", + "object": "model", + "owned_by": "minimax", + } + ], + }, + "minimax.models.identity.v1", + ), + ) + + for payload, expected_shape in cases: + resolution = pcs.resolve_provider(payload) + assert resolution.stage == pcs.RESOLUTION_NATIVE_SHAPE + assert resolution.catalog_shape.shape_id == expected_shape + + +def test_native_shape_detection_rejects_wrong_field_types_before_general_fallback(): + malformed_cohere = pcs.resolve_provider( + {"models": [{"name": "future", "endpoints": "chat", "context_length": 4096}]} + ) + malformed_mistral = pcs.resolve_provider( + {"data": [{"id": "future", "capabilities": ["completion_chat"]}]} + ) + + assert (malformed_cohere.provider_id, malformed_cohere.stage) == ( + pcs.PROVIDER_GENERIC_OPENAI, + pcs.RESOLUTION_GENERAL_SHAPE, + ) + assert (malformed_mistral.provider_id, malformed_mistral.stage) == ( + pcs.PROVIDER_GENERIC_OPENAI, + pcs.RESOLUTION_GENERAL_SHAPE, + ) + + +def test_general_reader_promotes_only_explicit_structural_fields_and_accepts_bare_lists(): + records = generic_openai.records_from_payload( + [ + { + "id": "future-rich-model", + "type": "chat", + "architecture": { + "input_modalities": ["text", "image"], + "output_modalities": ["text"], + }, + "supported_parameters": ["tools", "structured_outputs", "temperature"], + "max_model_len": 131072, + "future_capability": {"may_be_important_later": True}, + }, + { + "id": "vision-reasoning-tools-in-the-name-only", + "description": "Claims every capability in prose", + "type": "image", + "future_capability": True, + }, + ] + ) + + rich, identity_only = records + assert rich.capability.family == mc.FAMILY_CHAT + assert rich.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) + assert rich.capability.capabilities == ( + mc.CAP_TOOL_CALL, + mc.CAP_STRUCTURED_OUTPUT, + mc.CAP_VISION, + ) + assert dict(rich.capability.limits) == {"context_tokens": 131072} + assert [control.control for control in rich.deterministic_controls] == [mc.CONTROL_TEMPERATURE] + assert rich.raw["future_capability"] == {"may_be_important_later": True} + + assert identity_only.capability.family == mc.FAMILY_UNKNOWN + assert identity_only.capability.capabilities == () + assert identity_only.raw["future_capability"] is True + + +def test_general_reader_fails_soft_for_null_and_malformed_envelopes(): + for payload in ( + {"data": None}, + {"models": None}, + {"data": "not-a-list"}, + [None, "model", 42, {"id": None}], + None, + ): + assert generic_openai.records_from_payload(payload) == () + + +def test_mistral_reader_maps_per_model_capabilities_without_provider_wide_inheritance(): + records = mistral.records_from_payload( + { + "data": [ + { + "id": "vision-chat", + "root": "mistral-small", + "capabilities": { + "completion_chat": True, + "function_calling": True, + "vision": True, + "classification": False, + }, + "max_context_length": 32768, + }, + { + "id": "classifier", + "capabilities": { + "completion_chat": False, + "classification": True, + "vision": False, + }, + }, + { + "id": "future-card", + "capabilities": {"future_only": True}, + }, + ] + } + ) + + assert records[0].capability.family == mc.FAMILY_CHAT + assert records[0].capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) + assert records[0].capability.capabilities == (mc.CAP_VISION, mc.CAP_TOOL_CALL) + assert dict(records[0].capability.limits) == {"context_tokens": 32768} + assert records[0].model_family == "mistral-small" + assert records[1].capability.family == mc.FAMILY_CLASSIFICATION + assert records[2].capability.family == mc.FAMILY_UNKNOWN + assert records[2].capability.capabilities == () + + +def test_copilot_reader_uses_picker_and_nested_supports_shape(): + records = copilot.records_from_payload( + { + "data": [ + { + "id": "picker-model", + "model_picker_enabled": True, + "capabilities": {"supports": {"tool_calls": True, "vision": True}}, + "limits": {"max_prompt_tokens": 64000, "max_output_tokens": 8192}, + }, + { + "id": "utility-model", + "model_picker_enabled": False, + "capabilities": {"supports": {}}, + }, + ] + } + ) + + assert records[0].capability.family == mc.FAMILY_CHAT + assert records[0].capability.capabilities == (mc.CAP_TOOL_CALL, mc.CAP_VISION) + assert dict(records[0].capability.limits) == {"input_tokens": 64000, "output_tokens": 8192} + assert records[1].capability.family == mc.FAMILY_UNKNOWN + + +def test_sglang_model_info_is_structural_and_non_generation_stays_unknown(): + generation = sglang.records_from_payload( + { + "model_path": "org/vision-model", + "tokenizer_path": "org/vision-model", + "is_generation": True, + "has_image_understanding": True, + "has_audio_understanding": True, + "model_type": "future_arch", + "preferred_sampling_params": {"temperature": 0.2, "top_p": 0.9}, + } + )[0] + pooling = sglang.records_from_payload( + { + "model_path": "org/pooling-model", + "tokenizer_path": "org/pooling-model", + "is_generation": False, + "has_image_understanding": False, + } + )[0] + + assert generation.capability.family == mc.FAMILY_CHAT + assert generation.capability.modalities.input == ( + mc.MODALITY_TEXT, + mc.MODALITY_IMAGE, + mc.MODALITY_AUDIO, + ) + assert generation.capability.capabilities == (mc.CAP_VISION, mc.CAP_AUDIO_INPUT) + assert [control.control for control in generation.deterministic_controls] == [ + mc.CONTROL_TEMPERATURE, + mc.CONTROL_TOP_P, + ] + assert generation.model_family == "future_arch" + assert pooling.capability.family == mc.FAMILY_UNKNOWN + + +def test_identity_only_catalogs_do_not_claim_model_capability(): + anthropic_record = anthropic.records_from_payload( + { + "data": [ + { + "id": "claude-example", + "type": "model", + "display_name": "Claude Example", + "created_at": "2026-01-01T00:00:00Z", + } + ] + } + )[0] + chatgpt_record = chatgpt_subscription.records_from_payload( + {"models": [{"slug": "gpt-example", "visibility": "list", "priority": 1}]} + )[0] + minimax_record = records_from_payload( + { + "object": "list", + "data": [ + { + "id": "MiniMax-M2-example", + "object": "model", + "owned_by": "minimax", + } + ], + } + )[0] + + assert anthropic_record.capability.family == mc.FAMILY_UNKNOWN + assert chatgpt_record.capability.family == mc.FAMILY_UNKNOWN + assert minimax_record.vendor == "minimax" + assert minimax_record.capability.family == mc.FAMILY_UNKNOWN + + +def test_huggingface_reader_maps_pipeline_tag_as_registry_evidence(): + record = huggingface.records_from_payload( + { + "modelId": "org/vision-model", + "pipeline_tag": "image-text-to-text", + "config": {"model_type": "future_vlm"}, + "tags": ["untrusted-prose-tag"], + } + )[0] + + assert record.capability.family == mc.FAMILY_CHAT + assert record.capability.modalities.input == (mc.MODALITY_TEXT, mc.MODALITY_IMAGE) + assert record.capability.capabilities == (mc.CAP_VISION,) + assert record.capability.source == mc.SOURCE_COOKBOOK_HF + assert record.capability.confidence == mc.CONFIDENCE_REGISTRY + assert record.model_family == "future_vlm" + + +def test_cohere_reader_maps_only_native_endpoint_and_limit_fields(): + chat, ambiguous = cohere.records_from_payload( + { + "models": [ + { + "name": "command-example", + "endpoints": ["chat", "generate"], + "context_length": 131072, + "sampling_defaults": {"temperature": 0.3, "p": 0.9, "k": 40}, + "features": ["unmapped-future-feature"], + }, + { + "name": "multi-endpoint-example", + "endpoints": ["chat", "embed"], + "context_length": 4096, + }, + ] + } + ) + + assert chat.capability.family == mc.FAMILY_CHAT + assert chat.capability.modalities.input == (mc.MODALITY_TEXT,) + assert dict(chat.capability.limits) == {"context_tokens": 131072} + assert [control.control for control in chat.deterministic_controls] == [ + mc.CONTROL_TEMPERATURE, + mc.CONTROL_TOP_P, + mc.CONTROL_TOP_K, + ] + assert chat.raw["features"] == ["unmapped-future-feature"] + assert ambiguous.capability.family == mc.FAMILY_UNKNOWN + + +def test_registry_wrapper_records_resolution_and_preserves_compatible_provider_identity(): + mistral_records = records_from_payload( + { + "data": [ + { + "id": "mistral-model", + "capabilities": {"completion_chat": True, "function_calling": True}, + } + ] + } + ) + together_records = records_from_payload( + [{"id": "served/model", "type": "chat", "supported_parameters": ["tools"]}], + vendor="together", + ) + + assert mistral_records[0].vendor == "mistral" + assert mistral_records[0].provider_schema_id == "mistral" + assert mistral_records[0].catalog_shape_id == "mistral.models.rich.v1" + assert mistral_records[0].provider_resolution == pcs.RESOLUTION_NATIVE_SHAPE + + assert together_records[0].vendor == "together" + assert together_records[0].capability.family == mc.FAMILY_CHAT + assert together_records[0].provider_schema_id == "together" + assert together_records[0].provider_resolution == pcs.RESOLUTION_EXPLICIT + + +def test_reasoning_control_preserves_canonical_and_native_values(): + control = mc.ReasoningControl.build( + mechanism="reasoning_effort", + values=("enabled", "disabled"), + native_values=("high", "medium", "low", "none"), + request_path="reasoning_effort", + response_paths=("choices[].delta.reasoning",), + status="claimed", + source="provider_docs_registry", + confidence="registry", + ) + + assert control.values == (mc.REASONING_CONTROL_VALUE_ON, mc.REASONING_CONTROL_VALUE_OFF) + assert control.native_values == ("high", "medium", "low", "none") + assert mc.ReasoningControl.from_dict(control.to_dict()) == control + + +def test_model_quirks_require_structured_exact_identity_not_name_parsing(): + matching = quirks.matching_quirks( + provider="moonshot", + model_id="kimi-k2.5", + model_family="", + api_dialect=pcs.DIALECT_OPENAI_CHAT, + capabilities=(mc.CAP_REASONING,), + ) + lookalike = quirks.matching_quirks( + provider="moonshot", + model_id="proxy/kimi-k2.5-lookalike", + model_family="", + api_dialect=pcs.DIALECT_OPENAI_CHAT, + capabilities=(mc.CAP_REASONING,), + ) + opus_without_version = quirks.matching_quirks( + provider="anthropic", + model_family="claude-opus", + model_id="claude-opus-4-8-in-name-only", + api_dialect=pcs.DIALECT_ANTHROPIC_MESSAGES, + ) + opus_structured = quirks.matching_quirks( + provider="anthropic", + model_family="claude-opus", + model_version=(4, 8), + api_dialect=pcs.DIALECT_ANTHROPIC_MESSAGES, + ) + + assert {quirk.quirk_id for quirk in matching} == { + "moonshot.kimi-k2.5-k2.6.provider-fixed-temperature", + "moonshot.kimi-k2.5-k2.6.tool-history-reasoning-content", + } + assert lookalike == () + assert opus_without_version == () + assert [quirk.quirk_id for quirk in opus_structured] == [ + "anthropic.claude-opus-4.7-plus.omit-sampling-controls" + ]