feat(models): log capability normalization diagnostics

This commit is contained in:
RaresKeY 2026-07-17 12:46:09 +00:00
parent ef3409ed53
commit 5dedd262d1
3 changed files with 105 additions and 3 deletions

11
app.py
View file

@ -85,7 +85,12 @@ import logging.handlers
from core.constants import DATA_DIR
_root_logger = logging.getLogger()
_root_logger.setLevel(logging.INFO)
_log_level_name = os.getenv("LOG_LEVEL", "INFO").strip().upper()
_log_level = getattr(logging, _log_level_name, logging.INFO)
if not isinstance(_log_level, int):
_log_level_name = "INFO"
_log_level = logging.INFO
_root_logger.setLevel(_log_level)
_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# Clear existing handlers to avoid duplicates
@ -93,6 +98,7 @@ for _h in list(_root_logger.handlers):
_root_logger.removeHandler(_h)
_console_h = logging.StreamHandler()
_console_h.setLevel(_log_level)
_console_h.setFormatter(_formatter)
_root_logger.addHandler(_console_h)
@ -107,6 +113,7 @@ try:
_file_h = logging.handlers.RotatingFileHandler(
_log_file, maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8"
)
_file_h.setLevel(_log_level)
_file_h.setFormatter(_formatter)
_root_logger.addHandler(_file_h)
except Exception as e:
@ -1278,4 +1285,4 @@ if __name__ == "__main__":
bind_host = os.getenv("APP_BIND", "127.0.0.1")
bind_port = int(os.getenv("APP_PORT", "7000"))
uvicorn.run(app, host=bind_host, port=bind_port, log_level="info")
uvicorn.run(app, host=bind_host, port=bind_port, log_level=_log_level_name.lower())

View file

@ -2,6 +2,7 @@
from __future__ import annotations
import logging
from dataclasses import replace
from typing import Any
@ -56,6 +57,9 @@ from src.model_capability_readers.base import (
)
logger = logging.getLogger(__name__)
READER_MODULES = {
VENDOR_GENERIC_OPENAI: generic_openai,
VENDOR_OPENAI: openai,
@ -114,7 +118,7 @@ def records_from_payload(
)
else:
records = reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url)
return tuple(
normalized = tuple(
replace(
record,
provider_source=resolution.provider_source,
@ -123,6 +127,38 @@ def records_from_payload(
)
for record in records
)
if logger.isEnabledFor(logging.DEBUG):
families = sorted({record.capability.family for record in normalized})
features = sorted(
{
feature
for record in normalized
for feature in record.capability.capabilities
}
)
controls = sorted(
{
control.control
for record in normalized
for control in record.deterministic_controls
if control.control
}
)
logger.debug(
"[model-capability] normalized: canonical_version=%s provider=%s "
"provider_source=%s catalog_shape=%s fallback=%s records=%d "
"families=%s features=%s controls=%s",
CANONICAL_MODEL_SHAPE_VERSION,
resolution.provider_id,
resolution.provider_source,
resolution.shape_id or "unknown",
resolution.fallback,
len(normalized),
families,
features,
controls,
)
return normalized
__all__ = [

View file

@ -0,0 +1,59 @@
import logging
from pathlib import Path
from src.model_capability_readers import records_from_payload
def test_normalization_debug_log_reports_shape_without_payload_identity(caplog):
payload = {
"data": [
{
"id": "sensitive-model-id",
"architecture": {"modality": "text+image->text"},
"supported_parameters": ["tools", "temperature"],
"private_field": "secret-value",
}
]
}
with caplog.at_level(logging.DEBUG, logger="src.model_capability_readers"):
records = records_from_payload(payload, vendor="openrouter")
assert len(records) == 1
message = caplog.messages[-1]
assert "[model-capability] normalized:" in message
assert "canonical_version=1" in message
assert "provider=openrouter" in message
assert "provider_source=explicit" in message
assert "catalog_shape=openrouter.models.rich.v1" in message
assert "fallback=False" in message
assert "records=1" in message
assert "families=['chat']" in message
assert "features=['tool_call', 'vision']" in message
assert "controls=['temperature']" in message
assert "sensitive-model-id" not in message
assert "secret-value" not in message
def test_fallback_debug_log_is_explicit_and_has_no_capability_claims(caplog):
payload = [{"id": "future-model", "capabilities": {"tools": True}}]
with caplog.at_level(logging.DEBUG, logger="src.model_capability_readers"):
records = records_from_payload(payload, vendor="future-provider")
assert records[0].capability.capabilities == ()
message = caplog.messages[-1]
assert "provider=future_provider" in message
assert "catalog_shape=fallback.models.list.v1" in message
assert "fallback=True" in message
assert "features=[]" in message
def test_web_app_logging_uses_existing_log_level_environment_toggle():
source = (Path(__file__).resolve().parents[1] / "app.py").read_text(encoding="utf-8")
assert 'os.getenv("LOG_LEVEL", "INFO")' in source
assert "_root_logger.setLevel(_log_level)" in source
assert "_console_h.setLevel(_log_level)" in source
assert "_file_h.setLevel(_log_level)" in source
assert "log_level=_log_level_name.lower()" in source