From 5dedd262d1eb99c7a544140cde1dc178dcf8e5d5 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:46:09 +0000 Subject: [PATCH] feat(models): log capability normalization diagnostics --- app.py | 11 +++- src/model_capability_readers/__init__.py | 38 +++++++++++++- tests/test_model_capability_diagnostics.py | 59 ++++++++++++++++++++++ 3 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 tests/test_model_capability_diagnostics.py diff --git a/app.py b/app.py index e740ad518..ac90ed789 100644 --- a/app.py +++ b/app.py @@ -85,7 +85,12 @@ import logging.handlers from core.constants import DATA_DIR _root_logger = logging.getLogger() -_root_logger.setLevel(logging.INFO) +_log_level_name = os.getenv("LOG_LEVEL", "INFO").strip().upper() +_log_level = getattr(logging, _log_level_name, logging.INFO) +if not isinstance(_log_level, int): + _log_level_name = "INFO" + _log_level = logging.INFO +_root_logger.setLevel(_log_level) _formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') # Clear existing handlers to avoid duplicates @@ -93,6 +98,7 @@ for _h in list(_root_logger.handlers): _root_logger.removeHandler(_h) _console_h = logging.StreamHandler() +_console_h.setLevel(_log_level) _console_h.setFormatter(_formatter) _root_logger.addHandler(_console_h) @@ -107,6 +113,7 @@ try: _file_h = logging.handlers.RotatingFileHandler( _log_file, maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8" ) + _file_h.setLevel(_log_level) _file_h.setFormatter(_formatter) _root_logger.addHandler(_file_h) except Exception as e: @@ -1278,4 +1285,4 @@ if __name__ == "__main__": bind_host = os.getenv("APP_BIND", "127.0.0.1") bind_port = int(os.getenv("APP_PORT", "7000")) - uvicorn.run(app, host=bind_host, port=bind_port, log_level="info") + uvicorn.run(app, host=bind_host, port=bind_port, log_level=_log_level_name.lower()) diff --git a/src/model_capability_readers/__init__.py b/src/model_capability_readers/__init__.py index 56ff56e13..5a67edf3f 100644 --- a/src/model_capability_readers/__init__.py +++ b/src/model_capability_readers/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from dataclasses import replace from typing import Any @@ -56,6 +57,9 @@ from src.model_capability_readers.base import ( ) +logger = logging.getLogger(__name__) + + READER_MODULES = { VENDOR_GENERIC_OPENAI: generic_openai, VENDOR_OPENAI: openai, @@ -114,7 +118,7 @@ def records_from_payload( ) else: records = reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url) - return tuple( + normalized = tuple( replace( record, provider_source=resolution.provider_source, @@ -123,6 +127,38 @@ def records_from_payload( ) for record in records ) + if logger.isEnabledFor(logging.DEBUG): + families = sorted({record.capability.family for record in normalized}) + features = sorted( + { + feature + for record in normalized + for feature in record.capability.capabilities + } + ) + controls = sorted( + { + control.control + for record in normalized + for control in record.deterministic_controls + if control.control + } + ) + logger.debug( + "[model-capability] normalized: canonical_version=%s provider=%s " + "provider_source=%s catalog_shape=%s fallback=%s records=%d " + "families=%s features=%s controls=%s", + CANONICAL_MODEL_SHAPE_VERSION, + resolution.provider_id, + resolution.provider_source, + resolution.shape_id or "unknown", + resolution.fallback, + len(normalized), + families, + features, + controls, + ) + return normalized __all__ = [ diff --git a/tests/test_model_capability_diagnostics.py b/tests/test_model_capability_diagnostics.py new file mode 100644 index 000000000..3582ae425 --- /dev/null +++ b/tests/test_model_capability_diagnostics.py @@ -0,0 +1,59 @@ +import logging +from pathlib import Path + +from src.model_capability_readers import records_from_payload + + +def test_normalization_debug_log_reports_shape_without_payload_identity(caplog): + payload = { + "data": [ + { + "id": "sensitive-model-id", + "architecture": {"modality": "text+image->text"}, + "supported_parameters": ["tools", "temperature"], + "private_field": "secret-value", + } + ] + } + + with caplog.at_level(logging.DEBUG, logger="src.model_capability_readers"): + records = records_from_payload(payload, vendor="openrouter") + + assert len(records) == 1 + message = caplog.messages[-1] + assert "[model-capability] normalized:" in message + assert "canonical_version=1" in message + assert "provider=openrouter" in message + assert "provider_source=explicit" in message + assert "catalog_shape=openrouter.models.rich.v1" in message + assert "fallback=False" in message + assert "records=1" in message + assert "families=['chat']" in message + assert "features=['tool_call', 'vision']" in message + assert "controls=['temperature']" in message + assert "sensitive-model-id" not in message + assert "secret-value" not in message + + +def test_fallback_debug_log_is_explicit_and_has_no_capability_claims(caplog): + payload = [{"id": "future-model", "capabilities": {"tools": True}}] + + with caplog.at_level(logging.DEBUG, logger="src.model_capability_readers"): + records = records_from_payload(payload, vendor="future-provider") + + assert records[0].capability.capabilities == () + message = caplog.messages[-1] + assert "provider=future_provider" in message + assert "catalog_shape=fallback.models.list.v1" in message + assert "fallback=True" in message + assert "features=[]" in message + + +def test_web_app_logging_uses_existing_log_level_environment_toggle(): + source = (Path(__file__).resolve().parents[1] / "app.py").read_text(encoding="utf-8") + + assert 'os.getenv("LOG_LEVEL", "INFO")' in source + assert "_root_logger.setLevel(_log_level)" in source + assert "_console_h.setLevel(_log_level)" in source + assert "_file_h.setLevel(_log_level)" in source + assert "log_level=_log_level_name.lower()" in source