mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-08-18 14:28:14 +00:00
feat(models): log capability normalization diagnostics
This commit is contained in:
parent
ef3409ed53
commit
5dedd262d1
3 changed files with 105 additions and 3 deletions
11
app.py
11
app.py
|
|
@ -85,7 +85,12 @@ import logging.handlers
|
|||
from core.constants import DATA_DIR
|
||||
|
||||
_root_logger = logging.getLogger()
|
||||
_root_logger.setLevel(logging.INFO)
|
||||
_log_level_name = os.getenv("LOG_LEVEL", "INFO").strip().upper()
|
||||
_log_level = getattr(logging, _log_level_name, logging.INFO)
|
||||
if not isinstance(_log_level, int):
|
||||
_log_level_name = "INFO"
|
||||
_log_level = logging.INFO
|
||||
_root_logger.setLevel(_log_level)
|
||||
_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
|
||||
# Clear existing handlers to avoid duplicates
|
||||
|
|
@ -93,6 +98,7 @@ for _h in list(_root_logger.handlers):
|
|||
_root_logger.removeHandler(_h)
|
||||
|
||||
_console_h = logging.StreamHandler()
|
||||
_console_h.setLevel(_log_level)
|
||||
_console_h.setFormatter(_formatter)
|
||||
_root_logger.addHandler(_console_h)
|
||||
|
||||
|
|
@ -107,6 +113,7 @@ try:
|
|||
_file_h = logging.handlers.RotatingFileHandler(
|
||||
_log_file, maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8"
|
||||
)
|
||||
_file_h.setLevel(_log_level)
|
||||
_file_h.setFormatter(_formatter)
|
||||
_root_logger.addHandler(_file_h)
|
||||
except Exception as e:
|
||||
|
|
@ -1278,4 +1285,4 @@ if __name__ == "__main__":
|
|||
bind_host = os.getenv("APP_BIND", "127.0.0.1")
|
||||
bind_port = int(os.getenv("APP_PORT", "7000"))
|
||||
|
||||
uvicorn.run(app, host=bind_host, port=bind_port, log_level="info")
|
||||
uvicorn.run(app, host=bind_host, port=bind_port, log_level=_log_level_name.lower())
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import replace
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -56,6 +57,9 @@ from src.model_capability_readers.base import (
|
|||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
READER_MODULES = {
|
||||
VENDOR_GENERIC_OPENAI: generic_openai,
|
||||
VENDOR_OPENAI: openai,
|
||||
|
|
@ -114,7 +118,7 @@ def records_from_payload(
|
|||
)
|
||||
else:
|
||||
records = reader.records_from_payload(payload, endpoint_id=endpoint_id, base_url=base_url)
|
||||
return tuple(
|
||||
normalized = tuple(
|
||||
replace(
|
||||
record,
|
||||
provider_source=resolution.provider_source,
|
||||
|
|
@ -123,6 +127,38 @@ def records_from_payload(
|
|||
)
|
||||
for record in records
|
||||
)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
families = sorted({record.capability.family for record in normalized})
|
||||
features = sorted(
|
||||
{
|
||||
feature
|
||||
for record in normalized
|
||||
for feature in record.capability.capabilities
|
||||
}
|
||||
)
|
||||
controls = sorted(
|
||||
{
|
||||
control.control
|
||||
for record in normalized
|
||||
for control in record.deterministic_controls
|
||||
if control.control
|
||||
}
|
||||
)
|
||||
logger.debug(
|
||||
"[model-capability] normalized: canonical_version=%s provider=%s "
|
||||
"provider_source=%s catalog_shape=%s fallback=%s records=%d "
|
||||
"families=%s features=%s controls=%s",
|
||||
CANONICAL_MODEL_SHAPE_VERSION,
|
||||
resolution.provider_id,
|
||||
resolution.provider_source,
|
||||
resolution.shape_id or "unknown",
|
||||
resolution.fallback,
|
||||
len(normalized),
|
||||
families,
|
||||
features,
|
||||
controls,
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
|
|
|||
59
tests/test_model_capability_diagnostics.py
Normal file
59
tests/test_model_capability_diagnostics.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from src.model_capability_readers import records_from_payload
|
||||
|
||||
|
||||
def test_normalization_debug_log_reports_shape_without_payload_identity(caplog):
|
||||
payload = {
|
||||
"data": [
|
||||
{
|
||||
"id": "sensitive-model-id",
|
||||
"architecture": {"modality": "text+image->text"},
|
||||
"supported_parameters": ["tools", "temperature"],
|
||||
"private_field": "secret-value",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
with caplog.at_level(logging.DEBUG, logger="src.model_capability_readers"):
|
||||
records = records_from_payload(payload, vendor="openrouter")
|
||||
|
||||
assert len(records) == 1
|
||||
message = caplog.messages[-1]
|
||||
assert "[model-capability] normalized:" in message
|
||||
assert "canonical_version=1" in message
|
||||
assert "provider=openrouter" in message
|
||||
assert "provider_source=explicit" in message
|
||||
assert "catalog_shape=openrouter.models.rich.v1" in message
|
||||
assert "fallback=False" in message
|
||||
assert "records=1" in message
|
||||
assert "families=['chat']" in message
|
||||
assert "features=['tool_call', 'vision']" in message
|
||||
assert "controls=['temperature']" in message
|
||||
assert "sensitive-model-id" not in message
|
||||
assert "secret-value" not in message
|
||||
|
||||
|
||||
def test_fallback_debug_log_is_explicit_and_has_no_capability_claims(caplog):
|
||||
payload = [{"id": "future-model", "capabilities": {"tools": True}}]
|
||||
|
||||
with caplog.at_level(logging.DEBUG, logger="src.model_capability_readers"):
|
||||
records = records_from_payload(payload, vendor="future-provider")
|
||||
|
||||
assert records[0].capability.capabilities == ()
|
||||
message = caplog.messages[-1]
|
||||
assert "provider=future_provider" in message
|
||||
assert "catalog_shape=fallback.models.list.v1" in message
|
||||
assert "fallback=True" in message
|
||||
assert "features=[]" in message
|
||||
|
||||
|
||||
def test_web_app_logging_uses_existing_log_level_environment_toggle():
|
||||
source = (Path(__file__).resolve().parents[1] / "app.py").read_text(encoding="utf-8")
|
||||
|
||||
assert 'os.getenv("LOG_LEVEL", "INFO")' in source
|
||||
assert "_root_logger.setLevel(_log_level)" in source
|
||||
assert "_console_h.setLevel(_log_level)" in source
|
||||
assert "_file_h.setLevel(_log_level)" in source
|
||||
assert "log_level=_log_level_name.lower()" in source
|
||||
Loading…
Add table
Reference in a new issue