test: add scenario eval runner, recorded fixtures, and baseline comparison

This commit is contained in:
Justin Visser 2026-08-10 22:21:04 +02:00
parent 3dc1af5f0c
commit 0664cc2d27
38 changed files with 6986 additions and 5 deletions

View file

@ -6,8 +6,10 @@ requires-python = "==3.13.*"
dependencies = [
"anthropic==0.121.0",
"fastapi>=0.116",
"httpx>=0.28",
"httpx2>=2.10",
"pydantic-settings>=2.10",
"pyyaml>=6.0",
"structlog>=25.4",
"uvicorn[standard]>=0.35",
]
@ -17,6 +19,7 @@ dev = [
"mypy>=1.17",
"pytest>=8.4",
"ruff>=0.12",
"types-pyyaml>=6.0",
]
[tool.ruff]
@ -36,4 +39,5 @@ module = ["app.domain.*", "app.ports.*", "app.pipeline.*"]
strict = true
[tool.pytest.ini_options]
testpaths = ["tests"]
pythonpath = [".", "../eval"]
testpaths = ["tests", "../eval/tests"]