feat: add compose deliverable, scenario definitions, eval stub, and CI

This commit is contained in:
Justin Visser 2026-08-09 20:59:58 +02:00
parent ef9e102b05
commit 7c3934daee
8 changed files with 148 additions and 0 deletions

23
eval/run_eval.py Normal file
View file

@ -0,0 +1,23 @@
"""Eval harness entry point.
Fixture mode (default, key-free): byte-snapshots of deterministic pipeline
stages against recorded cassettes. Live mode: property assertions only
LLM output is constrained, not pinned.
"""
from pathlib import Path
import yaml
SCENARIOS_FILE = Path(__file__).parent / "scenarios.yaml"
def load_scenarios() -> dict:
"""Load the shared scenario definitions."""
with SCENARIOS_FILE.open() as handle:
return yaml.safe_load(handle)
if __name__ == "__main__":
scenarios = load_scenarios()
print(f"{len(scenarios['scenarios'])} scenarios, {len(scenarios['refinements'])} refinements")