test: add scenario eval runner, recorded fixtures, and baseline comparison

.github/workflows/ci.yml

@@ -15,9 +15,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v5
       - run: uv sync --frozen
-      - run: uv run ruff check .
+      - run: uv run ruff check . ../eval
-      - run: uv run ruff format --check .
+      - run: uv run ruff format --check . ../eval
-      - run: uv run mypy app
+      - run: uv run mypy app tests ../eval
       - run: uv run pytest
 
   frontend:

.gitignore

@@ -18,3 +18,8 @@ frontend/dist/
 
 # eval: raw (unredacted) recordings never enter the repo
 eval/fixtures/raw/
+eval/reports/
+
+# eval outputs (reports, baseline snapshots) are generated, never shipped
+eval/reports/
+eval/snapshots/

backend/app/main.py

@@ -4,6 +4,7 @@ from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
 from pathlib import Path
 
+import httpx
 import httpx2
 import structlog
 from anthropic import AsyncAnthropic
@@ -28,6 +29,7 @@ FRONTEND_DIST = Path(__file__).parent / "static"
 def create_app(
     application_settings: Settings | None = None,
     http_transport: httpx2.AsyncBaseTransport | None = None,
+    anthropic_http_client: httpx.AsyncClient | None = None,
 ) -> FastAPI:
     """Build the FastAPI app: API routes plus the built SPA on one port."""
     active_settings = application_settings or settings
@@ -48,6 +50,7 @@ def create_app(
             anthropic_client = AsyncAnthropic(
                 api_key=active_settings.anthropic_api_key or "unused-demo-key",
                 timeout=active_settings.llm_timeout_seconds,
+                http_client=anthropic_http_client,
             )
             application.state.anthropic = anthropic_client
             application.state.recommendation_pipeline = RecommendationPipeline(

backend/pyproject.toml

@@ -6,8 +6,10 @@ requires-python = "==3.13.*"
 dependencies = [
     "anthropic==0.121.0",
     "fastapi>=0.116",
+    "httpx>=0.28",
     "httpx2>=2.10",
     "pydantic-settings>=2.10",
+    "pyyaml>=6.0",
     "structlog>=25.4",
     "uvicorn[standard]>=0.35",
 ]
@@ -17,6 +19,7 @@ dev = [
     "mypy>=1.17",
     "pytest>=8.4",
     "ruff>=0.12",
+    "types-pyyaml>=6.0",
 ]
 
 [tool.ruff]
@@ -36,4 +39,5 @@ module = ["app.domain.*", "app.ports.*", "app.pipeline.*"]
 strict = true
 
 [tool.pytest.ini_options]
-testpaths = ["tests"]
+pythonpath = [".", "../eval"]
+testpaths = ["tests", "../eval/tests"]

backend/tests/test_health.py

@@ -26,4 +26,4 @@ def test_anthropic_client_uses_configured_timeout(monkeypatch: pytest.MonkeyPatc
         response = client.get("/api/health")
 
     assert response.status_code == 200
-    constructor.assert_called_once_with(api_key="unused-demo-key", timeout=42.0)
+    constructor.assert_called_once_with(api_key="unused-demo-key", timeout=42.0, http_client=None)

backend/uv.lock

@@ -132,8 +132,10 @@ source = { virtual = "." }
 dependencies = [
     { name = "anthropic" },
     { name = "fastapi" },
+    { name = "httpx" },
     { name = "httpx2" },
     { name = "pydantic-settings" },
+    { name = "pyyaml" },
     { name = "structlog" },
     { name = "uvicorn", extra = ["standard"] },
 ]
@@ -143,14 +145,17 @@ dev = [
     { name = "mypy" },
     { name = "pytest" },
     { name = "ruff" },
+    { name = "types-pyyaml" },
 ]
 
 [package.metadata]
 requires-dist = [
     { name = "anthropic", specifier = "==0.121.0" },
     { name = "fastapi", specifier = ">=0.116" },
+    { name = "httpx", specifier = ">=0.28" },
     { name = "httpx2", specifier = ">=2.10" },
     { name = "pydantic-settings", specifier = ">=2.10" },
+    { name = "pyyaml", specifier = ">=6.0" },
     { name = "structlog", specifier = ">=25.4" },
     { name = "uvicorn", extras = ["standard"], specifier = ">=0.35" },
 ]
@@ -160,6 +165,7 @@ dev = [
     { name = "mypy", specifier = ">=1.17" },
     { name = "pytest", specifier = ">=8.4" },
     { name = "ruff", specifier = ">=0.12" },
+    { name = "types-pyyaml", specifier = ">=6.0" },
 ]
 
 [[package]]
@@ -580,6 +586,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" },
 ]
 
+[[package]]
+name = "types-pyyaml"
+version = "6.0.12.20260724"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/3f/6f/a28f44bcd56bebed42b028a2894c79853e2f5e6b5279e633cb3f287a05e7/types_pyyaml-6.0.12.20260724.tar.gz", hash = "sha256:3c1ce1bb73cd5ec02e90390c2b1f00e810d241d8825fd73ff359696839271b6b", size = 17893, upload-time = "2026-07-24T04:58:43.453Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/8b/42/0337fefc615e20ee55d1c8f71b774a9b2b734a04669139c20753b27a2a3a/types_pyyaml-6.0.12.20260724-py3-none-any.whl", hash = "sha256:d57db930a4b2efbc57cf430ec8882765d246929432fa253092f383902329a453", size = 20312, upload-time = "2026-07-24T04:58:42.486Z" },
+]
+
 [[package]]
 name = "typing-extensions"
 version = "4.16.0"

docs/logboek.md

@@ -370,3 +370,29 @@ Wat ik heb laten vallen of uitgesteld:
 
 - Een repair loop die gefaalde kandidaten opnieuw aan het model
   voorlegt; blijft staan als potentiele vervolgstap.
+
+### Evaluatie en fixtures
+
+Wat ik deed:
+
+- Een eval-script dat de app de 8 standaardvragen stelt en de antwoorden
+  controleert: komen de events in de juiste volgorde, zijn alle tracks
+  uniek, heeft elke track een reden, kwam de eerste kaart binnen het
+  tijdsbudget, en gaat het antwoord echt over wat er gevraagd werd.
+- Dezelfde 8 vragen ook aan de kale Spotify search gesteld; de
+  vergelijking tussen die twee wordt de tabel in de README.
+- Elk scenario 1 keer opgenomen tegen de echte API's en opgeslagen als
+  cassettes; demo mode speelt die af, dus de app werkt straks ook zonder
+  keys. Persoonlijke data gaat er bij het opnemen uit (nepprofiel, geen
+  tokens); de ruwe opnames blijven buiten git.
+- Dit draait ook in CI, zonder keys.
+
+Waarom:
+
+- De claim dat de LLM-laag iets toevoegt moet meetbaar zijn, niet
+  beweerd. En de reviewer moet de app kunnen starten zonder eigen keys.
+
+Wat ik heb laten vallen of uitgesteld:
+
+- Byte-snapshots per pipeline-stap; de checks hierboven en de cassettes
+  zijn nu het bewijs.

eval/baseline.py

@@ -0,0 +1,114 @@
+"""Record bare Spotify search results and render side-by-side comparisons."""
+
+import json
+from pathlib import Path
+
+import httpx
+from scenario import Scenario
+
+TOKEN_URL = "https://accounts.spotify.com/api/token"
+SEARCH_URL = "https://api.spotify.com/v1/search"
+
+
+async def record_baselines(
+    scenarios: tuple[Scenario, ...],
+    client_id: str,
+    client_secret: str,
+    snapshot_root: Path,
+) -> None:
+    """Snapshot the top ten results from bare searches for every query."""
+    snapshot_root.mkdir(parents=True, exist_ok=True)
+    async with httpx.AsyncClient(timeout=20.0) as client:
+        token_response = await client.post(
+            TOKEN_URL,
+            data={"grant_type": "client_credentials"},
+            auth=(client_id, client_secret),
+        )
+        token_response.raise_for_status()
+        access_token = token_response.json()["access_token"]
+        for scenario in scenarios:
+            response = await client.get(
+                SEARCH_URL,
+                params={"q": scenario.query, "type": "track", "limit": 10},
+                headers={"Authorization": f"Bearer {access_token}"},
+            )
+            response.raise_for_status()
+            tracks = response.json().get("tracks", {}).get("items", [])[:10]
+            _write_json(
+                snapshot_root / f"{scenario.key}.json",
+                {"key": scenario.key, "query": scenario.query, "tracks": tracks},
+            )
+
+
+def write_comparison(
+    scenarios: tuple[Scenario, ...],
+    snapshot_root: Path,
+    report_root: Path,
+    top_n: int,
+) -> Path:
+    """Write a markdown comparison for scenarios with both result arms."""
+    report_root.mkdir(parents=True, exist_ok=True)
+    lines = ["# Pipeline and bare search comparison", ""]
+    compared = 0
+    for scenario in scenarios:
+        baseline_path = snapshot_root / f"{scenario.key}.json"
+        pipeline_path = report_root / f"{scenario.key}.json"
+        if not baseline_path.exists() or not pipeline_path.exists():
+            continue
+        baseline = json.loads(baseline_path.read_text(encoding="ascii"))
+        pipeline = json.loads(pipeline_path.read_text(encoding="ascii"))
+        lines.extend(_comparison_section(scenario, pipeline, baseline, top_n))
+        compared += 1
+    if compared == 0:
+        lines.extend(["No scenario has results from both arms yet.", ""])
+    output_path = report_root / "baseline-comparison.md"
+    output_path.write_text("\n".join(lines), encoding="utf-8")
+    return output_path
+
+
+def _comparison_section(
+    scenario: Scenario,
+    pipeline: object,
+    baseline: object,
+    top_n: int,
+) -> list[str]:
+    pipeline_tracks = pipeline.get("tracks", []) if isinstance(pipeline, dict) else []
+    baseline_tracks = baseline.get("tracks", []) if isinstance(baseline, dict) else []
+    lines = [f"## {scenario.chip}", "", "| Rank | Pipeline | Bare search |", "| ---: | --- | --- |"]
+    for index in range(top_n):
+        pipeline_label = _pipeline_label(pipeline_tracks, index)
+        baseline_label = _spotify_label(baseline_tracks, index)
+        lines.append(f"| {index + 1} | {pipeline_label} | {baseline_label} |")
+    lines.append("")
+    return lines
+
+
+def _pipeline_label(tracks: object, index: int) -> str:
+    if not isinstance(tracks, list) or index >= len(tracks):
+        return ""
+    event = tracks[index]
+    track = event.get("track") if isinstance(event, dict) else None
+    return _track_label(track, "title")
+
+
+def _spotify_label(tracks: object, index: int) -> str:
+    if not isinstance(tracks, list) or index >= len(tracks):
+        return ""
+    return _track_label(tracks[index], "name")
+
+
+def _track_label(track: object, title_key: str) -> str:
+    if not isinstance(track, dict):
+        return ""
+    artists = track.get("artists", [])
+    names = [
+        artist.get("name", "") if isinstance(artist, dict) else str(artist) for artist in artists
+    ]
+    return f"{track.get(title_key, '')} by {', '.join(names)}"
+
+
+def _write_json(path: Path, payload: object) -> None:
+    path.write_text(
+        json.dumps(payload, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
+        encoding="ascii",
+    )

eval/live.py

@@ -0,0 +1,284 @@
+"""Exercise the live recommendation stream and assert behavior properties."""
+
+import asyncio
+import json
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import httpx
+from scenario import Scenario
+from wire import validate_event
+
+FACET_TERMS: dict[tuple[str, str], tuple[str, ...]] = {
+    ("activity", "coding"): ("coding", "programming"),
+    ("activity", "workout"): ("workout", "gym"),
+    ("familiarity", "mix"): ("mix", "familiar", "new", "discover"),
+    ("familiarity", "new"): ("new", "unfamiliar", "discover", "surprise"),
+    ("language", "nl"): ("dutch", "nederlandse"),
+    ("era", "1990s"): ("1990", "nineties", "90s"),
+    ("mood", "energetic"): ("energetic", "high-energy", "high energy"),
+    ("mood", "relaxed"): ("relaxed", "laid-back", "laid back", "couch-friendly"),
+    ("mood", "subdued"): ("subdued", "background", "unobtrusive", "not too present"),
+}
+
+
+@dataclass(frozen=True)
+class LiveLimits:
+    """Quality and latency thresholds for one live request."""
+
+    minimum_tracks: int
+    maximum_tracks: int
+    minimum_artists: int
+    first_track_budget_ms: int
+    total_budget_ms: int
+
+
+@dataclass
+class ScenarioResult:
+    """Serializable observations and property failures for one scenario."""
+
+    scenario: Scenario
+    events: list[dict[str, object]] = field(default_factory=list)
+    failures: list[str] = field(default_factory=list)
+    checks: dict[str, bool] = field(default_factory=dict)
+    first_track_ms: int | None = None
+    total_ms: int | None = None
+
+    @property
+    def tracks(self) -> list[dict[str, object]]:
+        """Return track event payloads in streamed order."""
+        return [event for event in self.events if event.get("type") == "track"]
+
+    @property
+    def passed(self) -> bool:
+        """Return whether all required properties passed."""
+        return not self.failures
+
+    def as_report(self) -> dict[str, object]:
+        """Render the stable JSON report shape."""
+        return {
+            "key": self.scenario.key,
+            "query": self.scenario.query,
+            "after": self.scenario.after,
+            "status": "passed" if self.passed else "failed",
+            "first_track_ms": self.first_track_ms,
+            "total_ms": self.total_ms,
+            "checks": self.checks,
+            "failures": self.failures,
+            "events": self.events,
+            "tracks": self.tracks,
+        }
+
+
+async def run_live_scenarios(
+    scenarios: tuple[Scenario, ...],
+    base_url: str,
+    report_root: Path,
+    limits: LiveLimits,
+) -> tuple[ScenarioResult, ...]:
+    """Run all scenarios sequentially and write one report for each."""
+    report_root.mkdir(parents=True, exist_ok=True)
+    results_by_key: dict[str, ScenarioResult] = {}
+    async with httpx.AsyncClient(base_url=base_url, timeout=None) as client:
+        for scenario in scenarios:
+            parent = results_by_key.get(scenario.after) if scenario.after is not None else None
+            result = await run_live_scenario(client, scenario, limits, parent)
+            _write_report(report_root / f"{scenario.key}.json", result.as_report())
+            results_by_key[scenario.key] = result
+    return tuple(results_by_key[scenario.key] for scenario in scenarios)
+
+
+async def run_live_scenario(
+    client: httpx.AsyncClient,
+    scenario: Scenario,
+    limits: LiveLimits,
+    parent: ScenarioResult | None = None,
+) -> ScenarioResult:
+    """Run one strict NDJSON request and evaluate its properties."""
+    result = ScenarioResult(scenario)
+    if scenario.after is not None and (parent is None or not parent.tracks):
+        result.failures.append("parent scenario did not produce usable recommendations")
+        return result
+    payload = build_request(scenario, parent)
+    started_at = time.monotonic()
+    try:
+        async with asyncio.timeout(limits.total_budget_ms / 1000):
+            async with client.stream("POST", "/api/recommendations", json=payload) as response:
+                if response.status_code != 200:
+                    body = (await response.aread()).decode("utf-8", errors="replace")
+                    result.failures.append(f"HTTP {response.status_code}: {body}")
+                else:
+                    await _consume_response(response, result, started_at)
+    except TimeoutError:
+        result.failures.append("stream exceeded the configured total latency budget")
+    except (httpx.HTTPError, ValueError) as error:
+        result.failures.append(f"stream failed: {error}")
+    result.total_ms = _elapsed_ms(started_at)
+    _evaluate_result(result, limits)
+    return result
+
+
+def build_request(scenario: Scenario, parent: ScenarioResult | None) -> dict[str, object]:
+    """Build the frozen request shape, including client-owned turn context."""
+    payload: dict[str, object] = {"schema_version": 1, "query": scenario.query}
+    if parent is None:
+        return payload
+    prior = []
+    labels = []
+    for event in parent.tracks:
+        track = event["track"]
+        if not isinstance(track, dict):
+            continue
+        rank = event["rank"]
+        artists = track["artists"]
+        prior_artists = list(artists)[:10] if isinstance(artists, list) else []
+        prior.append(
+            {
+                "rank": rank,
+                "track_id": track["id"],
+                "title": track["title"],
+                "artists": prior_artists,
+            }
+        )
+        labels.append(
+            f"{rank}. {track['title']} by {', '.join(str(name) for name in prior_artists)}"
+        )
+    payload["history"] = [
+        {"role": "user", "content": parent.scenario.query},
+        {"role": "assistant", "content": "\n".join(labels)[:2000]},
+    ]
+    payload["prior_recommendations"] = prior
+    return payload
+
+
+def _evaluate_result(result: ScenarioResult, limits: LiveLimits) -> None:
+    event_types = [str(event.get("type")) for event in result.events]
+    terminal_type = event_types[-1] if event_types else None
+    order_is_valid = (
+        bool(event_types)
+        and event_types[0] == "metadata"
+        and terminal_type in {"done", "error"}
+        and all(event_type in {"track", "warning"} for event_type in event_types[1:-1])
+        and event_types.count("metadata") == 1
+    )
+    _record_check(result, "event_order", order_is_valid, "event order is invalid")
+    if terminal_type == "error":
+        terminal = result.events[-1]
+        result.failures.append(f"terminal error: {terminal.get('code', 'unknown')}")
+
+    tracks = result.tracks
+    if terminal_type == "done":
+        done_count = result.events[-1].get("track_count")
+        _record_check(
+            result,
+            "done_track_count",
+            done_count == len(tracks),
+            "done event track count did not match streamed tracks",
+        )
+    track_ids = [_track_field(event, "id") for event in tracks]
+    _record_check(
+        result,
+        "unique_track_ids",
+        len(track_ids) == len(set(track_ids)),
+        "duplicate track ids were returned",
+    )
+    justifications = [event.get("justification") for event in tracks]
+    _record_check(
+        result,
+        "non_empty_justifications",
+        all(isinstance(value, str) and bool(value.strip()) for value in justifications),
+        "a track justification was empty",
+    )
+    _record_check(
+        result,
+        "track_count",
+        limits.minimum_tracks <= len(tracks) <= limits.maximum_tracks,
+        f"track count {len(tracks)} is outside configured bounds",
+    )
+    artists = {str(artist).casefold() for event in tracks for artist in _track_artists(event)}
+    _record_check(
+        result,
+        "distinct_artists",
+        len(artists) >= limits.minimum_artists,
+        f"distinct artist count {len(artists)} is below configured minimum",
+    )
+    _record_check(
+        result,
+        "first_track_latency",
+        result.first_track_ms is not None and result.first_track_ms <= limits.first_track_budget_ms,
+        "time to first track exceeded the configured budget",
+    )
+    _record_check(
+        result,
+        "total_latency",
+        result.total_ms is not None and result.total_ms <= limits.total_budget_ms,
+        "total latency exceeded the configured budget",
+    )
+    _evaluate_facets(result)
+
+
+def _evaluate_facets(result: ScenarioResult) -> None:
+    metadata = next(
+        (event for event in result.events if event.get("type") == "metadata"),
+        None,
+    )
+    summary = str(metadata.get("intent_summary", "")).casefold() if metadata else ""
+    for name, value in result.scenario.facets.items():
+        if not isinstance(value, str):
+            continue
+        terms = FACET_TERMS.get((name, value), (value.casefold(),))
+        _record_check(
+            result,
+            f"facet_{name}",
+            any(term in summary for term in terms),
+            f"intent summary did not express {name}={value}",
+        )
+
+
+async def _consume_response(
+    response: httpx.Response,
+    result: ScenarioResult,
+    started_at: float,
+) -> None:
+    content_type = response.headers.get("content-type", "").split(";", 1)[0]
+    if content_type != "application/x-ndjson":
+        result.failures.append(f"unexpected content type: {content_type or 'missing'}")
+        return
+    async for line in response.aiter_lines():
+        if not line:
+            result.failures.append("NDJSON stream contained an empty line")
+            continue
+        event = validate_event(line)
+        event_payload = event.model_dump(mode="json")
+        result.events.append(event_payload)
+        if event_payload["type"] == "track" and result.first_track_ms is None:
+            result.first_track_ms = _elapsed_ms(started_at)
+
+
+def _record_check(result: ScenarioResult, name: str, passed: bool, failure: str) -> None:
+    result.checks[name] = passed
+    if not passed:
+        result.failures.append(failure)
+
+
+def _track_field(event: dict[str, object], key: str) -> str:
+    track = event.get("track")
+    return str(track.get(key, "")) if isinstance(track, dict) else ""
+
+
+def _track_artists(event: dict[str, object]) -> list[object]:
+    track = event.get("track")
+    artists = track.get("artists") if isinstance(track, dict) else None
+    return artists if isinstance(artists, list) else []
+
+
+def _elapsed_ms(started_at: float) -> int:
+    return round((time.monotonic() - started_at) * 1000)
+
+
+def _write_report(path: Path, report: dict[str, object]) -> None:
+    path.write_text(
+        json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
+        encoding="ascii",
+    )

eval/record_fixtures.py

@@ -0,0 +1,142 @@
+"""Record one scenario through a locally constructed application."""
+
+import argparse
+import os
+import sys
+from importlib import import_module
+from pathlib import Path
+from typing import cast
+
+import httpx
+import httpx2
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+from live import ScenarioResult, build_request
+from recording import Httpx2RecordingTransport, HttpxRecordingTransport, write_cassettes
+from scenario import SCENARIO_PATH, Scenario, load_scenarios
+from wire import BACKEND_ROOT, validate_event
+
+EVAL_ROOT = Path(__file__).parent
+
+
+def main() -> int:
+    """Record the selected scenario and persist raw plus safe cassettes."""
+    arguments = _parse_arguments()
+    scenarios = load_scenarios(arguments.scenarios)
+    selected = next((item for item in scenarios if item.key == arguments.scenario), None)
+    if selected is None:
+        raise SystemExit(f"Unknown scenario: {arguments.scenario}")
+    spotify_client_id = _required_environment("SPOTIFY_CLIENT_ID")
+    spotify_refresh_token = _required_environment("SPOTIFY_SEED_REFRESH_TOKEN")
+    anthropic_api_key = _required_environment("ANTHROPIC_API_KEY")
+
+    spotify_transport = Httpx2RecordingTransport(httpx2.AsyncHTTPTransport())
+    anthropic_transport = HttpxRecordingTransport(httpx.AsyncHTTPTransport())
+    anthropic_http_client = httpx.AsyncClient(transport=anthropic_transport)
+    app = _create_recording_app(
+        spotify_client_id,
+        spotify_refresh_token,
+        anthropic_api_key,
+        spotify_transport,
+        anthropic_http_client,
+    )
+    try:
+        with TestClient(app, base_url=arguments.base_url) as client:
+            _record_selected(client, scenarios, selected, spotify_transport)
+    finally:
+        raw_root, redacted_root = write_cassettes(
+            arguments.fixture_dir,
+            selected.key,
+            {
+                "spotify": spotify_transport.interactions,
+                "anthropic": anthropic_transport.interactions,
+            },
+        )
+    print(f"Raw cassettes: {raw_root}")
+    print(f"Redacted cassettes: {redacted_root}")
+    return 0
+
+
+def _record_selected(
+    client: TestClient,
+    scenarios: tuple[Scenario, ...],
+    selected: Scenario,
+    spotify_transport: Httpx2RecordingTransport,
+) -> None:
+    if selected.after is not None:
+        parent_scenario = next(item for item in scenarios if item.key == selected.after)
+        parent = _send_turn(client, parent_scenario, None)
+        search_count = _search_count(spotify_transport)
+        _send_turn(client, selected, parent)
+        if _search_count(spotify_transport) != search_count:
+            raise RuntimeError("Refinement issued new Spotify searches")
+    else:
+        _send_turn(client, selected, None)
+
+
+def _send_turn(
+    client: TestClient,
+    scenario: Scenario,
+    parent: ScenarioResult | None,
+) -> ScenarioResult:
+    response = client.post("/api/recommendations", json=build_request(scenario, parent))
+    response.raise_for_status()
+    result = ScenarioResult(scenario)
+    for line in response.text.splitlines():
+        event = validate_event(line)
+        result.events.append(cast(dict[str, object], event.model_dump(mode="json")))
+    if not result.events or result.events[-1].get("type") != "done":
+        raise RuntimeError(f"Scenario {scenario.key} did not complete")
+    return result
+
+
+def _create_recording_app(
+    spotify_client_id: str,
+    spotify_refresh_token: str,
+    anthropic_api_key: str,
+    spotify_transport: Httpx2RecordingTransport,
+    anthropic_http_client: httpx.AsyncClient,
+) -> FastAPI:
+    backend_path = str(BACKEND_ROOT)
+    if backend_path not in sys.path:
+        sys.path.insert(0, backend_path)
+    config_module = import_module("app.config")
+    main_module = import_module("app.main")
+    application_settings = config_module.Settings(
+        app_mode=config_module.AppMode.LIVE,
+        spotify_client_id=spotify_client_id,
+        spotify_seed_refresh_token=spotify_refresh_token,
+        anthropic_api_key=anthropic_api_key,
+    )
+    return cast(
+        FastAPI,
+        main_module.create_app(
+            application_settings=application_settings,
+            http_transport=spotify_transport,
+            anthropic_http_client=anthropic_http_client,
+        ),
+    )
+
+
+def _search_count(transport: Httpx2RecordingTransport) -> int:
+    return sum("/search" in interaction.url for interaction in transport.interactions)
+
+
+def _required_environment(name: str) -> str:
+    value = os.environ.get(name)
+    if not value:
+        raise SystemExit(f"{name} is required for recording")
+    return value
+
+
+def _parse_arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--scenario", required=True)
+    parser.add_argument("--base-url", required=True)
+    parser.add_argument("--scenarios", type=Path, default=SCENARIO_PATH)
+    parser.add_argument("--fixture-dir", type=Path, default=EVAL_ROOT / "fixtures")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

eval/recording/__init__.py

@@ -0,0 +1,14 @@
+"""Record and redact external HTTP interactions for fixture replay."""
+
+from recording.model import RecordedInteraction
+from recording.redaction import redact_cassette
+from recording.storage import write_cassettes
+from recording.transport import Httpx2RecordingTransport, HttpxRecordingTransport
+
+__all__ = [
+    "Httpx2RecordingTransport",
+    "HttpxRecordingTransport",
+    "RecordedInteraction",
+    "redact_cassette",
+    "write_cassettes",
+]

eval/recording/model.py

@@ -0,0 +1,30 @@
+"""Transport-neutral cassette interaction model."""
+
+import base64
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class RecordedInteraction:
+    """One completed HTTP exchange with exact response chunks."""
+
+    method: str
+    url: str
+    status: int
+    request_body: bytes
+    response_chunks: tuple[bytes, ...]
+
+    def as_dict(self) -> dict[str, object]:
+        """Encode byte fields losslessly for JSON storage."""
+        return {
+            "method": self.method,
+            "url": self.url,
+            "status": self.status,
+            "request_body_base64": _encode(self.request_body),
+            "response_body_base64": _encode(b"".join(self.response_chunks)),
+            "response_chunks_base64": [_encode(chunk) for chunk in self.response_chunks],
+        }
+
+
+def _encode(value: bytes) -> str:
+    return base64.b64encode(value).decode("ascii")

eval/recording/redaction.py

@@ -0,0 +1,219 @@
+"""Remove credentials and listener identity from persisted cassettes."""
+
+import base64
+import copy
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast
+from urllib.parse import urlparse
+
+SYNTHETIC_ACCOUNT_ID = "synthetic-account"
+SYNTHETIC_DISPLAY_NAME = "Synthetic Listener"
+SYNTHETIC_TASTE_PATH = Path(__file__).with_name("synthetic_taste.json")
+BEARER_PATTERN = re.compile(r"Bearer\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE)
+SENSITIVE_PATH_PARTS = {"authorize", "oauth", "token"}
+
+
+@dataclass(frozen=True)
+class _ListenerIdentities:
+    account_ids: frozenset[str]
+    display_names: frozenset[str]
+
+
+def is_sensitive_url(url: str) -> bool:
+    """Return whether an authentication exchange must never be recorded."""
+    parsed = urlparse(url)
+    path_parts = {part.casefold() for part in parsed.path.split("/") if part}
+    return parsed.hostname == "accounts.spotify.com" or bool(path_parts & SENSITIVE_PATH_PARTS)
+
+
+def redact_cassette(interactions: list[dict[str, object]]) -> list[dict[str, object]]:
+    """Return safe interactions without mutating or persisting the source."""
+    safe = [
+        _drop_headers(copy.deepcopy(interaction))
+        for interaction in interactions
+        if not is_sensitive_url(str(interaction.get("url", "")))
+    ]
+    identities = _collect_identities(safe)
+    synthetic_taste = _load_synthetic_taste()
+    return [_redact_interaction(interaction, identities, synthetic_taste) for interaction in safe]
+
+
+def _drop_headers(value: object) -> object:
+    if isinstance(value, dict):
+        return {
+            key: _drop_headers(item)
+            for key, item in value.items()
+            if not str(key).casefold().endswith("headers")
+        }
+    if isinstance(value, list):
+        return [_drop_headers(item) for item in value]
+    return value
+
+
+def _collect_identities(interactions: list[object]) -> _ListenerIdentities:
+    account_ids: set[str] = set()
+    display_names: set[str] = set()
+    for interaction in interactions:
+        if not isinstance(interaction, dict) or not _is_current_user_url(
+            str(interaction.get("url", ""))
+        ):
+            continue
+        payload = _decode_json(str(interaction.get("response_body_base64", "")))
+        if not isinstance(payload, dict):
+            continue
+        for key in ("id", "account_id"):
+            value = payload.get(key)
+            if isinstance(value, str) and value:
+                account_ids.add(value)
+        display_name = payload.get("display_name")
+        if isinstance(display_name, str) and display_name:
+            display_names.add(display_name)
+    return _ListenerIdentities(frozenset(account_ids), frozenset(display_names))
+
+
+def _redact_interaction(
+    interaction: object,
+    identities: _ListenerIdentities,
+    synthetic_taste: dict[str, object],
+) -> dict[str, object]:
+    if not isinstance(interaction, dict):
+        raise ValueError("Cassette interaction must be a mapping")
+    redacted = {str(key): value for key, value in interaction.items()}
+    url = _replace_text(str(redacted.get("url", "")), identities)
+    redacted["url"] = url
+    redacted["request_body_base64"] = _redact_body(
+        str(redacted.get("request_body_base64", "")), identities, None, synthetic_taste
+    )
+    replacement = _taste_replacement(url, synthetic_taste)
+    original_body = str(redacted.get("response_body_base64", ""))
+    response_body = _redact_body(original_body, identities, replacement, synthetic_taste)
+    redacted["response_body_base64"] = response_body
+    chunks = redacted.get("response_chunks_base64", [])
+    if response_body != original_body:
+        redacted["response_chunks_base64"] = [response_body] if response_body else []
+    elif isinstance(chunks, list):
+        redacted["response_chunks_base64"] = [
+            _redact_body(str(chunk), identities, None, synthetic_taste) for chunk in chunks
+        ]
+    return cast(dict[str, object], _redact_value(redacted, identities, synthetic_taste))
+
+
+def _redact_body(
+    encoded: str,
+    identities: _ListenerIdentities,
+    replacement: object | None,
+    synthetic_taste: dict[str, object],
+) -> str:
+    try:
+        raw = base64.b64decode(encoded, validate=True)
+    except ValueError:
+        raw = b""
+    if replacement is not None:
+        value = replacement
+    else:
+        try:
+            value = json.loads(raw)
+        except (json.JSONDecodeError, UnicodeDecodeError):
+            text = raw.decode("utf-8", errors="replace")
+            return _encode(_replace_text(text, identities).encode())
+    redacted = _redact_value(value, identities, synthetic_taste)
+    return _encode(json.dumps(redacted, ensure_ascii=True, separators=(",", ":")).encode("ascii"))
+
+
+def _redact_value(
+    value: object,
+    identities: _ListenerIdentities,
+    synthetic_taste: dict[str, object],
+) -> object:
+    if isinstance(value, dict):
+        return {
+            str(key): _taste_summary(synthetic_taste)
+            if str(key) == "taste_profile"
+            else _synthetic_identity(item, identities, synthetic_taste)
+            for key, item in value.items()
+            if not str(key).casefold().endswith("headers")
+        }
+    if isinstance(value, list):
+        return [_redact_value(item, identities, synthetic_taste) for item in value]
+    if isinstance(value, str):
+        text = _replace_text(value, identities)
+        try:
+            nested = json.loads(text)
+        except json.JSONDecodeError:
+            return text
+        if isinstance(nested, (dict, list)):
+            return json.dumps(
+                _redact_value(nested, identities, synthetic_taste),
+                ensure_ascii=True,
+                separators=(",", ":"),
+            )
+        return text
+    return value
+
+
+def _synthetic_identity(
+    value: object,
+    identities: _ListenerIdentities,
+    synthetic_taste: dict[str, object],
+) -> object:
+    if isinstance(value, str) and value in identities.display_names:
+        return SYNTHETIC_DISPLAY_NAME
+    if isinstance(value, str) and value in identities.account_ids:
+        return SYNTHETIC_ACCOUNT_ID
+    return _redact_value(value, identities, synthetic_taste)
+
+
+def _replace_text(value: str, identities: _ListenerIdentities) -> str:
+    redacted = BEARER_PATTERN.sub("[REDACTED]", value)
+    replacements = {
+        **{identity: SYNTHETIC_ACCOUNT_ID for identity in identities.account_ids},
+        **{identity: SYNTHETIC_DISPLAY_NAME for identity in identities.display_names},
+    }
+    for identity, replacement in sorted(
+        replacements.items(), key=lambda item: len(item[0]), reverse=True
+    ):
+        redacted = redacted.replace(identity, replacement)
+    return redacted
+
+
+def _taste_replacement(url: str, synthetic_taste: dict[str, object]) -> object | None:
+    path = urlparse(url).path.rstrip("/")
+    if path.endswith("/me/top/artists"):
+        return synthetic_taste.get("top_artists")
+    if path.endswith("/me/top/tracks"):
+        return synthetic_taste.get("top_tracks")
+    if path.endswith("/me/tracks"):
+        return synthetic_taste.get("saved_tracks")
+    return None
+
+
+def _is_current_user_url(url: str) -> bool:
+    return urlparse(url).path.rstrip("/").endswith("/v1/me")
+
+
+def _taste_summary(synthetic_taste: dict[str, object]) -> str:
+    summary = synthetic_taste.get("summary")
+    if not isinstance(summary, str):
+        raise ValueError("Synthetic taste fixture must contain a summary")
+    return summary
+
+
+def _load_synthetic_taste() -> dict[str, object]:
+    payload: object = json.loads(SYNTHETIC_TASTE_PATH.read_text(encoding="ascii"))
+    if not isinstance(payload, dict):
+        raise ValueError("Synthetic taste fixture must be a mapping")
+    return cast(dict[str, object], payload)
+
+
+def _decode_json(encoded: str) -> object:
+    try:
+        return json.loads(base64.b64decode(encoded, validate=True))
+    except (ValueError, json.JSONDecodeError, UnicodeDecodeError):
+        return None
+
+
+def _encode(value: bytes) -> str:
+    return base64.b64encode(value).decode("ascii")

eval/recording/storage.py

@@ -0,0 +1,32 @@
+"""Persist raw and redacted cassette stores."""
+
+import json
+from pathlib import Path
+
+from recording.model import RecordedInteraction
+from recording.redaction import redact_cassette
+
+
+def write_cassettes(
+    fixture_root: Path,
+    scenario_key: str,
+    recordings: dict[str, list[RecordedInteraction]],
+) -> tuple[Path, Path]:
+    """Write ignored raw data first, then separately mapped safe cassettes."""
+    raw_root = fixture_root / "raw" / scenario_key
+    redacted_root = fixture_root / scenario_key
+    raw_root.mkdir(parents=True, exist_ok=True)
+    redacted_root.mkdir(parents=True, exist_ok=True)
+    for service, interactions in recordings.items():
+        raw_payload = [interaction.as_dict() for interaction in interactions]
+        _write_json(raw_root / f"{service}.json", raw_payload)
+        redacted_payload = redact_cassette(raw_payload)
+        _write_json(redacted_root / f"{service}.json", redacted_payload)
+    return raw_root, redacted_root
+
+
+def _write_json(path: Path, payload: object) -> None:
+    path.write_text(
+        json.dumps(payload, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
+        encoding="ascii",
+    )

eval/recording/synthetic_taste.json

@@ -0,0 +1,95 @@
+{
+  "saved_tracks": {
+    "href": "https://api.spotify.com/v1/me/tracks",
+    "items": [
+      {
+        "added_at": "2026-01-01T00:00:00Z",
+        "track": {
+          "album": {
+            "images": [],
+            "name": "Synthetic Saved Album"
+          },
+          "artists": [
+            {
+              "name": "Synthetic Saved Artist"
+            }
+          ],
+          "external_urls": {
+            "spotify": "https://open.spotify.com/track/synthetic-saved-track"
+          },
+          "id": "synthetic-saved-track",
+          "name": "Synthetic Saved Track",
+          "uri": "spotify:track:synthetic-saved-track"
+        }
+      }
+    ],
+    "limit": 50,
+    "next": null,
+    "offset": 0,
+    "previous": null,
+    "total": 1
+  },
+  "summary": "Short-term top artists: Synthetic Focus Artist; Long-term top artists: Synthetic Jazz Artist; Short-term top tracks: Synthetic Focus Track by Synthetic Focus Artist; Long-term top tracks: Synthetic Jazz Track by Synthetic Jazz Artist; Saved-track sample: Synthetic Saved Track by Synthetic Saved Artist",
+  "top_artists": {
+    "href": "https://api.spotify.com/v1/me/top/artists",
+    "items": [
+      {
+        "id": "synthetic-focus-artist",
+        "name": "Synthetic Focus Artist"
+      },
+      {
+        "id": "synthetic-jazz-artist",
+        "name": "Synthetic Jazz Artist"
+      }
+    ],
+    "limit": 50,
+    "next": null,
+    "offset": 0,
+    "previous": null,
+    "total": 2
+  },
+  "top_tracks": {
+    "href": "https://api.spotify.com/v1/me/top/tracks",
+    "items": [
+      {
+        "album": {
+          "images": [],
+          "name": "Synthetic Focus Album"
+        },
+        "artists": [
+          {
+            "name": "Synthetic Focus Artist"
+          }
+        ],
+        "external_urls": {
+          "spotify": "https://open.spotify.com/track/synthetic-focus-track"
+        },
+        "id": "synthetic-focus-track",
+        "name": "Synthetic Focus Track",
+        "uri": "spotify:track:synthetic-focus-track"
+      },
+      {
+        "album": {
+          "images": [],
+          "name": "Synthetic Jazz Album"
+        },
+        "artists": [
+          {
+            "name": "Synthetic Jazz Artist"
+          }
+        ],
+        "external_urls": {
+          "spotify": "https://open.spotify.com/track/synthetic-jazz-track"
+        },
+        "id": "synthetic-jazz-track",
+        "name": "Synthetic Jazz Track",
+        "uri": "spotify:track:synthetic-jazz-track"
+      }
+    ],
+    "limit": 50,
+    "next": null,
+    "offset": 0,
+    "previous": null,
+    "total": 2
+  }
+}

eval/recording/transport.py

@@ -0,0 +1,154 @@
+"""Recording transports for Spotify's httpx2 and Anthropic's httpx."""
+
+from collections.abc import AsyncIterator, Callable
+from typing import cast
+
+import httpx
+import httpx2
+
+from recording.model import RecordedInteraction
+from recording.redaction import is_sensitive_url
+
+
+class HttpxRecordingTransport(httpx.AsyncBaseTransport):
+    """Wrap an httpx transport and retain completed response chunk sequences."""
+
+    def __init__(self, transport: httpx.AsyncBaseTransport) -> None:
+        """Bind the real transport and start an empty in-memory cassette."""
+        self.transport = transport
+        self.interactions: list[RecordedInteraction] = []
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        """Forward one request and wrap its response stream for capture."""
+        request.headers["Accept-Encoding"] = "identity"
+        request_body = await request.aread()
+        response = await self.transport.handle_async_request(request)
+        if is_sensitive_url(str(request.url)):
+            return response
+        stream = _HttpxRecordingStream(
+            cast(httpx.AsyncByteStream, response.stream),
+            lambda chunks: self._finish(request, request_body, response.status_code, chunks),
+        )
+        return httpx.Response(
+            response.status_code,
+            headers=response.headers,
+            stream=stream,
+            extensions=response.extensions,
+            request=request,
+        )
+
+    async def aclose(self) -> None:
+        """Close the wrapped transport."""
+        await self.transport.aclose()
+
+    def _finish(
+        self,
+        request: httpx.Request,
+        request_body: bytes,
+        status: int,
+        chunks: tuple[bytes, ...],
+    ) -> None:
+        self.interactions.append(
+            RecordedInteraction(request.method, str(request.url), status, request_body, chunks)
+        )
+
+
+class Httpx2RecordingTransport(httpx2.AsyncBaseTransport):
+    """Wrap an httpx2 transport and retain completed response chunk sequences."""
+
+    def __init__(self, transport: httpx2.AsyncBaseTransport) -> None:
+        """Bind the real transport and start an empty in-memory cassette."""
+        self.transport = transport
+        self.interactions: list[RecordedInteraction] = []
+
+    async def handle_async_request(self, request: httpx2.Request) -> httpx2.Response:
+        """Forward one request and wrap its response stream for capture."""
+        request.headers["Accept-Encoding"] = "identity"
+        request_body = await request.aread()
+        response = await self.transport.handle_async_request(request)
+        if is_sensitive_url(str(request.url)):
+            return response
+        stream = _Httpx2RecordingStream(
+            cast(httpx2.AsyncByteStream, response.stream),
+            lambda chunks: self._finish(request, request_body, response.status_code, chunks),
+        )
+        return httpx2.Response(
+            response.status_code,
+            headers=response.headers,
+            stream=stream,
+            extensions=response.extensions,
+            request=request,
+        )
+
+    async def aclose(self) -> None:
+        """Close the wrapped transport."""
+        await self.transport.aclose()
+
+    def _finish(
+        self,
+        request: httpx2.Request,
+        request_body: bytes,
+        status: int,
+        chunks: tuple[bytes, ...],
+    ) -> None:
+        self.interactions.append(
+            RecordedInteraction(request.method, str(request.url), status, request_body, chunks)
+        )
+
+
+class _HttpxRecordingStream(httpx.AsyncByteStream):
+    def __init__(
+        self,
+        stream: httpx.AsyncByteStream,
+        finish: Callable[[tuple[bytes, ...]], None],
+    ) -> None:
+        self.stream = stream
+        self.finish = finish
+        self.chunks: list[bytes] = []
+        self.is_finished = False
+
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        try:
+            async for chunk in self.stream:
+                self.chunks.append(chunk)
+                yield chunk
+        finally:
+            self._finish()
+
+    async def aclose(self) -> None:
+        await self.stream.aclose()
+        self._finish()
+
+    def _finish(self) -> None:
+        if not self.is_finished:
+            self.is_finished = True
+            self.finish(tuple(self.chunks))
+
+
+class _Httpx2RecordingStream(httpx2.AsyncByteStream):
+    def __init__(
+        self,
+        stream: httpx2.AsyncByteStream,
+        finish: Callable[[tuple[bytes, ...]], None],
+    ) -> None:
+        self.stream = stream
+        self.finish = finish
+        self.chunks: list[bytes] = []
+        self.is_finished = False
+
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        try:
+            async for chunk in self.stream:
+                self.chunks.append(chunk)
+                yield chunk
+        finally:
+            self._finish()
+
+    async def aclose(self) -> None:
+        await self.stream.aclose()
+        self._finish()
+
+    def _finish(self) -> None:
+        if not self.is_finished:
+            self.is_finished = True
+            self.finish(tuple(self.chunks))

eval/run_eval.py

@@ -0,0 +1,98 @@
+"""Command-line entry point for live and baseline evaluation."""
+
+import argparse
+import asyncio
+import os
+from pathlib import Path
+
+from baseline import record_baselines, write_comparison
+from live import LiveLimits, ScenarioResult, run_live_scenarios
+from scenario import SCENARIO_PATH, load_scenarios
+
+EVAL_ROOT = Path(__file__).parent
+
+
+def main() -> int:
+    """Run the selected evaluation arms and return a process status."""
+    arguments = _parse_arguments()
+    scenarios = load_scenarios(arguments.scenarios)
+    exit_code = 0
+    if arguments.base_url is None:
+        raise SystemExit("--base-url is required")
+    limits = LiveLimits(
+        minimum_tracks=arguments.min_tracks,
+        maximum_tracks=arguments.max_tracks,
+        minimum_artists=arguments.min_artists,
+        first_track_budget_ms=arguments.first_track_budget_ms,
+        total_budget_ms=arguments.total_budget_ms,
+    )
+    results = asyncio.run(
+        run_live_scenarios(scenarios, arguments.base_url, arguments.report_dir, limits)
+    )
+    _print_summary(results)
+    if not all(result.passed for result in results):
+        exit_code = 1
+
+    if arguments.baseline:
+        client_id = os.environ.get("SPOTIFY_CLIENT_ID")
+        client_secret = os.environ.get("SPOTIFY_CLIENT_SECRET")
+        if not client_id or not client_secret:
+            print("Spotify client credentials are absent; skipping baseline.")
+        else:
+            baseline_root = arguments.snapshot_dir / "baseline"
+            asyncio.run(record_baselines(scenarios, client_id, client_secret, baseline_root))
+        comparison_path = write_comparison(
+            scenarios,
+            arguments.snapshot_dir / "baseline",
+            arguments.report_dir,
+            arguments.comparison_top_n,
+        )
+        print(f"Comparison: {comparison_path}")
+    return exit_code
+
+
+def _parse_arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--base-url")
+    parser.add_argument("--baseline", action="store_true")
+    parser.add_argument("--scenarios", type=Path, default=SCENARIO_PATH)
+    parser.add_argument("--snapshot-dir", type=Path, default=EVAL_ROOT / "snapshots")
+    parser.add_argument("--report-dir", type=Path, default=EVAL_ROOT / "reports")
+    parser.add_argument("--min-tracks", type=int, default=8)
+    parser.add_argument("--max-tracks", type=int, default=15)
+    parser.add_argument("--min-artists", type=int, default=5)
+    parser.add_argument("--first-track-budget-ms", type=int, default=20_000)
+    parser.add_argument("--total-budget-ms", type=int, default=30_000)
+    parser.add_argument("--comparison-top-n", type=int, default=10)
+    arguments = parser.parse_args()
+    if arguments.min_tracks < 0 or arguments.max_tracks < arguments.min_tracks:
+        parser.error("track bounds are invalid")
+    if min(arguments.min_artists, arguments.first_track_budget_ms, arguments.total_budget_ms) < 0:
+        parser.error("artist and latency limits must be non-negative")
+    if arguments.comparison_top_n < 1:
+        parser.error("comparison top N must be positive")
+    return arguments
+
+
+def _print_summary(results: tuple[ScenarioResult, ...]) -> None:
+    print("| Scenario | Status | Tracks | Artists | First track | Total |")
+    print("| --- | --- | ---: | ---: | ---: | ---: |")
+    for result in results:
+        artists = {str(artist).casefold() for event in result.tracks for artist in _artists(event)}
+        first = f"{result.first_track_ms} ms" if result.first_track_ms is not None else "n/a"
+        total = f"{result.total_ms} ms" if result.total_ms is not None else "n/a"
+        status = "PASS" if result.passed else "FAIL"
+        print(
+            f"| {result.scenario.key} | {status} | {len(result.tracks)} | "
+            f"{len(artists)} | {first} | {total} |"
+        )
+
+
+def _artists(event: dict[str, object]) -> list[object]:
+    track = event.get("track")
+    artists = track.get("artists") if isinstance(track, dict) else None
+    return artists if isinstance(artists, list) else []
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

eval/scenario.py

@@ -0,0 +1,79 @@
+"""Load the shared evaluation scenario catalog."""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast
+
+import yaml
+
+SCENARIO_PATH = Path(__file__).with_name("scenarios.yaml")
+FacetValue = str | bool
+
+
+@dataclass(frozen=True)
+class Scenario:
+    """One golden query and its expected intent facets."""
+
+    key: str
+    chip: str
+    query: str
+    facets: dict[str, FacetValue]
+    after: str | None = None
+
+    @property
+    def is_refinement(self) -> bool:
+        """Return whether this query follows another scenario."""
+        return self.after is not None
+
+
+def load_scenarios(path: Path = SCENARIO_PATH) -> tuple[Scenario, ...]:
+    """Read and validate the shared YAML scenario catalog."""
+    payload = yaml.safe_load(path.read_text(encoding="ascii"))
+    if not isinstance(payload, dict) or payload.get("version") != 1:
+        raise ValueError("Scenario catalog must have version 1")
+    entries = payload.get("scenarios")
+    if not isinstance(entries, list) or not entries:
+        raise ValueError("Scenario catalog must contain scenarios")
+
+    scenarios = tuple(_parse_scenario(entry) for entry in entries)
+    keys = [scenario.key for scenario in scenarios]
+    if len(keys) != len(set(keys)):
+        raise ValueError("Scenario keys must be unique")
+    known_keys = set(keys)
+    for scenario in scenarios:
+        if scenario.after is not None and scenario.after not in known_keys:
+            raise ValueError(f"Unknown parent scenario: {scenario.after}")
+    return scenarios
+
+
+def _parse_scenario(payload: object) -> Scenario:
+    if not isinstance(payload, dict):
+        raise ValueError("Each scenario must be a mapping")
+    required = {"key", "chip", "query", "facets"}
+    allowed = required | {"after"}
+    if set(payload) - allowed or not required <= set(payload):
+        raise ValueError("Scenario fields must match the catalog schema")
+    facets = payload["facets"]
+    if not isinstance(facets, dict) or not facets:
+        raise ValueError("Scenario facets must be a non-empty mapping")
+    if not all(
+        isinstance(key, str) and isinstance(value, (str, bool)) for key, value in facets.items()
+    ):
+        raise ValueError("Scenario facets must contain string or boolean values")
+    after = payload.get("after")
+    if after is not None and not isinstance(after, str):
+        raise ValueError("Scenario after must be a key")
+    return Scenario(
+        key=_required_string(payload, "key"),
+        chip=_required_string(payload, "chip"),
+        query=_required_string(payload, "query"),
+        facets=cast(dict[str, FacetValue], facets),
+        after=after,
+    )
+
+
+def _required_string(payload: dict[object, object], key: str) -> str:
+    value = payload.get(key)
+    if not isinstance(value, str) or not value:
+        raise ValueError(f"Scenario {key} must be a non-empty string")
+    return value

eval/scenarios.yaml

@@ -0,0 +1,49 @@
+version: 1
+scenarios:
+  - key: focus-coding
+    chip: Focus while coding
+    query: something calm for while I am programming, but not boring
+    facets:
+      activity: coding
+      familiarity: mix
+  - key: workout-energy
+    chip: Energy for the gym
+    query: high-energy music for a heavy workout
+    facets:
+      activity: workout
+      mood: energetic
+  - key: dutch-chill
+    chip: Dutch and relaxed
+    query: relaxed Dutch-language music for the couch
+    facets:
+      language: nl
+      mood: relaxed
+  - key: nineties-nostalgia
+    chip: 90s nostalgia
+    query: take me back to the nineties
+    facets:
+      era: 1990s
+  - key: discover-new
+    chip: Surprise me with something new
+    query: something I do not know yet but will probably like
+    facets:
+      familiarity: new
+  - key: rainy-sunday
+    chip: Rainy Sunday
+    query: melancholic but warm, for a rainy Sunday morning
+    facets:
+      mood: melancholic
+  - key: dinner-background
+    chip: Background for dinner
+    query: something jazzy for during dinner, not too present
+    facets:
+      genre: jazz
+      mood: subdued
+  - key: focus-coding-refine
+    chip: More electronic
+    query: a bit more electronic, and drop number 3
+    after: focus-coding
+    facets:
+      refinement: true
+      pool_reuse: true
+      re_grounding: false

eval/tests/test_live.py

@@ -0,0 +1,78 @@
+"""Tests for strict live stream property evaluation."""
+
+import asyncio
+import json
+
+import httpx
+import pytest
+from fastapi import FastAPI
+from fastapi.responses import StreamingResponse
+from live import LiveLimits, run_live_scenario
+from scenario import Scenario
+from wire import validate_event
+
+
+def test_live_scenario_accepts_a_valid_property_stream() -> None:
+    async def run() -> None:
+        app = FastAPI()
+
+        @app.post("/api/recommendations")
+        async def recommendations() -> StreamingResponse:
+            events = [
+                {
+                    "type": "metadata",
+                    "request_id": "request",
+                    "intent_summary": "Calm music for coding with a familiar discovery mix.",
+                    "candidate_count": 10,
+                },
+                {
+                    "type": "track",
+                    "rank": 1,
+                    "track": {
+                        "id": "track",
+                        "uri": "spotify:track:track",
+                        "title": "Track",
+                        "artists": ["Artist"],
+                        "album_name": "Album",
+                        "album_art_url": None,
+                        "external_url": None,
+                    },
+                    "justification": "A focused fit.",
+                },
+                {"type": "done", "track_count": 1, "total_ms": 1},
+            ]
+            body = "".join(json.dumps(event) + "\n" for event in events)
+            return StreamingResponse(iter((body,)), media_type="application/x-ndjson")
+
+        transport = httpx.ASGITransport(app=app)
+        async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+            result = await run_live_scenario(
+                client,
+                Scenario(
+                    "focus-coding",
+                    "Focus while coding",
+                    "calm coding music",
+                    {"activity": "coding", "familiarity": "mix"},
+                ),
+                LiveLimits(1, 1, 1, 1_000, 1_000),
+            )
+
+        assert result.passed
+        assert result.checks["event_order"]
+        assert result.checks["done_track_count"]
+
+    asyncio.run(run())
+
+
+def test_wire_validation_rejects_extra_fields() -> None:
+    line = json.dumps(
+        {
+            "type": "done",
+            "track_count": 1,
+            "total_ms": 1,
+            "unexpected": True,
+        }
+    )
+
+    with pytest.raises(ValueError, match="fields do not match"):
+        validate_event(line)

eval/tests/test_recording.py

@@ -0,0 +1,134 @@
+"""Tests for cassette transport fidelity and redaction."""
+
+import asyncio
+import base64
+import json
+from collections.abc import AsyncIterator
+
+import httpx
+import httpx2
+from recording import Httpx2RecordingTransport, HttpxRecordingTransport, redact_cassette
+
+
+class HttpxChunks(httpx.AsyncByteStream):
+    """Yield fixed httpx byte chunks."""
+
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        yield b"first-"
+        yield b"second"
+
+
+class Httpx2Chunks(httpx2.AsyncByteStream):
+    """Yield fixed httpx2 byte chunks."""
+
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        yield b"first-"
+        yield b"second"
+
+
+def test_httpx_recording_transport_preserves_chunk_boundaries() -> None:
+    async def run() -> None:
+        async def handler(request: httpx.Request) -> httpx.Response:
+            return httpx.Response(200, stream=HttpxChunks())
+
+        transport = HttpxRecordingTransport(httpx.MockTransport(handler))
+        async with httpx.AsyncClient(transport=transport) as client:
+            response = await client.get("https://api.anthropic.com/v1/messages")
+
+        assert response.content == b"first-second"
+        assert transport.interactions[0].response_chunks == (b"first-", b"second")
+
+    asyncio.run(run())
+
+
+def test_httpx2_recording_transport_preserves_chunk_boundaries() -> None:
+    async def run() -> None:
+        async def handler(request: httpx2.Request) -> httpx2.Response:
+            return httpx2.Response(200, stream=Httpx2Chunks())
+
+        transport = Httpx2RecordingTransport(httpx2.MockTransport(handler))
+        async with httpx2.AsyncClient(transport=transport) as client:
+            response = await client.get("https://api.spotify.com/v1/search?q=test")
+
+        assert response.content == b"first-second"
+        assert transport.interactions[0].response_chunks == (b"first-", b"second")
+
+    asyncio.run(run())
+
+
+def test_redaction_removes_headers_tokens_identity_and_taste() -> None:
+    interactions = [
+        {
+            "method": "POST",
+            "url": "https://accounts.spotify.com/api/token",
+            "request_headers": {"Authorization": "Bearer raw-token"},
+            "request_body_base64": _encode(b"client_secret=secret"),
+            "response_body_base64": _encode(b'{"access_token":"raw-token"}'),
+            "response_chunks_base64": [],
+            "status": 200,
+        },
+        {
+            "method": "GET",
+            "url": "https://api.spotify.com/v1/me",
+            "response_headers": {"Set-Cookie": "secret"},
+            "request_body_base64": "",
+            "response_body_base64": _encode(
+                b'{"id":"real-account-id","display_name":"Real Listener"}'
+            ),
+            "response_chunks_base64": [],
+            "status": 200,
+        },
+        {
+            "method": "GET",
+            "url": "https://api.spotify.com/v1/me/top/artists?limit=50",
+            "headers": {"Authorization": "Bearer raw-token"},
+            "request_body_base64": "",
+            "response_body_base64": _encode(b'{"items":[{"name":"Private Artist"}]}'),
+            "response_chunks_base64": [],
+            "status": 200,
+        },
+        {
+            "method": "POST",
+            "url": "https://api.anthropic.com/v1/messages",
+            "request_body_base64": _encode(
+                json.dumps(
+                    {
+                        "messages": [
+                            {
+                                "content": json.dumps(
+                                    {
+                                        "taste_profile": "Private Artist",
+                                        "account": "real-account-id",
+                                    }
+                                )
+                            }
+                        ]
+                    }
+                ).encode()
+            ),
+            "response_body_base64": _encode(b"Bearer raw-token"),
+            "response_chunks_base64": [_encode(b"Bearer raw-token")],
+            "status": 200,
+        },
+    ]
+
+    redacted = redact_cassette(interactions)
+    rendered = json.dumps(redacted)
+    decoded_bodies = " ".join(
+        base64.b64decode(str(item.get(field, ""))).decode(errors="replace")
+        for item in redacted
+        for field in ("request_body_base64", "response_body_base64")
+    )
+
+    assert len(redacted) == 3
+    assert "headers" not in rendered.casefold()
+    assert "raw-token" not in rendered + decoded_bodies
+    assert "real-account-id" not in rendered + decoded_bodies
+    assert "Real Listener" not in rendered + decoded_bodies
+    assert "Private Artist" not in rendered + decoded_bodies
+    assert "Synthetic Focus Artist" in decoded_bodies
+    assert "accounts.spotify.com" not in rendered
+
+
+def _encode(value: bytes) -> str:
+    return base64.b64encode(value).decode("ascii")

eval/wire.py

@@ -0,0 +1,46 @@
+"""Validate NDJSON events against the application-owned wire schemas."""
+
+import json
+import sys
+from importlib import import_module
+from pathlib import Path
+from types import ModuleType
+
+from pydantic import BaseModel, TypeAdapter
+
+BACKEND_ROOT = Path(__file__).parents[1] / "backend"
+EVENT_CLASS_NAMES = {
+    "metadata": "MetadataEvent",
+    "track": "TrackEvent",
+    "warning": "WarningEvent",
+    "error": "ErrorEvent",
+    "done": "DoneEvent",
+}
+
+
+def validate_event(line: str) -> BaseModel:
+    """Parse one complete line with exact fields and strict value types."""
+    schemas = _load_schemas()
+    payload = json.loads(line)
+    if not isinstance(payload, dict):
+        raise ValueError("NDJSON event must be an object")
+    event_type = payload.get("type")
+    if not isinstance(event_type, str) or event_type not in EVENT_CLASS_NAMES:
+        raise ValueError("NDJSON event has an unknown type")
+    event_class = getattr(schemas, EVENT_CLASS_NAMES[event_type])
+    if set(payload) != set(event_class.model_fields):
+        raise ValueError(f"{event_type} event fields do not match the wire schema")
+    if event_type == "track":
+        track = payload.get("track")
+        track_class = schemas.TrackCard
+        if not isinstance(track, dict) or set(track) != set(track_class.model_fields):
+            raise ValueError("track card fields do not match the wire schema")
+    adapter: TypeAdapter[BaseModel] = TypeAdapter(schemas.StreamEvent)
+    return adapter.validate_json(line, strict=True)
+
+
+def _load_schemas() -> ModuleType:
+    backend_path = str(BACKEND_ROOT)
+    if backend_path not in sys.path:
+        sys.path.insert(0, backend_path)
+    return import_module("app.api.schemas")