test: add scenario eval runner, recorded fixtures, and baseline comparison

eval/baseline.py

@@ -0,0 +1,114 @@
+"""Record bare Spotify search results and render side-by-side comparisons."""
+
+import json
+from pathlib import Path
+
+import httpx
+from scenario import Scenario
+
+TOKEN_URL = "https://accounts.spotify.com/api/token"
+SEARCH_URL = "https://api.spotify.com/v1/search"
+
+
+async def record_baselines(
+    scenarios: tuple[Scenario, ...],
+    client_id: str,
+    client_secret: str,
+    snapshot_root: Path,
+) -> None:
+    """Snapshot the top ten results from bare searches for every query."""
+    snapshot_root.mkdir(parents=True, exist_ok=True)
+    async with httpx.AsyncClient(timeout=20.0) as client:
+        token_response = await client.post(
+            TOKEN_URL,
+            data={"grant_type": "client_credentials"},
+            auth=(client_id, client_secret),
+        )
+        token_response.raise_for_status()
+        access_token = token_response.json()["access_token"]
+        for scenario in scenarios:
+            response = await client.get(
+                SEARCH_URL,
+                params={"q": scenario.query, "type": "track", "limit": 10},
+                headers={"Authorization": f"Bearer {access_token}"},
+            )
+            response.raise_for_status()
+            tracks = response.json().get("tracks", {}).get("items", [])[:10]
+            _write_json(
+                snapshot_root / f"{scenario.key}.json",
+                {"key": scenario.key, "query": scenario.query, "tracks": tracks},
+            )
+
+
+def write_comparison(
+    scenarios: tuple[Scenario, ...],
+    snapshot_root: Path,
+    report_root: Path,
+    top_n: int,
+) -> Path:
+    """Write a markdown comparison for scenarios with both result arms."""
+    report_root.mkdir(parents=True, exist_ok=True)
+    lines = ["# Pipeline and bare search comparison", ""]
+    compared = 0
+    for scenario in scenarios:
+        baseline_path = snapshot_root / f"{scenario.key}.json"
+        pipeline_path = report_root / f"{scenario.key}.json"
+        if not baseline_path.exists() or not pipeline_path.exists():
+            continue
+        baseline = json.loads(baseline_path.read_text(encoding="ascii"))
+        pipeline = json.loads(pipeline_path.read_text(encoding="ascii"))
+        lines.extend(_comparison_section(scenario, pipeline, baseline, top_n))
+        compared += 1
+    if compared == 0:
+        lines.extend(["No scenario has results from both arms yet.", ""])
+    output_path = report_root / "baseline-comparison.md"
+    output_path.write_text("\n".join(lines), encoding="utf-8")
+    return output_path
+
+
+def _comparison_section(
+    scenario: Scenario,
+    pipeline: object,
+    baseline: object,
+    top_n: int,
+) -> list[str]:
+    pipeline_tracks = pipeline.get("tracks", []) if isinstance(pipeline, dict) else []
+    baseline_tracks = baseline.get("tracks", []) if isinstance(baseline, dict) else []
+    lines = [f"## {scenario.chip}", "", "| Rank | Pipeline | Bare search |", "| ---: | --- | --- |"]
+    for index in range(top_n):
+        pipeline_label = _pipeline_label(pipeline_tracks, index)
+        baseline_label = _spotify_label(baseline_tracks, index)
+        lines.append(f"| {index + 1} | {pipeline_label} | {baseline_label} |")
+    lines.append("")
+    return lines
+
+
+def _pipeline_label(tracks: object, index: int) -> str:
+    if not isinstance(tracks, list) or index >= len(tracks):
+        return ""
+    event = tracks[index]
+    track = event.get("track") if isinstance(event, dict) else None
+    return _track_label(track, "title")
+
+
+def _spotify_label(tracks: object, index: int) -> str:
+    if not isinstance(tracks, list) or index >= len(tracks):
+        return ""
+    return _track_label(tracks[index], "name")
+
+
+def _track_label(track: object, title_key: str) -> str:
+    if not isinstance(track, dict):
+        return ""
+    artists = track.get("artists", [])
+    names = [
+        artist.get("name", "") if isinstance(artist, dict) else str(artist) for artist in artists
+    ]
+    return f"{track.get(title_key, '')} by {', '.join(names)}"
+
+
+def _write_json(path: Path, payload: object) -> None:
+    path.write_text(
+        json.dumps(payload, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
+        encoding="ascii",
+    )