test: add scenario eval runner, recorded fixtures, and baseline comparison

eval/tests/test_live.py

@@ -0,0 +1,78 @@
+"""Tests for strict live stream property evaluation."""
+
+import asyncio
+import json
+
+import httpx
+import pytest
+from fastapi import FastAPI
+from fastapi.responses import StreamingResponse
+from live import LiveLimits, run_live_scenario
+from scenario import Scenario
+from wire import validate_event
+
+
+def test_live_scenario_accepts_a_valid_property_stream() -> None:
+    async def run() -> None:
+        app = FastAPI()
+
+        @app.post("/api/recommendations")
+        async def recommendations() -> StreamingResponse:
+            events = [
+                {
+                    "type": "metadata",
+                    "request_id": "request",
+                    "intent_summary": "Calm music for coding with a familiar discovery mix.",
+                    "candidate_count": 10,
+                },
+                {
+                    "type": "track",
+                    "rank": 1,
+                    "track": {
+                        "id": "track",
+                        "uri": "spotify:track:track",
+                        "title": "Track",
+                        "artists": ["Artist"],
+                        "album_name": "Album",
+                        "album_art_url": None,
+                        "external_url": None,
+                    },
+                    "justification": "A focused fit.",
+                },
+                {"type": "done", "track_count": 1, "total_ms": 1},
+            ]
+            body = "".join(json.dumps(event) + "\n" for event in events)
+            return StreamingResponse(iter((body,)), media_type="application/x-ndjson")
+
+        transport = httpx.ASGITransport(app=app)
+        async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+            result = await run_live_scenario(
+                client,
+                Scenario(
+                    "focus-coding",
+                    "Focus while coding",
+                    "calm coding music",
+                    {"activity": "coding", "familiarity": "mix"},
+                ),
+                LiveLimits(1, 1, 1, 1_000, 1_000),
+            )
+
+        assert result.passed
+        assert result.checks["event_order"]
+        assert result.checks["done_track_count"]
+
+    asyncio.run(run())
+
+
+def test_wire_validation_rejects_extra_fields() -> None:
+    line = json.dumps(
+        {
+            "type": "done",
+            "track_count": 1,
+            "total_ms": 1,
+            "unexpected": True,
+        }
+    )
+
+    with pytest.raises(ValueError, match="fields do not match"):
+        validate_event(line)

eval/tests/test_recording.py

@@ -0,0 +1,134 @@
+"""Tests for cassette transport fidelity and redaction."""
+
+import asyncio
+import base64
+import json
+from collections.abc import AsyncIterator
+
+import httpx
+import httpx2
+from recording import Httpx2RecordingTransport, HttpxRecordingTransport, redact_cassette
+
+
+class HttpxChunks(httpx.AsyncByteStream):
+    """Yield fixed httpx byte chunks."""
+
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        yield b"first-"
+        yield b"second"
+
+
+class Httpx2Chunks(httpx2.AsyncByteStream):
+    """Yield fixed httpx2 byte chunks."""
+
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        yield b"first-"
+        yield b"second"
+
+
+def test_httpx_recording_transport_preserves_chunk_boundaries() -> None:
+    async def run() -> None:
+        async def handler(request: httpx.Request) -> httpx.Response:
+            return httpx.Response(200, stream=HttpxChunks())
+
+        transport = HttpxRecordingTransport(httpx.MockTransport(handler))
+        async with httpx.AsyncClient(transport=transport) as client:
+            response = await client.get("https://api.anthropic.com/v1/messages")
+
+        assert response.content == b"first-second"
+        assert transport.interactions[0].response_chunks == (b"first-", b"second")
+
+    asyncio.run(run())
+
+
+def test_httpx2_recording_transport_preserves_chunk_boundaries() -> None:
+    async def run() -> None:
+        async def handler(request: httpx2.Request) -> httpx2.Response:
+            return httpx2.Response(200, stream=Httpx2Chunks())
+
+        transport = Httpx2RecordingTransport(httpx2.MockTransport(handler))
+        async with httpx2.AsyncClient(transport=transport) as client:
+            response = await client.get("https://api.spotify.com/v1/search?q=test")
+
+        assert response.content == b"first-second"
+        assert transport.interactions[0].response_chunks == (b"first-", b"second")
+
+    asyncio.run(run())
+
+
+def test_redaction_removes_headers_tokens_identity_and_taste() -> None:
+    interactions = [
+        {
+            "method": "POST",
+            "url": "https://accounts.spotify.com/api/token",
+            "request_headers": {"Authorization": "Bearer raw-token"},
+            "request_body_base64": _encode(b"client_secret=secret"),
+            "response_body_base64": _encode(b'{"access_token":"raw-token"}'),
+            "response_chunks_base64": [],
+            "status": 200,
+        },
+        {
+            "method": "GET",
+            "url": "https://api.spotify.com/v1/me",
+            "response_headers": {"Set-Cookie": "secret"},
+            "request_body_base64": "",
+            "response_body_base64": _encode(
+                b'{"id":"real-account-id","display_name":"Real Listener"}'
+            ),
+            "response_chunks_base64": [],
+            "status": 200,
+        },
+        {
+            "method": "GET",
+            "url": "https://api.spotify.com/v1/me/top/artists?limit=50",
+            "headers": {"Authorization": "Bearer raw-token"},
+            "request_body_base64": "",
+            "response_body_base64": _encode(b'{"items":[{"name":"Private Artist"}]}'),
+            "response_chunks_base64": [],
+            "status": 200,
+        },
+        {
+            "method": "POST",
+            "url": "https://api.anthropic.com/v1/messages",
+            "request_body_base64": _encode(
+                json.dumps(
+                    {
+                        "messages": [
+                            {
+                                "content": json.dumps(
+                                    {
+                                        "taste_profile": "Private Artist",
+                                        "account": "real-account-id",
+                                    }
+                                )
+                            }
+                        ]
+                    }
+                ).encode()
+            ),
+            "response_body_base64": _encode(b"Bearer raw-token"),
+            "response_chunks_base64": [_encode(b"Bearer raw-token")],
+            "status": 200,
+        },
+    ]
+
+    redacted = redact_cassette(interactions)
+    rendered = json.dumps(redacted)
+    decoded_bodies = " ".join(
+        base64.b64decode(str(item.get(field, ""))).decode(errors="replace")
+        for item in redacted
+        for field in ("request_body_base64", "response_body_base64")
+    )
+
+    assert len(redacted) == 3
+    assert "headers" not in rendered.casefold()
+    assert "raw-token" not in rendered + decoded_bodies
+    assert "real-account-id" not in rendered + decoded_bodies
+    assert "Real Listener" not in rendered + decoded_bodies
+    assert "Private Artist" not in rendered + decoded_bodies
+    assert "Synthetic Focus Artist" in decoded_bodies
+    assert "accounts.spotify.com" not in rendered
+
+
+def _encode(value: bytes) -> str:
+    return base64.b64encode(value).decode("ascii")