test: add scenario eval runner, recorded fixtures, and baseline comparison
This commit is contained in:
Justin Visser
parent
3dc1af5f0c
commit
0664cc2d27
38 changed files with 6986 additions and 5 deletions
14
eval/recording/__init__.py
Normal file
14
eval/recording/__init__.py
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
"""Record and redact external HTTP interactions for fixture replay."""
|
||||
|
||||
from recording.model import RecordedInteraction
|
||||
from recording.redaction import redact_cassette
|
||||
from recording.storage import write_cassettes
|
||||
from recording.transport import Httpx2RecordingTransport, HttpxRecordingTransport
|
||||
|
||||
__all__ = [
|
||||
"Httpx2RecordingTransport",
|
||||
"HttpxRecordingTransport",
|
||||
"RecordedInteraction",
|
||||
"redact_cassette",
|
||||
"write_cassettes",
|
||||
]
|
||||
30
eval/recording/model.py
Normal file
30
eval/recording/model.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
"""Transport-neutral cassette interaction model."""
|
||||
|
||||
import base64
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RecordedInteraction:
|
||||
"""One completed HTTP exchange with exact response chunks."""
|
||||
|
||||
method: str
|
||||
url: str
|
||||
status: int
|
||||
request_body: bytes
|
||||
response_chunks: tuple[bytes, ...]
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
"""Encode byte fields losslessly for JSON storage."""
|
||||
return {
|
||||
"method": self.method,
|
||||
"url": self.url,
|
||||
"status": self.status,
|
||||
"request_body_base64": _encode(self.request_body),
|
||||
"response_body_base64": _encode(b"".join(self.response_chunks)),
|
||||
"response_chunks_base64": [_encode(chunk) for chunk in self.response_chunks],
|
||||
}
|
||||
|
||||
|
||||
def _encode(value: bytes) -> str:
|
||||
return base64.b64encode(value).decode("ascii")
|
||||
219
eval/recording/redaction.py
Normal file
219
eval/recording/redaction.py
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
"""Remove credentials and listener identity from persisted cassettes."""
|
||||
|
||||
import base64
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
SYNTHETIC_ACCOUNT_ID = "synthetic-account"
|
||||
SYNTHETIC_DISPLAY_NAME = "Synthetic Listener"
|
||||
SYNTHETIC_TASTE_PATH = Path(__file__).with_name("synthetic_taste.json")
|
||||
BEARER_PATTERN = re.compile(r"Bearer\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE)
|
||||
SENSITIVE_PATH_PARTS = {"authorize", "oauth", "token"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ListenerIdentities:
|
||||
account_ids: frozenset[str]
|
||||
display_names: frozenset[str]
|
||||
|
||||
|
||||
def is_sensitive_url(url: str) -> bool:
|
||||
"""Return whether an authentication exchange must never be recorded."""
|
||||
parsed = urlparse(url)
|
||||
path_parts = {part.casefold() for part in parsed.path.split("/") if part}
|
||||
return parsed.hostname == "accounts.spotify.com" or bool(path_parts & SENSITIVE_PATH_PARTS)
|
||||
|
||||
|
||||
def redact_cassette(interactions: list[dict[str, object]]) -> list[dict[str, object]]:
|
||||
"""Return safe interactions without mutating or persisting the source."""
|
||||
safe = [
|
||||
_drop_headers(copy.deepcopy(interaction))
|
||||
for interaction in interactions
|
||||
if not is_sensitive_url(str(interaction.get("url", "")))
|
||||
]
|
||||
identities = _collect_identities(safe)
|
||||
synthetic_taste = _load_synthetic_taste()
|
||||
return [_redact_interaction(interaction, identities, synthetic_taste) for interaction in safe]
|
||||
|
||||
|
||||
def _drop_headers(value: object) -> object:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: _drop_headers(item)
|
||||
for key, item in value.items()
|
||||
if not str(key).casefold().endswith("headers")
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [_drop_headers(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _collect_identities(interactions: list[object]) -> _ListenerIdentities:
|
||||
account_ids: set[str] = set()
|
||||
display_names: set[str] = set()
|
||||
for interaction in interactions:
|
||||
if not isinstance(interaction, dict) or not _is_current_user_url(
|
||||
str(interaction.get("url", ""))
|
||||
):
|
||||
continue
|
||||
payload = _decode_json(str(interaction.get("response_body_base64", "")))
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
for key in ("id", "account_id"):
|
||||
value = payload.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
account_ids.add(value)
|
||||
display_name = payload.get("display_name")
|
||||
if isinstance(display_name, str) and display_name:
|
||||
display_names.add(display_name)
|
||||
return _ListenerIdentities(frozenset(account_ids), frozenset(display_names))
|
||||
|
||||
|
||||
def _redact_interaction(
|
||||
interaction: object,
|
||||
identities: _ListenerIdentities,
|
||||
synthetic_taste: dict[str, object],
|
||||
) -> dict[str, object]:
|
||||
if not isinstance(interaction, dict):
|
||||
raise ValueError("Cassette interaction must be a mapping")
|
||||
redacted = {str(key): value for key, value in interaction.items()}
|
||||
url = _replace_text(str(redacted.get("url", "")), identities)
|
||||
redacted["url"] = url
|
||||
redacted["request_body_base64"] = _redact_body(
|
||||
str(redacted.get("request_body_base64", "")), identities, None, synthetic_taste
|
||||
)
|
||||
replacement = _taste_replacement(url, synthetic_taste)
|
||||
original_body = str(redacted.get("response_body_base64", ""))
|
||||
response_body = _redact_body(original_body, identities, replacement, synthetic_taste)
|
||||
redacted["response_body_base64"] = response_body
|
||||
chunks = redacted.get("response_chunks_base64", [])
|
||||
if response_body != original_body:
|
||||
redacted["response_chunks_base64"] = [response_body] if response_body else []
|
||||
elif isinstance(chunks, list):
|
||||
redacted["response_chunks_base64"] = [
|
||||
_redact_body(str(chunk), identities, None, synthetic_taste) for chunk in chunks
|
||||
]
|
||||
return cast(dict[str, object], _redact_value(redacted, identities, synthetic_taste))
|
||||
|
||||
|
||||
def _redact_body(
|
||||
encoded: str,
|
||||
identities: _ListenerIdentities,
|
||||
replacement: object | None,
|
||||
synthetic_taste: dict[str, object],
|
||||
) -> str:
|
||||
try:
|
||||
raw = base64.b64decode(encoded, validate=True)
|
||||
except ValueError:
|
||||
raw = b""
|
||||
if replacement is not None:
|
||||
value = replacement
|
||||
else:
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
text = raw.decode("utf-8", errors="replace")
|
||||
return _encode(_replace_text(text, identities).encode())
|
||||
redacted = _redact_value(value, identities, synthetic_taste)
|
||||
return _encode(json.dumps(redacted, ensure_ascii=True, separators=(",", ":")).encode("ascii"))
|
||||
|
||||
|
||||
def _redact_value(
|
||||
value: object,
|
||||
identities: _ListenerIdentities,
|
||||
synthetic_taste: dict[str, object],
|
||||
) -> object:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
str(key): _taste_summary(synthetic_taste)
|
||||
if str(key) == "taste_profile"
|
||||
else _synthetic_identity(item, identities, synthetic_taste)
|
||||
for key, item in value.items()
|
||||
if not str(key).casefold().endswith("headers")
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [_redact_value(item, identities, synthetic_taste) for item in value]
|
||||
if isinstance(value, str):
|
||||
text = _replace_text(value, identities)
|
||||
try:
|
||||
nested = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return text
|
||||
if isinstance(nested, (dict, list)):
|
||||
return json.dumps(
|
||||
_redact_value(nested, identities, synthetic_taste),
|
||||
ensure_ascii=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return text
|
||||
return value
|
||||
|
||||
|
||||
def _synthetic_identity(
|
||||
value: object,
|
||||
identities: _ListenerIdentities,
|
||||
synthetic_taste: dict[str, object],
|
||||
) -> object:
|
||||
if isinstance(value, str) and value in identities.display_names:
|
||||
return SYNTHETIC_DISPLAY_NAME
|
||||
if isinstance(value, str) and value in identities.account_ids:
|
||||
return SYNTHETIC_ACCOUNT_ID
|
||||
return _redact_value(value, identities, synthetic_taste)
|
||||
|
||||
|
||||
def _replace_text(value: str, identities: _ListenerIdentities) -> str:
|
||||
redacted = BEARER_PATTERN.sub("[REDACTED]", value)
|
||||
replacements = {
|
||||
**{identity: SYNTHETIC_ACCOUNT_ID for identity in identities.account_ids},
|
||||
**{identity: SYNTHETIC_DISPLAY_NAME for identity in identities.display_names},
|
||||
}
|
||||
for identity, replacement in sorted(
|
||||
replacements.items(), key=lambda item: len(item[0]), reverse=True
|
||||
):
|
||||
redacted = redacted.replace(identity, replacement)
|
||||
return redacted
|
||||
|
||||
|
||||
def _taste_replacement(url: str, synthetic_taste: dict[str, object]) -> object | None:
|
||||
path = urlparse(url).path.rstrip("/")
|
||||
if path.endswith("/me/top/artists"):
|
||||
return synthetic_taste.get("top_artists")
|
||||
if path.endswith("/me/top/tracks"):
|
||||
return synthetic_taste.get("top_tracks")
|
||||
if path.endswith("/me/tracks"):
|
||||
return synthetic_taste.get("saved_tracks")
|
||||
return None
|
||||
|
||||
|
||||
def _is_current_user_url(url: str) -> bool:
|
||||
return urlparse(url).path.rstrip("/").endswith("/v1/me")
|
||||
|
||||
|
||||
def _taste_summary(synthetic_taste: dict[str, object]) -> str:
|
||||
summary = synthetic_taste.get("summary")
|
||||
if not isinstance(summary, str):
|
||||
raise ValueError("Synthetic taste fixture must contain a summary")
|
||||
return summary
|
||||
|
||||
|
||||
def _load_synthetic_taste() -> dict[str, object]:
|
||||
payload: object = json.loads(SYNTHETIC_TASTE_PATH.read_text(encoding="ascii"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("Synthetic taste fixture must be a mapping")
|
||||
return cast(dict[str, object], payload)
|
||||
|
||||
|
||||
def _decode_json(encoded: str) -> object:
|
||||
try:
|
||||
return json.loads(base64.b64decode(encoded, validate=True))
|
||||
except (ValueError, json.JSONDecodeError, UnicodeDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _encode(value: bytes) -> str:
|
||||
return base64.b64encode(value).decode("ascii")
|
||||
32
eval/recording/storage.py
Normal file
32
eval/recording/storage.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
"""Persist raw and redacted cassette stores."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from recording.model import RecordedInteraction
|
||||
from recording.redaction import redact_cassette
|
||||
|
||||
|
||||
def write_cassettes(
|
||||
fixture_root: Path,
|
||||
scenario_key: str,
|
||||
recordings: dict[str, list[RecordedInteraction]],
|
||||
) -> tuple[Path, Path]:
|
||||
"""Write ignored raw data first, then separately mapped safe cassettes."""
|
||||
raw_root = fixture_root / "raw" / scenario_key
|
||||
redacted_root = fixture_root / scenario_key
|
||||
raw_root.mkdir(parents=True, exist_ok=True)
|
||||
redacted_root.mkdir(parents=True, exist_ok=True)
|
||||
for service, interactions in recordings.items():
|
||||
raw_payload = [interaction.as_dict() for interaction in interactions]
|
||||
_write_json(raw_root / f"{service}.json", raw_payload)
|
||||
redacted_payload = redact_cassette(raw_payload)
|
||||
_write_json(redacted_root / f"{service}.json", redacted_payload)
|
||||
return raw_root, redacted_root
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: object) -> None:
|
||||
path.write_text(
|
||||
json.dumps(payload, ensure_ascii=True, indent=2, sort_keys=True) + "\n",
|
||||
encoding="ascii",
|
||||
)
|
||||
95
eval/recording/synthetic_taste.json
Normal file
95
eval/recording/synthetic_taste.json
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
{
|
||||
"saved_tracks": {
|
||||
"href": "https://api.spotify.com/v1/me/tracks",
|
||||
"items": [
|
||||
{
|
||||
"added_at": "2026-01-01T00:00:00Z",
|
||||
"track": {
|
||||
"album": {
|
||||
"images": [],
|
||||
"name": "Synthetic Saved Album"
|
||||
},
|
||||
"artists": [
|
||||
{
|
||||
"name": "Synthetic Saved Artist"
|
||||
}
|
||||
],
|
||||
"external_urls": {
|
||||
"spotify": "https://open.spotify.com/track/synthetic-saved-track"
|
||||
},
|
||||
"id": "synthetic-saved-track",
|
||||
"name": "Synthetic Saved Track",
|
||||
"uri": "spotify:track:synthetic-saved-track"
|
||||
}
|
||||
}
|
||||
],
|
||||
"limit": 50,
|
||||
"next": null,
|
||||
"offset": 0,
|
||||
"previous": null,
|
||||
"total": 1
|
||||
},
|
||||
"summary": "Short-term top artists: Synthetic Focus Artist; Long-term top artists: Synthetic Jazz Artist; Short-term top tracks: Synthetic Focus Track by Synthetic Focus Artist; Long-term top tracks: Synthetic Jazz Track by Synthetic Jazz Artist; Saved-track sample: Synthetic Saved Track by Synthetic Saved Artist",
|
||||
"top_artists": {
|
||||
"href": "https://api.spotify.com/v1/me/top/artists",
|
||||
"items": [
|
||||
{
|
||||
"id": "synthetic-focus-artist",
|
||||
"name": "Synthetic Focus Artist"
|
||||
},
|
||||
{
|
||||
"id": "synthetic-jazz-artist",
|
||||
"name": "Synthetic Jazz Artist"
|
||||
}
|
||||
],
|
||||
"limit": 50,
|
||||
"next": null,
|
||||
"offset": 0,
|
||||
"previous": null,
|
||||
"total": 2
|
||||
},
|
||||
"top_tracks": {
|
||||
"href": "https://api.spotify.com/v1/me/top/tracks",
|
||||
"items": [
|
||||
{
|
||||
"album": {
|
||||
"images": [],
|
||||
"name": "Synthetic Focus Album"
|
||||
},
|
||||
"artists": [
|
||||
{
|
||||
"name": "Synthetic Focus Artist"
|
||||
}
|
||||
],
|
||||
"external_urls": {
|
||||
"spotify": "https://open.spotify.com/track/synthetic-focus-track"
|
||||
},
|
||||
"id": "synthetic-focus-track",
|
||||
"name": "Synthetic Focus Track",
|
||||
"uri": "spotify:track:synthetic-focus-track"
|
||||
},
|
||||
{
|
||||
"album": {
|
||||
"images": [],
|
||||
"name": "Synthetic Jazz Album"
|
||||
},
|
||||
"artists": [
|
||||
{
|
||||
"name": "Synthetic Jazz Artist"
|
||||
}
|
||||
],
|
||||
"external_urls": {
|
||||
"spotify": "https://open.spotify.com/track/synthetic-jazz-track"
|
||||
},
|
||||
"id": "synthetic-jazz-track",
|
||||
"name": "Synthetic Jazz Track",
|
||||
"uri": "spotify:track:synthetic-jazz-track"
|
||||
}
|
||||
],
|
||||
"limit": 50,
|
||||
"next": null,
|
||||
"offset": 0,
|
||||
"previous": null,
|
||||
"total": 2
|
||||
}
|
||||
}
|
||||
154
eval/recording/transport.py
Normal file
154
eval/recording/transport.py
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
"""Recording transports for Spotify's httpx2 and Anthropic's httpx."""
|
||||
|
||||
from collections.abc import AsyncIterator, Callable
|
||||
from typing import cast
|
||||
|
||||
import httpx
|
||||
import httpx2
|
||||
|
||||
from recording.model import RecordedInteraction
|
||||
from recording.redaction import is_sensitive_url
|
||||
|
||||
|
||||
class HttpxRecordingTransport(httpx.AsyncBaseTransport):
|
||||
"""Wrap an httpx transport and retain completed response chunk sequences."""
|
||||
|
||||
def __init__(self, transport: httpx.AsyncBaseTransport) -> None:
|
||||
"""Bind the real transport and start an empty in-memory cassette."""
|
||||
self.transport = transport
|
||||
self.interactions: list[RecordedInteraction] = []
|
||||
|
||||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
||||
"""Forward one request and wrap its response stream for capture."""
|
||||
request.headers["Accept-Encoding"] = "identity"
|
||||
request_body = await request.aread()
|
||||
response = await self.transport.handle_async_request(request)
|
||||
if is_sensitive_url(str(request.url)):
|
||||
return response
|
||||
stream = _HttpxRecordingStream(
|
||||
cast(httpx.AsyncByteStream, response.stream),
|
||||
lambda chunks: self._finish(request, request_body, response.status_code, chunks),
|
||||
)
|
||||
return httpx.Response(
|
||||
response.status_code,
|
||||
headers=response.headers,
|
||||
stream=stream,
|
||||
extensions=response.extensions,
|
||||
request=request,
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""Close the wrapped transport."""
|
||||
await self.transport.aclose()
|
||||
|
||||
def _finish(
|
||||
self,
|
||||
request: httpx.Request,
|
||||
request_body: bytes,
|
||||
status: int,
|
||||
chunks: tuple[bytes, ...],
|
||||
) -> None:
|
||||
self.interactions.append(
|
||||
RecordedInteraction(request.method, str(request.url), status, request_body, chunks)
|
||||
)
|
||||
|
||||
|
||||
class Httpx2RecordingTransport(httpx2.AsyncBaseTransport):
|
||||
"""Wrap an httpx2 transport and retain completed response chunk sequences."""
|
||||
|
||||
def __init__(self, transport: httpx2.AsyncBaseTransport) -> None:
|
||||
"""Bind the real transport and start an empty in-memory cassette."""
|
||||
self.transport = transport
|
||||
self.interactions: list[RecordedInteraction] = []
|
||||
|
||||
async def handle_async_request(self, request: httpx2.Request) -> httpx2.Response:
|
||||
"""Forward one request and wrap its response stream for capture."""
|
||||
request.headers["Accept-Encoding"] = "identity"
|
||||
request_body = await request.aread()
|
||||
response = await self.transport.handle_async_request(request)
|
||||
if is_sensitive_url(str(request.url)):
|
||||
return response
|
||||
stream = _Httpx2RecordingStream(
|
||||
cast(httpx2.AsyncByteStream, response.stream),
|
||||
lambda chunks: self._finish(request, request_body, response.status_code, chunks),
|
||||
)
|
||||
return httpx2.Response(
|
||||
response.status_code,
|
||||
headers=response.headers,
|
||||
stream=stream,
|
||||
extensions=response.extensions,
|
||||
request=request,
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""Close the wrapped transport."""
|
||||
await self.transport.aclose()
|
||||
|
||||
def _finish(
|
||||
self,
|
||||
request: httpx2.Request,
|
||||
request_body: bytes,
|
||||
status: int,
|
||||
chunks: tuple[bytes, ...],
|
||||
) -> None:
|
||||
self.interactions.append(
|
||||
RecordedInteraction(request.method, str(request.url), status, request_body, chunks)
|
||||
)
|
||||
|
||||
|
||||
class _HttpxRecordingStream(httpx.AsyncByteStream):
|
||||
def __init__(
|
||||
self,
|
||||
stream: httpx.AsyncByteStream,
|
||||
finish: Callable[[tuple[bytes, ...]], None],
|
||||
) -> None:
|
||||
self.stream = stream
|
||||
self.finish = finish
|
||||
self.chunks: list[bytes] = []
|
||||
self.is_finished = False
|
||||
|
||||
async def __aiter__(self) -> AsyncIterator[bytes]:
|
||||
try:
|
||||
async for chunk in self.stream:
|
||||
self.chunks.append(chunk)
|
||||
yield chunk
|
||||
finally:
|
||||
self._finish()
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self.stream.aclose()
|
||||
self._finish()
|
||||
|
||||
def _finish(self) -> None:
|
||||
if not self.is_finished:
|
||||
self.is_finished = True
|
||||
self.finish(tuple(self.chunks))
|
||||
|
||||
|
||||
class _Httpx2RecordingStream(httpx2.AsyncByteStream):
|
||||
def __init__(
|
||||
self,
|
||||
stream: httpx2.AsyncByteStream,
|
||||
finish: Callable[[tuple[bytes, ...]], None],
|
||||
) -> None:
|
||||
self.stream = stream
|
||||
self.finish = finish
|
||||
self.chunks: list[bytes] = []
|
||||
self.is_finished = False
|
||||
|
||||
async def __aiter__(self) -> AsyncIterator[bytes]:
|
||||
try:
|
||||
async for chunk in self.stream:
|
||||
self.chunks.append(chunk)
|
||||
yield chunk
|
||||
finally:
|
||||
self._finish()
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self.stream.aclose()
|
||||
self._finish()
|
||||
|
||||
def _finish(self) -> None:
|
||||
if not self.is_finished:
|
||||
self.is_finished = True
|
||||
self.finish(tuple(self.chunks))
|
||||
Loading…
Add table
Add a link
Reference in a new issue