test: add scenario eval runner, recorded fixtures, and baseline comparison

eval/recording/redaction.py

@@ -0,0 +1,219 @@
+"""Remove credentials and listener identity from persisted cassettes."""
+
+import base64
+import copy
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast
+from urllib.parse import urlparse
+
+SYNTHETIC_ACCOUNT_ID = "synthetic-account"
+SYNTHETIC_DISPLAY_NAME = "Synthetic Listener"
+SYNTHETIC_TASTE_PATH = Path(__file__).with_name("synthetic_taste.json")
+BEARER_PATTERN = re.compile(r"Bearer\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE)
+SENSITIVE_PATH_PARTS = {"authorize", "oauth", "token"}
+
+
+@dataclass(frozen=True)
+class _ListenerIdentities:
+    account_ids: frozenset[str]
+    display_names: frozenset[str]
+
+
+def is_sensitive_url(url: str) -> bool:
+    """Return whether an authentication exchange must never be recorded."""
+    parsed = urlparse(url)
+    path_parts = {part.casefold() for part in parsed.path.split("/") if part}
+    return parsed.hostname == "accounts.spotify.com" or bool(path_parts & SENSITIVE_PATH_PARTS)
+
+
+def redact_cassette(interactions: list[dict[str, object]]) -> list[dict[str, object]]:
+    """Return safe interactions without mutating or persisting the source."""
+    safe = [
+        _drop_headers(copy.deepcopy(interaction))
+        for interaction in interactions
+        if not is_sensitive_url(str(interaction.get("url", "")))
+    ]
+    identities = _collect_identities(safe)
+    synthetic_taste = _load_synthetic_taste()
+    return [_redact_interaction(interaction, identities, synthetic_taste) for interaction in safe]
+
+
+def _drop_headers(value: object) -> object:
+    if isinstance(value, dict):
+        return {
+            key: _drop_headers(item)
+            for key, item in value.items()
+            if not str(key).casefold().endswith("headers")
+        }
+    if isinstance(value, list):
+        return [_drop_headers(item) for item in value]
+    return value
+
+
+def _collect_identities(interactions: list[object]) -> _ListenerIdentities:
+    account_ids: set[str] = set()
+    display_names: set[str] = set()
+    for interaction in interactions:
+        if not isinstance(interaction, dict) or not _is_current_user_url(
+            str(interaction.get("url", ""))
+        ):
+            continue
+        payload = _decode_json(str(interaction.get("response_body_base64", "")))
+        if not isinstance(payload, dict):
+            continue
+        for key in ("id", "account_id"):
+            value = payload.get(key)
+            if isinstance(value, str) and value:
+                account_ids.add(value)
+        display_name = payload.get("display_name")
+        if isinstance(display_name, str) and display_name:
+            display_names.add(display_name)
+    return _ListenerIdentities(frozenset(account_ids), frozenset(display_names))
+
+
+def _redact_interaction(
+    interaction: object,
+    identities: _ListenerIdentities,
+    synthetic_taste: dict[str, object],
+) -> dict[str, object]:
+    if not isinstance(interaction, dict):
+        raise ValueError("Cassette interaction must be a mapping")
+    redacted = {str(key): value for key, value in interaction.items()}
+    url = _replace_text(str(redacted.get("url", "")), identities)
+    redacted["url"] = url
+    redacted["request_body_base64"] = _redact_body(
+        str(redacted.get("request_body_base64", "")), identities, None, synthetic_taste
+    )
+    replacement = _taste_replacement(url, synthetic_taste)
+    original_body = str(redacted.get("response_body_base64", ""))
+    response_body = _redact_body(original_body, identities, replacement, synthetic_taste)
+    redacted["response_body_base64"] = response_body
+    chunks = redacted.get("response_chunks_base64", [])
+    if response_body != original_body:
+        redacted["response_chunks_base64"] = [response_body] if response_body else []
+    elif isinstance(chunks, list):
+        redacted["response_chunks_base64"] = [
+            _redact_body(str(chunk), identities, None, synthetic_taste) for chunk in chunks
+        ]
+    return cast(dict[str, object], _redact_value(redacted, identities, synthetic_taste))
+
+
+def _redact_body(
+    encoded: str,
+    identities: _ListenerIdentities,
+    replacement: object | None,
+    synthetic_taste: dict[str, object],
+) -> str:
+    try:
+        raw = base64.b64decode(encoded, validate=True)
+    except ValueError:
+        raw = b""
+    if replacement is not None:
+        value = replacement
+    else:
+        try:
+            value = json.loads(raw)
+        except (json.JSONDecodeError, UnicodeDecodeError):
+            text = raw.decode("utf-8", errors="replace")
+            return _encode(_replace_text(text, identities).encode())
+    redacted = _redact_value(value, identities, synthetic_taste)
+    return _encode(json.dumps(redacted, ensure_ascii=True, separators=(",", ":")).encode("ascii"))
+
+
+def _redact_value(
+    value: object,
+    identities: _ListenerIdentities,
+    synthetic_taste: dict[str, object],
+) -> object:
+    if isinstance(value, dict):
+        return {
+            str(key): _taste_summary(synthetic_taste)
+            if str(key) == "taste_profile"
+            else _synthetic_identity(item, identities, synthetic_taste)
+            for key, item in value.items()
+            if not str(key).casefold().endswith("headers")
+        }
+    if isinstance(value, list):
+        return [_redact_value(item, identities, synthetic_taste) for item in value]
+    if isinstance(value, str):
+        text = _replace_text(value, identities)
+        try:
+            nested = json.loads(text)
+        except json.JSONDecodeError:
+            return text
+        if isinstance(nested, (dict, list)):
+            return json.dumps(
+                _redact_value(nested, identities, synthetic_taste),
+                ensure_ascii=True,
+                separators=(",", ":"),
+            )
+        return text
+    return value
+
+
+def _synthetic_identity(
+    value: object,
+    identities: _ListenerIdentities,
+    synthetic_taste: dict[str, object],
+) -> object:
+    if isinstance(value, str) and value in identities.display_names:
+        return SYNTHETIC_DISPLAY_NAME
+    if isinstance(value, str) and value in identities.account_ids:
+        return SYNTHETIC_ACCOUNT_ID
+    return _redact_value(value, identities, synthetic_taste)
+
+
+def _replace_text(value: str, identities: _ListenerIdentities) -> str:
+    redacted = BEARER_PATTERN.sub("[REDACTED]", value)
+    replacements = {
+        **{identity: SYNTHETIC_ACCOUNT_ID for identity in identities.account_ids},
+        **{identity: SYNTHETIC_DISPLAY_NAME for identity in identities.display_names},
+    }
+    for identity, replacement in sorted(
+        replacements.items(), key=lambda item: len(item[0]), reverse=True
+    ):
+        redacted = redacted.replace(identity, replacement)
+    return redacted
+
+
+def _taste_replacement(url: str, synthetic_taste: dict[str, object]) -> object | None:
+    path = urlparse(url).path.rstrip("/")
+    if path.endswith("/me/top/artists"):
+        return synthetic_taste.get("top_artists")
+    if path.endswith("/me/top/tracks"):
+        return synthetic_taste.get("top_tracks")
+    if path.endswith("/me/tracks"):
+        return synthetic_taste.get("saved_tracks")
+    return None
+
+
+def _is_current_user_url(url: str) -> bool:
+    return urlparse(url).path.rstrip("/").endswith("/v1/me")
+
+
+def _taste_summary(synthetic_taste: dict[str, object]) -> str:
+    summary = synthetic_taste.get("summary")
+    if not isinstance(summary, str):
+        raise ValueError("Synthetic taste fixture must contain a summary")
+    return summary
+
+
+def _load_synthetic_taste() -> dict[str, object]:
+    payload: object = json.loads(SYNTHETIC_TASTE_PATH.read_text(encoding="ascii"))
+    if not isinstance(payload, dict):
+        raise ValueError("Synthetic taste fixture must be a mapping")
+    return cast(dict[str, object], payload)
+
+
+def _decode_json(encoded: str) -> object:
+    try:
+        return json.loads(base64.b64decode(encoded, validate=True))
+    except (ValueError, json.JSONDecodeError, UnicodeDecodeError):
+        return None
+
+
+def _encode(value: bytes) -> str:
+    return base64.b64encode(value).decode("ascii")