feat: add key-free demo mode replaying recorded fixtures

This commit is contained in:
Justin Visser 2026-08-11 09:14:22 +02:00
parent 0664cc2d27
commit 401a0ddea7
21 changed files with 1040 additions and 31 deletions

View file

@ -2,12 +2,12 @@
import pytest
from app.adapters.anthropic.llm import _RecommendationObjectParser
from app.adapters.anthropic.llm import RecommendationObjectParser
from app.ports.protocols import RecommenderOutputError
def test_parser_missing_object_start_raises_typed_output_error() -> None:
parser = _RecommendationObjectParser()
parser = RecommendationObjectParser()
with pytest.raises(RecommenderOutputError, match="object start position"):
parser._finish_object()

View file

@ -0,0 +1,68 @@
"""Offline replay tests for demo service adapters."""
import asyncio
from app.adapters.demo.cassette import load_cassette
from app.adapters.demo.catalog import DemoCatalog
from app.adapters.demo.recommender import DemoRecommender
from app.config import Settings
def test_cassette_decodes_bodies_and_preserves_rerank_chunks() -> None:
cassette = load_cassette("focus-coding")
assert cassette.intent_response_body.startswith(b'{"model"')
assert len(cassette.spotify_search_responses) > 20
assert len(cassette.spotify_taste_responses) == 6
assert len(cassette.rerank_response_chunks) > 1
assert b"event: content_block_delta" in b"".join(cassette.rerank_response_chunks)
def test_demo_catalog_replays_search_and_synthetic_taste_pages() -> None:
async def run() -> None:
catalog = DemoCatalog(load_cassette("focus-coding"))
tracks = await catalog.search_tracks('track:"Stay" artist:"Hybrid Minds"')
artists = await catalog.fetch_top_artists("short_term", 50)
top_tracks = await catalog.fetch_top_tracks("long_term", 50)
saved_tracks = await catalog.fetch_saved_tracks(100)
assert tracks[0].title == "Stay"
assert artists[0] == "Synthetic Focus Artist"
assert {track.title for track in top_tracks} >= {
"Synthetic Focus Track",
"Synthetic Jazz Track",
}
assert saved_tracks[0].title == "Synthetic Saved Track"
asyncio.run(run())
def test_demo_recommender_parses_intent_and_streams_recorded_rerank() -> None:
async def run() -> None:
recommender = DemoRecommender(Settings(demo_chunk_delay_seconds=0))
intent = await recommender.create_intent(
"Focus while coding",
(),
(),
"Synthetic taste",
35,
)
selections = [
selection
async for selection in recommender.stream_rerank(
intent,
(),
"Synthetic taste",
(),
15,
)
]
assert intent.activity == "programming"
assert len(intent.candidates) == 35
assert len(selections) == 15
assert selections[0].track_id == "2nIixNuuV5eHCJydG5aYIB"
assert all(selection.justification for selection in selections)
asyncio.run(run())

View file

@ -0,0 +1,208 @@
"""Full HTTP contract tests for key-free demo mode."""
import asyncio
import httpx
from fastapi.testclient import TestClient
from pydantic import TypeAdapter
from starlette.types import ASGIApp, Message, Receive, Scope, Send
from app.adapters.demo.scenario import load_scenarios
from app.api.schemas import StreamEvent
from app.config import Settings
from app.main import create_app
class _MetadataBarrier:
"""Pause one marked ASGI response immediately after its metadata event."""
def __init__(self, app: ASGIApp) -> None:
self.app = app
self.metadata_sent = asyncio.Event()
self.resume_response = asyncio.Event()
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
is_refinement = dict(scope.get("headers", ())).get(b"x-demo-request") == b"refinement"
async def send_with_barrier(message: Message) -> None:
await send(message)
if message["type"] == "http.response.body" and b'"type":"metadata"' in message.get(
"body", b""
):
self.metadata_sent.set()
await self.resume_response.wait()
await self.app(scope, receive, send_with_barrier if is_refinement else send)
def test_demo_request_streams_valid_events_without_a_session() -> None:
app = create_app(Settings(demo_chunk_delay_seconds=0))
with TestClient(app) as client:
response = client.post(
"/api/recommendations",
json={"schema_version": 1, "query": "Focus while coding"},
)
adapter: TypeAdapter[StreamEvent] = TypeAdapter(StreamEvent)
events = [adapter.validate_json(line) for line in response.text.splitlines()]
assert response.status_code == 200
assert events[0].type == "metadata"
assert events[-1].type == "done"
assert sum(event.type == "track" for event in events) == 15
def test_unknown_demo_query_discloses_the_replayed_scenario_before_tracks() -> None:
app = create_app(Settings(demo_chunk_delay_seconds=0))
with TestClient(app) as client:
response = client.post(
"/api/recommendations",
json={"schema_version": 1, "query": "Focus while codign"},
)
adapter: TypeAdapter[StreamEvent] = TypeAdapter(StreamEvent)
events = [adapter.validate_json(line) for line in response.text.splitlines()]
warning_index = next(
index
for index, event in enumerate(events)
if event.type == "warning" and event.code == "demo_replay"
)
first_track_index = next(index for index, event in enumerate(events) if event.type == "track")
warning = events[warning_index]
assert warning_index < first_track_index
assert warning.type == "warning"
assert "Focus while coding" in warning.message
def test_demo_refinement_reconstructs_the_pool_recorded_with_its_cassette() -> None:
app = create_app(Settings(demo_chunk_delay_seconds=0))
with TestClient(app) as client:
initial_response = client.post(
"/api/recommendations",
json={"schema_version": 1, "query": "Focus while coding"},
)
initial_adapter: TypeAdapter[StreamEvent] = TypeAdapter(StreamEvent)
initial_events: list[StreamEvent] = [
initial_adapter.validate_json(line) for line in initial_response.text.splitlines()
]
prior_recommendations = [
{
"rank": event.rank,
"track_id": event.track.id,
"title": event.track.title,
"artists": event.track.artists,
}
for event in initial_events
if event.type == "track"
]
response = client.post(
"/api/recommendations",
json={
"schema_version": 1,
"query": "More electronic",
"prior_recommendations": prior_recommendations,
},
)
adapter: TypeAdapter[StreamEvent] = TypeAdapter(StreamEvent)
events = [adapter.validate_json(line) for line in response.text.splitlines()]
warning_codes = [event.code for event in events if event.type == "warning"]
assert "rerank_fallback" not in warning_codes
assert sum(event.type == "track" for event in events) == 15
def test_concurrent_demo_refinement_uses_its_request_local_recorded_pool() -> None:
async def run() -> None:
settings = Settings(demo_chunk_delay_seconds=0)
previous = [
{
"rank": 1,
"track_id": "previous",
"title": "Previous track",
"artists": ["Previous artist"],
}
]
payload = {
"schema_version": 1,
"query": "More electronic",
"prior_recommendations": previous,
}
baseline_app = create_app(settings)
baseline_transport = httpx.ASGITransport(app=baseline_app)
async with (
baseline_app.router.lifespan_context(baseline_app),
httpx.AsyncClient(
transport=baseline_transport,
base_url="http://test",
) as client,
):
baseline_response = await client.post("/api/recommendations", json=payload)
baseline_events = _stream_events(baseline_response)
expected_track_ids = [event.track.id for event in baseline_events if event.type == "track"]
concurrent_app = create_app(settings)
barrier = _MetadataBarrier(concurrent_app)
transport = httpx.ASGITransport(app=barrier)
async with (
concurrent_app.router.lifespan_context(concurrent_app),
httpx.AsyncClient(transport=transport, base_url="http://test") as client,
):
refinement_task = asyncio.create_task(
client.post(
"/api/recommendations",
json=payload,
headers={"x-demo-request": "refinement"},
)
)
try:
await asyncio.wait_for(barrier.metadata_sent.wait(), timeout=2)
unrelated_response = await client.post(
"/api/recommendations",
json={"schema_version": 1, "query": "Energy for the gym"},
)
finally:
barrier.resume_response.set()
refinement_response = await refinement_task
events = _stream_events(refinement_response)
warning_codes = [event.code for event in events if event.type == "warning"]
track_ids = [event.track.id for event in events if event.type == "track"]
assert unrelated_response.status_code == 200
assert refinement_response.status_code == 200
assert "rerank_fallback" not in warning_codes
assert track_ids == expected_track_ids
asyncio.run(run())
def test_demo_auth_playlist_and_suggestions_are_explicitly_simulated() -> None:
app = create_app(Settings(demo_chunk_delay_seconds=0))
with TestClient(app, follow_redirects=False) as client:
current_user = client.get("/api/auth/me")
login = client.get("/api/auth/login")
playlist = client.post(
"/api/playlists",
json={
"schema_version": 1,
"name": "Night drive",
"track_uris": ["spotify:track:demo"],
},
)
suggestions = client.get("/api/suggestions")
expected_suggestions = [
{"chip": scenario.chip, "query": scenario.query}
for scenario in load_scenarios()
if not scenario.is_refinement
]
assert current_user.json() == {"display_name": "Demo Listener"}
assert login.headers["location"] == "/?login=demo"
assert playlist.json()["url"].startswith("https://open.spotify.com/playlist/demo-")
assert suggestions.json() == expected_suggestions
assert not hasattr(app.state, "http")
assert not hasattr(app.state, "anthropic")
def _stream_events(response: httpx.Response) -> list[StreamEvent]:
adapter: TypeAdapter[StreamEvent] = TypeAdapter(StreamEvent)
return [adapter.validate_json(line) for line in response.text.splitlines()]

View file

@ -0,0 +1,32 @@
"""Tests for deterministic demo scenario selection."""
from app.adapters.demo.scenario import select_replay_scenario, select_scenario
def test_scenario_selection_prefers_normalized_exact_query_or_chip() -> None:
query_match = select_scenario("something calm for while I am programming, but not boring")
chip_match = select_scenario(" FOCUS, while CODING! ")
assert query_match.key == "focus-coding"
assert query_match.is_exact
assert chip_match.key == "focus-coding"
assert chip_match.is_exact
def test_scenario_selection_falls_back_to_nearest_fixture() -> None:
match = select_scenario("Focus while codign")
assert match.key == "focus-coding"
assert not match.is_exact
def test_prior_results_select_a_matching_refinement_or_same_scenario() -> None:
refinement = select_replay_scenario("Focus while coding", True)
direct_refinement = select_replay_scenario("More electronic", False)
same_scenario = select_replay_scenario("Rainy Sunday", True)
assert refinement.key == "focus-coding-refine"
assert not refinement.is_exact
assert direct_refinement.key == "focus-coding-refine"
assert direct_refinement.is_exact
assert same_scenario.key == "rainy-sunday"

View file

@ -22,8 +22,14 @@ def test_anthropic_client_uses_configured_timeout(monkeypatch: pytest.MonkeyPatc
constructor = Mock(return_value=anthropic_client)
monkeypatch.setattr("app.main.AsyncAnthropic", constructor)
with TestClient(create_app(Settings(llm_timeout_seconds=42.0))) as client:
live_settings = Settings(
app_mode="live",
spotify_client_id="client-id",
anthropic_api_key="api-key",
llm_timeout_seconds=42.0,
)
with TestClient(create_app(live_settings)) as client:
response = client.get("/api/health")
assert response.status_code == 200
constructor.assert_called_once_with(api_key="unused-demo-key", timeout=42.0, http_client=None)
constructor.assert_called_once_with(api_key="api-key", timeout=42.0, http_client=None)

View file

@ -12,6 +12,7 @@ from app.adapters.spotify.errors import SpotifyAuthenticationError
from app.adapters.spotify.session import SessionStore, SpotifySession
from app.api.routes import SESSION_COOKIE_NAME
from app.api.schemas import StreamEvent
from app.config import AppMode, Settings
from app.domain.models import ConversationTurn, CreatedPlaylist, PreviousRecommendation, Track
from app.main import create_app
from app.pipeline.event import PipelineDoneEvent, PipelineMetadataEvent, PipelineTrackEvent
@ -81,8 +82,13 @@ def test_recommendations_stream_lines_validate_against_frozen_schemas(
assert [event.type for event in events] == ["metadata", "track", "done"]
def test_recommendations_require_a_valid_session_without_seed() -> None:
with TestClient(create_app()) as client:
def test_live_recommendations_require_a_valid_session_without_seed() -> None:
application_settings = Settings(
app_mode=AppMode.LIVE,
spotify_client_id="client-id",
anthropic_api_key="test-key",
)
with TestClient(create_app(application_settings)) as client:
response = client.post(
"/api/recommendations",
json={"schema_version": 1, "query": "focused electronic music"},