feat: add key-free demo mode replaying recorded fixtures
This commit is contained in:
parent
0664cc2d27
commit
401a0ddea7
21 changed files with 1040 additions and 31 deletions
|
|
@ -2,12 +2,12 @@
|
|||
|
||||
import pytest
|
||||
|
||||
from app.adapters.anthropic.llm import _RecommendationObjectParser
|
||||
from app.adapters.anthropic.llm import RecommendationObjectParser
|
||||
from app.ports.protocols import RecommenderOutputError
|
||||
|
||||
|
||||
def test_parser_missing_object_start_raises_typed_output_error() -> None:
|
||||
parser = _RecommendationObjectParser()
|
||||
parser = RecommendationObjectParser()
|
||||
|
||||
with pytest.raises(RecommenderOutputError, match="object start position"):
|
||||
parser._finish_object()
|
||||
|
|
|
|||
68
backend/tests/test_demo_adapters.py
Normal file
68
backend/tests/test_demo_adapters.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
"""Offline replay tests for demo service adapters."""
|
||||
|
||||
import asyncio
|
||||
|
||||
from app.adapters.demo.cassette import load_cassette
|
||||
from app.adapters.demo.catalog import DemoCatalog
|
||||
from app.adapters.demo.recommender import DemoRecommender
|
||||
from app.config import Settings
|
||||
|
||||
|
||||
def test_cassette_decodes_bodies_and_preserves_rerank_chunks() -> None:
|
||||
cassette = load_cassette("focus-coding")
|
||||
|
||||
assert cassette.intent_response_body.startswith(b'{"model"')
|
||||
assert len(cassette.spotify_search_responses) > 20
|
||||
assert len(cassette.spotify_taste_responses) == 6
|
||||
assert len(cassette.rerank_response_chunks) > 1
|
||||
assert b"event: content_block_delta" in b"".join(cassette.rerank_response_chunks)
|
||||
|
||||
|
||||
def test_demo_catalog_replays_search_and_synthetic_taste_pages() -> None:
|
||||
async def run() -> None:
|
||||
catalog = DemoCatalog(load_cassette("focus-coding"))
|
||||
|
||||
tracks = await catalog.search_tracks('track:"Stay" artist:"Hybrid Minds"')
|
||||
artists = await catalog.fetch_top_artists("short_term", 50)
|
||||
top_tracks = await catalog.fetch_top_tracks("long_term", 50)
|
||||
saved_tracks = await catalog.fetch_saved_tracks(100)
|
||||
|
||||
assert tracks[0].title == "Stay"
|
||||
assert artists[0] == "Synthetic Focus Artist"
|
||||
assert {track.title for track in top_tracks} >= {
|
||||
"Synthetic Focus Track",
|
||||
"Synthetic Jazz Track",
|
||||
}
|
||||
assert saved_tracks[0].title == "Synthetic Saved Track"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_demo_recommender_parses_intent_and_streams_recorded_rerank() -> None:
|
||||
async def run() -> None:
|
||||
recommender = DemoRecommender(Settings(demo_chunk_delay_seconds=0))
|
||||
intent = await recommender.create_intent(
|
||||
"Focus while coding",
|
||||
(),
|
||||
(),
|
||||
"Synthetic taste",
|
||||
35,
|
||||
)
|
||||
selections = [
|
||||
selection
|
||||
async for selection in recommender.stream_rerank(
|
||||
intent,
|
||||
(),
|
||||
"Synthetic taste",
|
||||
(),
|
||||
15,
|
||||
)
|
||||
]
|
||||
|
||||
assert intent.activity == "programming"
|
||||
assert len(intent.candidates) == 35
|
||||
assert len(selections) == 15
|
||||
assert selections[0].track_id == "2nIixNuuV5eHCJydG5aYIB"
|
||||
assert all(selection.justification for selection in selections)
|
||||
|
||||
asyncio.run(run())
|
||||
208
backend/tests/test_demo_app.py
Normal file
208
backend/tests/test_demo_app.py
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
"""Full HTTP contract tests for key-free demo mode."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import httpx
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import TypeAdapter
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from app.adapters.demo.scenario import load_scenarios
|
||||
from app.api.schemas import StreamEvent
|
||||
from app.config import Settings
|
||||
from app.main import create_app
|
||||
|
||||
|
||||
class _MetadataBarrier:
|
||||
"""Pause one marked ASGI response immediately after its metadata event."""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
self.app = app
|
||||
self.metadata_sent = asyncio.Event()
|
||||
self.resume_response = asyncio.Event()
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
is_refinement = dict(scope.get("headers", ())).get(b"x-demo-request") == b"refinement"
|
||||
|
||||
async def send_with_barrier(message: Message) -> None:
|
||||
await send(message)
|
||||
if message["type"] == "http.response.body" and b'"type":"metadata"' in message.get(
|
||||
"body", b""
|
||||
):
|
||||
self.metadata_sent.set()
|
||||
await self.resume_response.wait()
|
||||
|
||||
await self.app(scope, receive, send_with_barrier if is_refinement else send)
|
||||
|
||||
|
||||
def test_demo_request_streams_valid_events_without_a_session() -> None:
|
||||
app = create_app(Settings(demo_chunk_delay_seconds=0))
|
||||
with TestClient(app) as client:
|
||||
response = client.post(
|
||||
"/api/recommendations",
|
||||
json={"schema_version": 1, "query": "Focus while coding"},
|
||||
)
|
||||
|
||||
adapter: TypeAdapter[StreamEvent] = TypeAdapter(StreamEvent)
|
||||
events = [adapter.validate_json(line) for line in response.text.splitlines()]
|
||||
assert response.status_code == 200
|
||||
assert events[0].type == "metadata"
|
||||
assert events[-1].type == "done"
|
||||
assert sum(event.type == "track" for event in events) == 15
|
||||
|
||||
|
||||
def test_unknown_demo_query_discloses_the_replayed_scenario_before_tracks() -> None:
|
||||
app = create_app(Settings(demo_chunk_delay_seconds=0))
|
||||
with TestClient(app) as client:
|
||||
response = client.post(
|
||||
"/api/recommendations",
|
||||
json={"schema_version": 1, "query": "Focus while codign"},
|
||||
)
|
||||
|
||||
adapter: TypeAdapter[StreamEvent] = TypeAdapter(StreamEvent)
|
||||
events = [adapter.validate_json(line) for line in response.text.splitlines()]
|
||||
warning_index = next(
|
||||
index
|
||||
for index, event in enumerate(events)
|
||||
if event.type == "warning" and event.code == "demo_replay"
|
||||
)
|
||||
first_track_index = next(index for index, event in enumerate(events) if event.type == "track")
|
||||
warning = events[warning_index]
|
||||
assert warning_index < first_track_index
|
||||
assert warning.type == "warning"
|
||||
assert "Focus while coding" in warning.message
|
||||
|
||||
|
||||
def test_demo_refinement_reconstructs_the_pool_recorded_with_its_cassette() -> None:
|
||||
app = create_app(Settings(demo_chunk_delay_seconds=0))
|
||||
with TestClient(app) as client:
|
||||
initial_response = client.post(
|
||||
"/api/recommendations",
|
||||
json={"schema_version": 1, "query": "Focus while coding"},
|
||||
)
|
||||
initial_adapter: TypeAdapter[StreamEvent] = TypeAdapter(StreamEvent)
|
||||
initial_events: list[StreamEvent] = [
|
||||
initial_adapter.validate_json(line) for line in initial_response.text.splitlines()
|
||||
]
|
||||
prior_recommendations = [
|
||||
{
|
||||
"rank": event.rank,
|
||||
"track_id": event.track.id,
|
||||
"title": event.track.title,
|
||||
"artists": event.track.artists,
|
||||
}
|
||||
for event in initial_events
|
||||
if event.type == "track"
|
||||
]
|
||||
response = client.post(
|
||||
"/api/recommendations",
|
||||
json={
|
||||
"schema_version": 1,
|
||||
"query": "More electronic",
|
||||
"prior_recommendations": prior_recommendations,
|
||||
},
|
||||
)
|
||||
|
||||
adapter: TypeAdapter[StreamEvent] = TypeAdapter(StreamEvent)
|
||||
events = [adapter.validate_json(line) for line in response.text.splitlines()]
|
||||
warning_codes = [event.code for event in events if event.type == "warning"]
|
||||
assert "rerank_fallback" not in warning_codes
|
||||
assert sum(event.type == "track" for event in events) == 15
|
||||
|
||||
|
||||
def test_concurrent_demo_refinement_uses_its_request_local_recorded_pool() -> None:
|
||||
async def run() -> None:
|
||||
settings = Settings(demo_chunk_delay_seconds=0)
|
||||
previous = [
|
||||
{
|
||||
"rank": 1,
|
||||
"track_id": "previous",
|
||||
"title": "Previous track",
|
||||
"artists": ["Previous artist"],
|
||||
}
|
||||
]
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"query": "More electronic",
|
||||
"prior_recommendations": previous,
|
||||
}
|
||||
|
||||
baseline_app = create_app(settings)
|
||||
baseline_transport = httpx.ASGITransport(app=baseline_app)
|
||||
async with (
|
||||
baseline_app.router.lifespan_context(baseline_app),
|
||||
httpx.AsyncClient(
|
||||
transport=baseline_transport,
|
||||
base_url="http://test",
|
||||
) as client,
|
||||
):
|
||||
baseline_response = await client.post("/api/recommendations", json=payload)
|
||||
baseline_events = _stream_events(baseline_response)
|
||||
expected_track_ids = [event.track.id for event in baseline_events if event.type == "track"]
|
||||
|
||||
concurrent_app = create_app(settings)
|
||||
barrier = _MetadataBarrier(concurrent_app)
|
||||
transport = httpx.ASGITransport(app=barrier)
|
||||
async with (
|
||||
concurrent_app.router.lifespan_context(concurrent_app),
|
||||
httpx.AsyncClient(transport=transport, base_url="http://test") as client,
|
||||
):
|
||||
refinement_task = asyncio.create_task(
|
||||
client.post(
|
||||
"/api/recommendations",
|
||||
json=payload,
|
||||
headers={"x-demo-request": "refinement"},
|
||||
)
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(barrier.metadata_sent.wait(), timeout=2)
|
||||
unrelated_response = await client.post(
|
||||
"/api/recommendations",
|
||||
json={"schema_version": 1, "query": "Energy for the gym"},
|
||||
)
|
||||
finally:
|
||||
barrier.resume_response.set()
|
||||
refinement_response = await refinement_task
|
||||
|
||||
events = _stream_events(refinement_response)
|
||||
warning_codes = [event.code for event in events if event.type == "warning"]
|
||||
track_ids = [event.track.id for event in events if event.type == "track"]
|
||||
assert unrelated_response.status_code == 200
|
||||
assert refinement_response.status_code == 200
|
||||
assert "rerank_fallback" not in warning_codes
|
||||
assert track_ids == expected_track_ids
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_demo_auth_playlist_and_suggestions_are_explicitly_simulated() -> None:
|
||||
app = create_app(Settings(demo_chunk_delay_seconds=0))
|
||||
with TestClient(app, follow_redirects=False) as client:
|
||||
current_user = client.get("/api/auth/me")
|
||||
login = client.get("/api/auth/login")
|
||||
playlist = client.post(
|
||||
"/api/playlists",
|
||||
json={
|
||||
"schema_version": 1,
|
||||
"name": "Night drive",
|
||||
"track_uris": ["spotify:track:demo"],
|
||||
},
|
||||
)
|
||||
suggestions = client.get("/api/suggestions")
|
||||
|
||||
expected_suggestions = [
|
||||
{"chip": scenario.chip, "query": scenario.query}
|
||||
for scenario in load_scenarios()
|
||||
if not scenario.is_refinement
|
||||
]
|
||||
assert current_user.json() == {"display_name": "Demo Listener"}
|
||||
assert login.headers["location"] == "/?login=demo"
|
||||
assert playlist.json()["url"].startswith("https://open.spotify.com/playlist/demo-")
|
||||
assert suggestions.json() == expected_suggestions
|
||||
assert not hasattr(app.state, "http")
|
||||
assert not hasattr(app.state, "anthropic")
|
||||
|
||||
|
||||
def _stream_events(response: httpx.Response) -> list[StreamEvent]:
|
||||
adapter: TypeAdapter[StreamEvent] = TypeAdapter(StreamEvent)
|
||||
return [adapter.validate_json(line) for line in response.text.splitlines()]
|
||||
32
backend/tests/test_demo_scenario.py
Normal file
32
backend/tests/test_demo_scenario.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
"""Tests for deterministic demo scenario selection."""
|
||||
|
||||
from app.adapters.demo.scenario import select_replay_scenario, select_scenario
|
||||
|
||||
|
||||
def test_scenario_selection_prefers_normalized_exact_query_or_chip() -> None:
|
||||
query_match = select_scenario("something calm for while I am programming, but not boring")
|
||||
chip_match = select_scenario(" FOCUS, while CODING! ")
|
||||
|
||||
assert query_match.key == "focus-coding"
|
||||
assert query_match.is_exact
|
||||
assert chip_match.key == "focus-coding"
|
||||
assert chip_match.is_exact
|
||||
|
||||
|
||||
def test_scenario_selection_falls_back_to_nearest_fixture() -> None:
|
||||
match = select_scenario("Focus while codign")
|
||||
|
||||
assert match.key == "focus-coding"
|
||||
assert not match.is_exact
|
||||
|
||||
|
||||
def test_prior_results_select_a_matching_refinement_or_same_scenario() -> None:
|
||||
refinement = select_replay_scenario("Focus while coding", True)
|
||||
direct_refinement = select_replay_scenario("More electronic", False)
|
||||
same_scenario = select_replay_scenario("Rainy Sunday", True)
|
||||
|
||||
assert refinement.key == "focus-coding-refine"
|
||||
assert not refinement.is_exact
|
||||
assert direct_refinement.key == "focus-coding-refine"
|
||||
assert direct_refinement.is_exact
|
||||
assert same_scenario.key == "rainy-sunday"
|
||||
|
|
@ -22,8 +22,14 @@ def test_anthropic_client_uses_configured_timeout(monkeypatch: pytest.MonkeyPatc
|
|||
constructor = Mock(return_value=anthropic_client)
|
||||
monkeypatch.setattr("app.main.AsyncAnthropic", constructor)
|
||||
|
||||
with TestClient(create_app(Settings(llm_timeout_seconds=42.0))) as client:
|
||||
live_settings = Settings(
|
||||
app_mode="live",
|
||||
spotify_client_id="client-id",
|
||||
anthropic_api_key="api-key",
|
||||
llm_timeout_seconds=42.0,
|
||||
)
|
||||
with TestClient(create_app(live_settings)) as client:
|
||||
response = client.get("/api/health")
|
||||
|
||||
assert response.status_code == 200
|
||||
constructor.assert_called_once_with(api_key="unused-demo-key", timeout=42.0, http_client=None)
|
||||
constructor.assert_called_once_with(api_key="api-key", timeout=42.0, http_client=None)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from app.adapters.spotify.errors import SpotifyAuthenticationError
|
|||
from app.adapters.spotify.session import SessionStore, SpotifySession
|
||||
from app.api.routes import SESSION_COOKIE_NAME
|
||||
from app.api.schemas import StreamEvent
|
||||
from app.config import AppMode, Settings
|
||||
from app.domain.models import ConversationTurn, CreatedPlaylist, PreviousRecommendation, Track
|
||||
from app.main import create_app
|
||||
from app.pipeline.event import PipelineDoneEvent, PipelineMetadataEvent, PipelineTrackEvent
|
||||
|
|
@ -81,8 +82,13 @@ def test_recommendations_stream_lines_validate_against_frozen_schemas(
|
|||
assert [event.type for event in events] == ["metadata", "track", "done"]
|
||||
|
||||
|
||||
def test_recommendations_require_a_valid_session_without_seed() -> None:
|
||||
with TestClient(create_app()) as client:
|
||||
def test_live_recommendations_require_a_valid_session_without_seed() -> None:
|
||||
application_settings = Settings(
|
||||
app_mode=AppMode.LIVE,
|
||||
spotify_client_id="client-id",
|
||||
anthropic_api_key="test-key",
|
||||
)
|
||||
with TestClient(create_app(application_settings)) as client:
|
||||
response = client.post(
|
||||
"/api/recommendations",
|
||||
json={"schema_version": 1, "query": "focused electronic music"},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue