fix: harden request handling, pipeline containment, and startup boundaries

backend/tests/test_anthropic_llm.py

@@ -0,0 +1,13 @@
+"""Focused tests for Anthropic response parsing."""
+
+import pytest
+
+from app.adapters.anthropic.llm import _RecommendationObjectParser
+from app.ports.protocols import RecommenderOutputError
+
+
+def test_parser_missing_object_start_raises_typed_output_error() -> None:
+    parser = _RecommendationObjectParser()
+
+    with pytest.raises(RecommenderOutputError, match="object start position"):
+        parser._finish_object()

backend/tests/test_auth_routes.py

@@ -113,6 +113,29 @@ def test_seed_session_authenticates_requests_without_a_cookie() -> None:
     assert response.json() == {"display_name": "Seed Listener"}
 
 
+def test_seed_session_failure_keeps_application_serving() -> None:
+    async def spotify_handler(request: httpx2.Request) -> httpx2.Response:
+        return httpx2.Response(400)
+
+    app = create_app(
+        application_settings=Settings(
+            app_mode=AppMode.LIVE,
+            spotify_client_id="client-id",
+            anthropic_api_key="test-key",
+            spotify_seed_refresh_token="seed-refresh",
+        ),
+        http_transport=httpx2.MockTransport(spotify_handler),
+    )
+    with TestClient(app, follow_redirects=False) as client:
+        health_response = client.get("/api/health")
+        login_response = client.get("/api/auth/login")
+
+        assert app.state.seed_session_id is None
+
+    assert health_response.status_code == 200
+    assert login_response.status_code == 307
+
+
 def _live_settings() -> Settings:
     return Settings(
         app_mode=AppMode.LIVE,

backend/tests/test_grounding.py

@@ -1,8 +1,14 @@
 """Deterministic tests for bounded Spotify grounding."""
 
 import asyncio
+import time
 from collections.abc import Awaitable, Callable
 
+from app.adapters.spotify.errors import (
+    SpotifyQuotaExhaustedError,
+    SpotifyRateLimitedError,
+    SpotifyUnavailableError,
+)
 from app.config import Settings
 from app.domain.models import Familiarity, Track, TrackCandidate
 from app.pipeline.grounding import Grounder
@@ -47,7 +53,14 @@ def test_early_stop_honors_pool_target() -> None:
         grounder = Grounder(_settings(grounding_concurrency=2))
         candidates = tuple(TrackCandidate(f"track-{index}", "Artist") for index in range(6))
 
-        result = await grounder.ground(catalog, candidates, frozenset(), Familiarity.MIX, 2)
+        result = await grounder.ground(
+            catalog,
+            candidates,
+            frozenset(),
+            Familiarity.MIX,
+            2,
+            _deadline(),
+        )
 
         assert len(result.tracks) == 2
         assert len(catalog.search_queries) == 2
@@ -74,6 +87,7 @@ def test_miss_and_mismatch_are_counted_separately() -> None:
             frozenset(),
             Familiarity.MIX,
             2,
+            _deadline(),
         )
 
         assert result.metrics.miss_count == 1
@@ -92,9 +106,23 @@ def test_resolution_cache_hit_skips_catalog() -> None:
         grounder = Grounder(_settings())
         candidates = (TrackCandidate("Cached Song", "Artist"),)
 
-        await grounder.ground(catalog, candidates, frozenset(), Familiarity.MIX, 1)
+        await grounder.ground(
+            catalog,
+            candidates,
+            frozenset(),
+            Familiarity.MIX,
+            1,
+            _deadline(),
+        )
         first_call_count = len(catalog.search_queries)
-        second = await grounder.ground(catalog, candidates, frozenset(), Familiarity.MIX, 1)
+        second = await grounder.ground(
+            catalog,
+            candidates,
+            frozenset(),
+            Familiarity.MIX,
+            1,
+            _deadline(),
+        )
 
         assert len(catalog.search_queries) == first_call_count
         assert second.metrics.cache_hit_count == 1
@@ -124,6 +152,7 @@ def test_deadline_returns_resolved_partial_pool() -> None:
             frozenset(),
             Familiarity.MIX,
             2,
+            time.monotonic() + settings.request_deadline_seconds,
         )
 
         assert [track.id for track in result.tracks] == ["fast"]
@@ -132,6 +161,127 @@ def test_deadline_returns_resolved_partial_pool() -> None:
     asyncio.run(run())
 
 
+def test_spotify_failure_is_counted_and_remaining_candidates_continue() -> None:
+    async def run() -> None:
+        async def search(query: str) -> list[Track]:
+            if "Failing Song" in query:
+                raise SpotifyUnavailableError(503)
+            title = query.split('track:"', 1)[1].split('"', 1)[0]
+            return [_track(title.lower().replace(" ", "-"), title, "Artist")]
+
+        catalog = FakeCatalog(search)
+        candidates = (
+            TrackCandidate("Failing Song", "Artist"),
+            TrackCandidate("First Good Song", "Artist"),
+            TrackCandidate("Second Good Song", "Artist"),
+        )
+
+        result = await Grounder(_settings()).ground(
+            catalog,
+            candidates,
+            frozenset(),
+            Familiarity.MIX,
+            2,
+            _deadline(),
+        )
+
+        assert [track.title for track in result.tracks] == [
+            "First Good Song",
+            "Second Good Song",
+        ]
+        assert result.metrics.failed_count == 1
+        assert result.metrics.attempted_count == 3
+        assert not result.metrics.did_exhaust_quota
+
+    asyncio.run(run())
+
+
+def test_plain_rate_limit_continues_but_quota_exhaustion_stops_fanout() -> None:
+    async def run() -> None:
+        async def plain_rate_limited_search(query: str) -> list[Track]:
+            if "Rate Limited" in query:
+                raise SpotifyRateLimitedError(6.0)
+            return [_track("found", "Found Song", "Artist")]
+
+        plain_catalog = FakeCatalog(plain_rate_limited_search)
+        candidates = (
+            TrackCandidate("Rate Limited", "Artist"),
+            TrackCandidate("Found Song", "Artist"),
+        )
+        plain_result = await Grounder(_settings()).ground(
+            plain_catalog,
+            candidates,
+            frozenset(),
+            Familiarity.MIX,
+            1,
+            _deadline(),
+        )
+
+        async def quota_search(query: str) -> list[Track]:
+            raise SpotifyQuotaExhaustedError(0.0, "QUOTA_EXCEEDED")
+
+        quota_catalog = FakeCatalog(quota_search)
+        quota_result = await Grounder(_settings()).ground(
+            quota_catalog,
+            candidates,
+            frozenset(),
+            Familiarity.MIX,
+            1,
+            _deadline(),
+        )
+
+        assert [track.id for track in plain_result.tracks] == ["found"]
+        assert plain_result.metrics.failed_count == 1
+        assert not plain_result.metrics.did_exhaust_quota
+        assert quota_result.tracks == ()
+        assert quota_result.metrics.did_exhaust_quota
+        assert len(quota_catalog.search_queries) == 1
+
+    asyncio.run(run())
+
+
+def test_grounding_concurrency_is_shared_across_requests() -> None:
+    async def run() -> None:
+        active_searches = 0
+        maximum_active_searches = 0
+
+        async def search(query: str) -> list[Track]:
+            nonlocal active_searches, maximum_active_searches
+            active_searches += 1
+            maximum_active_searches = max(maximum_active_searches, active_searches)
+            await asyncio.sleep(0.01)
+            active_searches -= 1
+            title = query.split('track:"', 1)[1].split('"', 1)[0]
+            return [_track(title, title, "Artist")]
+
+        catalog = FakeCatalog(search)
+        grounder = Grounder(_settings(grounding_concurrency=2))
+        candidates = tuple(TrackCandidate(f"Song {index}", "Artist") for index in range(2))
+
+        await asyncio.gather(
+            grounder.ground(
+                catalog,
+                candidates,
+                frozenset(),
+                Familiarity.MIX,
+                2,
+                _deadline(),
+            ),
+            grounder.ground(
+                catalog,
+                candidates,
+                frozenset(),
+                Familiarity.MIX,
+                2,
+                _deadline(),
+            ),
+        )
+
+        assert maximum_active_searches == 2
+
+    asyncio.run(run())
+
+
 def _settings(**overrides: object) -> Settings:
     values: dict[str, object] = {
         "grounding_concurrency": 1,
@@ -141,6 +291,10 @@ def _settings(**overrides: object) -> Settings:
     return Settings.model_validate(values)
 
 
+def _deadline() -> float:
+    return time.monotonic() + 1.0
+
+
 def _track(track_id: str, title: str, artist: str) -> Track:
     return Track(
         id=track_id,

backend/tests/test_health.py

@@ -1,7 +1,11 @@
 """Smoke test for the application factory."""
 
+from unittest.mock import AsyncMock, Mock
+
+import pytest
 from fastapi.testclient import TestClient
 
+from app.config import Settings
 from app.main import create_app
 
 
@@ -10,3 +14,16 @@ def test_health_reports_mode() -> None:
     response = client.get("/api/health")
     assert response.status_code == 200
     assert response.json()["mode"] in ("live", "demo")
+
+
+def test_anthropic_client_uses_configured_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
+    anthropic_client = Mock()
+    anthropic_client.close = AsyncMock()
+    constructor = Mock(return_value=anthropic_client)
+    monkeypatch.setattr("app.main.AsyncAnthropic", constructor)
+
+    with TestClient(create_app(Settings(llm_timeout_seconds=42.0))) as client:
+        response = client.get("/api/health")
+
+    assert response.status_code == 200
+    constructor.assert_called_once_with(api_key="unused-demo-key", timeout=42.0)

backend/tests/test_orchestrator.py

@@ -3,6 +3,7 @@
 import asyncio
 from collections.abc import AsyncGenerator
 
+from app.adapters.spotify.errors import SpotifyQuotaExhaustedError, SpotifyUnavailableError
 from app.config import Settings
 from app.domain.models import (
     ConversationTurn,
@@ -13,7 +14,7 @@ from app.domain.models import (
     Track,
     TrackCandidate,
 )
-from app.pipeline.event import PipelineEvent, PipelineTrackEvent
+from app.pipeline.event import PipelineErrorEvent, PipelineEvent, PipelineTrackEvent
 from app.pipeline.orchestrator import RecommendationPipeline
 from app.ports.protocols import RecommenderOutputError, TimeRange
 
@@ -61,11 +62,15 @@ class FakeRecommender:
         intents: list[Intent],
         selection_ids: tuple[str, ...] = (),
         failure_count: int = 0,
+        intent_error: Exception | None = None,
+        intent_delay_seconds: float = 0.0,
     ) -> None:
         """Store deterministic outputs for successive calls."""
         self.intents = intents
         self.selection_ids = selection_ids
         self.failure_count = failure_count
+        self.intent_error = intent_error
+        self.intent_delay_seconds = intent_delay_seconds
         self.rerank_call_count = 0
 
     async def create_intent(
@@ -77,6 +82,9 @@ class FakeRecommender:
         candidate_count: int,
     ) -> Intent:
         """Return the next fixed intent."""
+        await asyncio.sleep(self.intent_delay_seconds)
+        if self.intent_error is not None:
+            raise self.intent_error
         return self.intents.pop(0)
 
     async def stream_rerank(
@@ -115,6 +123,108 @@ def test_event_order_and_rerank_ids_stay_inside_grounded_pool() -> None:
     asyncio.run(run())
 
 
+def test_intent_stage_failures_yield_one_typed_error_event() -> None:
+    async def run() -> None:
+        cases = (
+            (RecommenderOutputError("invalid intent"), "intent_failed"),
+            (
+                SpotifyQuotaExhaustedError(0.0, "QUOTA_EXCEEDED"),
+                "quota_exhausted",
+            ),
+            (SpotifyUnavailableError(503), "spotify_unavailable"),
+        )
+        for error, expected_code in cases:
+            catalog = FakeCatalog(())
+            recommender = FakeRecommender([], intent_error=error)
+
+            events = await _run_pipeline(catalog, recommender)
+
+            assert len(events) == 1
+            assert events[0].type == "error"
+            assert events[0].code == expected_code
+
+    asyncio.run(run())
+
+
+def test_taste_failure_cancels_sibling_fetches() -> None:
+    class FailingTasteCatalog(FakeCatalog):
+        """Fail one taste request after all sibling requests have started."""
+
+        def __init__(self) -> None:
+            super().__init__(())
+            self.started_count = 0
+            self.cancelled_count = 0
+            self.all_started = asyncio.Event()
+            self.never_finishes = asyncio.Event()
+
+        async def fetch_top_artists(self, time_range: TimeRange, limit: int) -> list[str]:
+            await self._wait_for_all_fetches()
+            await self._wait_until_cancelled()
+            return []
+
+        async def fetch_top_tracks(self, time_range: TimeRange, limit: int) -> list[Track]:
+            await self._wait_for_all_fetches()
+            await self._wait_until_cancelled()
+            return []
+
+        async def fetch_saved_tracks(self, limit: int) -> list[Track]:
+            await self._wait_for_all_fetches()
+            raise SpotifyUnavailableError(503)
+
+        async def _wait_for_all_fetches(self) -> None:
+            self.started_count += 1
+            if self.started_count == 5:
+                self.all_started.set()
+            await self.all_started.wait()
+
+        async def _wait_until_cancelled(self) -> None:
+            try:
+                await self.never_finishes.wait()
+            except asyncio.CancelledError:
+                self.cancelled_count += 1
+                raise
+
+    async def run() -> None:
+        catalog = FailingTasteCatalog()
+        events = await _run_pipeline(catalog, FakeRecommender([]))
+
+        assert len(events) == 1
+        assert events[0].type == "error"
+        assert events[0].code == "spotify_unavailable"
+        assert catalog.cancelled_count == 4
+
+    asyncio.run(run())
+
+
+def test_grounding_deadline_starts_before_intent_generation() -> None:
+    async def run() -> None:
+        track = _track("found", "Found Song")
+        catalog = FakeCatalog((track,))
+        recommender = FakeRecommender(
+            [_intent(track)],
+            intent_delay_seconds=0.02,
+        )
+        pipeline = RecommendationPipeline(
+            recommender,
+            Settings(
+                rerank_count=1,
+                rerank_pool_buffer=0,
+                grounding_floor=1,
+                grounding_concurrency=1,
+                request_deadline_seconds=0.01,
+            ),
+        )
+
+        events = await _collect(pipeline, catalog, "query")
+
+        assert [event.type for event in events] == ["metadata", "error"]
+        assert isinstance(events[-1], PipelineErrorEvent)
+        assert events[-1].code == "no_grounded_results"
+        assert catalog.search_call_count == 0
+
+    asyncio.run(run())
+
+
 def test_rerank_fallback_warns_then_streams_grounded_order() -> None:
     async def run() -> None:
         first = _track("first", "First Song")

backend/tests/test_recommendation_api.py

@@ -3,10 +3,12 @@
 import time
 from collections.abc import AsyncGenerator
 
+import pytest
 from fastapi.testclient import TestClient
 from pydantic import TypeAdapter
 
 from app.adapters.spotify.auth import TokenSet
+from app.adapters.spotify.errors import SpotifyAuthenticationError
 from app.adapters.spotify.session import SessionStore, SpotifySession
 from app.api.routes import SESSION_COOKIE_NAME
 from app.api.schemas import StreamEvent
@@ -37,13 +39,16 @@ class FakePipeline:
 class FakePlaylistWriter:
     """Capture playlist writes without external calls."""
 
-    def __init__(self) -> None:
+    def __init__(self, error: Exception | None = None) -> None:
         """Create an empty write trace."""
+        self.error = error
         self.name: str | None = None
         self.track_uris: list[str] = []
 
     async def create_playlist(self, name: str, description: str) -> CreatedPlaylist:
         """Record the prefixed name and return a stable playlist."""
+        if self.error is not None:
+            raise self.error
         self.name = name
         return CreatedPlaylist("playlist", "https://open.spotify.com/playlist/playlist")
 
@@ -52,7 +57,13 @@ class FakePlaylistWriter:
         self.track_uris = track_uris
 
 
-def test_recommendations_stream_lines_validate_against_frozen_schemas() -> None:
+def test_recommendations_stream_lines_validate_against_frozen_schemas(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    async def fail_if_polled(request: object) -> bool:
+        raise AssertionError("Request disconnect state must not be polled")
+
+    monkeypatch.setattr("starlette.requests.Request.is_disconnected", fail_if_polled)
     app = create_app()
     with TestClient(app) as client:
         _authenticate(client, session_store=app.state.session_store)
@@ -103,6 +114,26 @@ def test_playlist_endpoint_prefixes_name_and_adds_tracks() -> None:
     assert writer.track_uris == ["spotify:track:track"]
 
 
+def test_playlist_authentication_failure_signals_relogin() -> None:
+    app = create_app()
+    writer = FakePlaylistWriter(SpotifyAuthenticationError("expired"))
+    with TestClient(app) as client:
+        _authenticate(client, session_store=app.state.session_store)
+        app.state.spotify_client_factory = lambda session: writer
+
+        response = client.post(
+            "/api/playlists",
+            json={
+                "schema_version": 1,
+                "name": "Night drive",
+                "track_uris": ["spotify:track:track"],
+            },
+        )
+
+    assert response.status_code == 401
+    assert response.json() == {"detail": "Spotify authentication expired"}
+
+
 def _authenticate(client: TestClient, session_store: SessionStore) -> None:
     session_id = session_store.create(
         SpotifySession(

backend/tests/test_spotify_auth.py

@@ -6,12 +6,14 @@ import hashlib
 import time
 
 import httpx2
+import pytest
 
 from app.adapters.spotify.auth import (
     TokenSet,
     derive_code_challenge,
     refresh_access_token,
 )
+from app.adapters.spotify.session import PendingLogins
 
 
 def test_code_challenge_is_unpadded_base64url_sha256() -> None:
@@ -43,3 +45,18 @@ def test_refresh_keeps_existing_refresh_token_when_omitted() -> None:
 def test_token_expiry_uses_sixty_second_skew() -> None:
     assert TokenSet("access", "refresh", time.monotonic() + 59).is_expired
     assert not TokenSet("access", "refresh", time.monotonic() + 61).is_expired
+
+
+def test_pending_login_add_sweeps_expired_entries(monkeypatch: pytest.MonkeyPatch) -> None:
+    current_time = 0.0
+    monkeypatch.setattr(
+        "app.adapters.spotify.session.time.monotonic",
+        lambda: current_time,
+    )
+    pending_logins = PendingLogins()
+    pending_logins.add("expired", "old-verifier")
+
+    current_time = 601.0
+    pending_logins.add("current", "new-verifier")
+
+    assert set(pending_logins._entries) == {"current"}

backend/tests/test_spotify_client.py

@@ -9,10 +9,16 @@ import pytest
 
 from app.adapters.spotify.auth import TokenSet
 from app.adapters.spotify.client import SpotifyClient
-from app.adapters.spotify.errors import SpotifyRateLimitedError, SpotifyRequestError
+from app.adapters.spotify.errors import (
+    SpotifyQuotaExhaustedError,
+    SpotifyRateLimitedError,
+    SpotifyRequestError,
+    SpotifyUnavailableError,
+)
 from app.adapters.spotify.session import SpotifySession
 from app.config import Settings
 from app.domain.models import Track
+from app.ports.protocols import CatalogQuotaExhaustedError
 
 TransportHandler = Callable[[httpx2.Request], Coroutine[None, None, httpx2.Response]]
 
@@ -44,19 +50,19 @@ def test_unauthorized_response_refreshes_once_and_returns_result() -> None:
 def test_concurrent_unauthorized_responses_share_one_refresh() -> None:
     async def run() -> None:
         token_calls = 0
-        old_api_calls = 0
-        both_old_requests_arrived = asyncio.Event()
+        api_calls = 0
+        both_initial_requests_arrived = asyncio.Event()
 
         async def handler(request: httpx2.Request) -> httpx2.Response:
-            nonlocal old_api_calls, token_calls
+            nonlocal api_calls, token_calls
             if request.url.host == "accounts.spotify.com":
                 token_calls += 1
-                return _token_response()
-            if request.headers["Authorization"] == "Bearer old-access":
-                old_api_calls += 1
-                if old_api_calls == 2:
-                    both_old_requests_arrived.set()
-                await asyncio.wait_for(both_old_requests_arrived.wait(), timeout=1)
+                return _token_response("old-access")
+            api_calls += 1
+            if api_calls <= 2:
+                if api_calls == 2:
+                    both_initial_requests_arrived.set()
+                await asyncio.wait_for(both_initial_requests_arrived.wait(), timeout=1)
                 return httpx2.Response(401)
             return httpx2.Response(200, json=_search_payload())
 
@@ -69,6 +75,7 @@ def test_concurrent_unauthorized_responses_share_one_refresh() -> None:
 
         assert first == second
         assert token_calls == 1
+        assert client.session.refresh_generation == 1
 
     asyncio.run(run())
 
@@ -105,6 +112,7 @@ def test_get_rate_limit_above_cap_raises_without_retry() -> None:
             await _search_with_handler(handler)
 
         assert error.value.retry_after_seconds == 6
+        assert not isinstance(error.value, CatalogQuotaExhaustedError)
         assert api_calls == 1
 
     asyncio.run(run())
@@ -136,10 +144,11 @@ def test_quota_exhaustion_raises_without_retry_or_sleep(
             )
 
         monkeypatch.setattr("app.adapters.spotify.client.asyncio.sleep", fake_sleep)
-        with pytest.raises(SpotifyRateLimitedError) as error:
+        with pytest.raises(SpotifyQuotaExhaustedError) as error:
             await _search_with_handler(handler)
 
         assert error.value.reason == "QUOTA_EXCEEDED"
+        assert isinstance(error.value, CatalogQuotaExhaustedError)
         assert api_calls == 1
         assert sleep_calls == 0
 
@@ -172,6 +181,46 @@ def test_authentication_then_rate_limit_retries_each_policy_once() -> None:
     asyncio.run(run())
 
 
+def test_rate_limit_retry_landing_on_unauthorized_refreshes_once() -> None:
+    async def run() -> None:
+        token_calls = 0
+        api_calls = 0
+
+        async def handler(request: httpx2.Request) -> httpx2.Response:
+            nonlocal api_calls, token_calls
+            if request.url.host == "accounts.spotify.com":
+                token_calls += 1
+                return _token_response()
+            api_calls += 1
+            if api_calls == 1:
+                return httpx2.Response(429, headers={"Retry-After": "0"})
+            if api_calls == 2:
+                return httpx2.Response(401)
+            return httpx2.Response(200, json=_search_payload())
+
+        tracks = await _search_with_handler(handler)
+
+        assert len(tracks) == 1
+        assert token_calls == 1
+        assert api_calls == 3
+
+    asyncio.run(run())
+
+
+def test_transport_error_becomes_spotify_unavailable() -> None:
+    async def run() -> None:
+        async def handler(request: httpx2.Request) -> httpx2.Response:
+            raise httpx2.ConnectError("connection failed", request=request)
+
+        with pytest.raises(SpotifyUnavailableError) as error:
+            await _search_with_handler(handler)
+
+        assert error.value.status_code == 504
+        assert str(error.value) == "Spotify request failed"
+
+    asyncio.run(run())
+
+
 def test_request_error_includes_parsed_spotify_message() -> None:
     async def run() -> None:
         async def handler(request: httpx2.Request) -> httpx2.Response:
@@ -267,8 +316,8 @@ def _client(http: httpx2.AsyncClient) -> SpotifyClient:
     return SpotifyClient(http, session, Settings(spotify_client_id="client"))
 
 
-def _token_response() -> httpx2.Response:
-    return httpx2.Response(200, json={"access_token": "new-access", "expires_in": 3600})
+def _token_response(access_token: str = "new-access") -> httpx2.Response:
+    return httpx2.Response(200, json={"access_token": access_token, "expires_in": 3600})
 
 
 def _search_payload() -> dict[str, object]: