fix: harden request handling, pipeline containment, and startup boundaries

backend/tests/test_grounding.py

@@ -1,8 +1,14 @@
 """Deterministic tests for bounded Spotify grounding."""
 
 import asyncio
+import time
 from collections.abc import Awaitable, Callable
 
+from app.adapters.spotify.errors import (
+    SpotifyQuotaExhaustedError,
+    SpotifyRateLimitedError,
+    SpotifyUnavailableError,
+)
 from app.config import Settings
 from app.domain.models import Familiarity, Track, TrackCandidate
 from app.pipeline.grounding import Grounder
@@ -47,7 +53,14 @@ def test_early_stop_honors_pool_target() -> None:
         grounder = Grounder(_settings(grounding_concurrency=2))
         candidates = tuple(TrackCandidate(f"track-{index}", "Artist") for index in range(6))
 
-        result = await grounder.ground(catalog, candidates, frozenset(), Familiarity.MIX, 2)
+        result = await grounder.ground(
+            catalog,
+            candidates,
+            frozenset(),
+            Familiarity.MIX,
+            2,
+            _deadline(),
+        )
 
         assert len(result.tracks) == 2
         assert len(catalog.search_queries) == 2
@@ -74,6 +87,7 @@ def test_miss_and_mismatch_are_counted_separately() -> None:
             frozenset(),
             Familiarity.MIX,
             2,
+            _deadline(),
         )
 
         assert result.metrics.miss_count == 1
@@ -92,9 +106,23 @@ def test_resolution_cache_hit_skips_catalog() -> None:
         grounder = Grounder(_settings())
         candidates = (TrackCandidate("Cached Song", "Artist"),)
 
-        await grounder.ground(catalog, candidates, frozenset(), Familiarity.MIX, 1)
+        await grounder.ground(
+            catalog,
+            candidates,
+            frozenset(),
+            Familiarity.MIX,
+            1,
+            _deadline(),
+        )
         first_call_count = len(catalog.search_queries)
-        second = await grounder.ground(catalog, candidates, frozenset(), Familiarity.MIX, 1)
+        second = await grounder.ground(
+            catalog,
+            candidates,
+            frozenset(),
+            Familiarity.MIX,
+            1,
+            _deadline(),
+        )
 
         assert len(catalog.search_queries) == first_call_count
         assert second.metrics.cache_hit_count == 1
@@ -124,6 +152,7 @@ def test_deadline_returns_resolved_partial_pool() -> None:
             frozenset(),
             Familiarity.MIX,
             2,
+            time.monotonic() + settings.request_deadline_seconds,
         )
 
         assert [track.id for track in result.tracks] == ["fast"]
@@ -132,6 +161,127 @@ def test_deadline_returns_resolved_partial_pool() -> None:
     asyncio.run(run())
 
 
+def test_spotify_failure_is_counted_and_remaining_candidates_continue() -> None:
+    async def run() -> None:
+        async def search(query: str) -> list[Track]:
+            if "Failing Song" in query:
+                raise SpotifyUnavailableError(503)
+            title = query.split('track:"', 1)[1].split('"', 1)[0]
+            return [_track(title.lower().replace(" ", "-"), title, "Artist")]
+
+        catalog = FakeCatalog(search)
+        candidates = (
+            TrackCandidate("Failing Song", "Artist"),
+            TrackCandidate("First Good Song", "Artist"),
+            TrackCandidate("Second Good Song", "Artist"),
+        )
+
+        result = await Grounder(_settings()).ground(
+            catalog,
+            candidates,
+            frozenset(),
+            Familiarity.MIX,
+            2,
+            _deadline(),
+        )
+
+        assert [track.title for track in result.tracks] == [
+            "First Good Song",
+            "Second Good Song",
+        ]
+        assert result.metrics.failed_count == 1
+        assert result.metrics.attempted_count == 3
+        assert not result.metrics.did_exhaust_quota
+
+    asyncio.run(run())
+
+
+def test_plain_rate_limit_continues_but_quota_exhaustion_stops_fanout() -> None:
+    async def run() -> None:
+        async def plain_rate_limited_search(query: str) -> list[Track]:
+            if "Rate Limited" in query:
+                raise SpotifyRateLimitedError(6.0)
+            return [_track("found", "Found Song", "Artist")]
+
+        plain_catalog = FakeCatalog(plain_rate_limited_search)
+        candidates = (
+            TrackCandidate("Rate Limited", "Artist"),
+            TrackCandidate("Found Song", "Artist"),
+        )
+        plain_result = await Grounder(_settings()).ground(
+            plain_catalog,
+            candidates,
+            frozenset(),
+            Familiarity.MIX,
+            1,
+            _deadline(),
+        )
+
+        async def quota_search(query: str) -> list[Track]:
+            raise SpotifyQuotaExhaustedError(0.0, "QUOTA_EXCEEDED")
+
+        quota_catalog = FakeCatalog(quota_search)
+        quota_result = await Grounder(_settings()).ground(
+            quota_catalog,
+            candidates,
+            frozenset(),
+            Familiarity.MIX,
+            1,
+            _deadline(),
+        )
+
+        assert [track.id for track in plain_result.tracks] == ["found"]
+        assert plain_result.metrics.failed_count == 1
+        assert not plain_result.metrics.did_exhaust_quota
+        assert quota_result.tracks == ()
+        assert quota_result.metrics.did_exhaust_quota
+        assert len(quota_catalog.search_queries) == 1
+
+    asyncio.run(run())
+
+
+def test_grounding_concurrency_is_shared_across_requests() -> None:
+    async def run() -> None:
+        active_searches = 0
+        maximum_active_searches = 0
+
+        async def search(query: str) -> list[Track]:
+            nonlocal active_searches, maximum_active_searches
+            active_searches += 1
+            maximum_active_searches = max(maximum_active_searches, active_searches)
+            await asyncio.sleep(0.01)
+            active_searches -= 1
+            title = query.split('track:"', 1)[1].split('"', 1)[0]
+            return [_track(title, title, "Artist")]
+
+        catalog = FakeCatalog(search)
+        grounder = Grounder(_settings(grounding_concurrency=2))
+        candidates = tuple(TrackCandidate(f"Song {index}", "Artist") for index in range(2))
+
+        await asyncio.gather(
+            grounder.ground(
+                catalog,
+                candidates,
+                frozenset(),
+                Familiarity.MIX,
+                2,
+                _deadline(),
+            ),
+            grounder.ground(
+                catalog,
+                candidates,
+                frozenset(),
+                Familiarity.MIX,
+                2,
+                _deadline(),
+            ),
+        )
+
+        assert maximum_active_searches == 2
+
+    asyncio.run(run())
+
+
 def _settings(**overrides: object) -> Settings:
     values: dict[str, object] = {
         "grounding_concurrency": 1,
@@ -141,6 +291,10 @@ def _settings(**overrides: object) -> Settings:
     return Settings.model_validate(values)
 
 
+def _deadline() -> float:
+    return time.monotonic() + 1.0
+
+
 def _track(track_id: str, title: str, artist: str) -> Track:
     return Track(
         id=track_id,