justinzeus/discovery-by-llm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: harden request handling, pipeline containment, and startup boundaries

Browse source
This commit is contained in:
Justin Visser 2026-08-10 21:50:48 +02:00
parent 2cc33a721c
commit 3555256a02
18 changed files with 656 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

3
backend/app/adapters/anthropic/llm.py
View file

@ -221,7 +221,8 @@ class _RecommendationObjectParser:
return None
def _finish_object(self) -> RerankSelectionOutput:
assert self._object_start is not None
if self._object_start is None:
raise RecommenderOutputError("Rerank parser lost the object start position")
object_text = self.complete_text[self._object_start : self._scan_index + 1]
self._object_start = None
try:

103
backend/app/adapters/spotify/client.py
View file

@ -7,6 +7,7 @@ import httpx2
from app.adapters.spotify.auth import refresh_access_token
from app.adapters.spotify.errors import (
SpotifyAuthenticationError,
SpotifyQuotaExhaustedError,
SpotifyRateLimitedError,
SpotifyRequestError,
SpotifyUnavailableError,
@ -119,7 +120,7 @@ class SpotifyClient:
params: dict[str, str | int] | None = None,
json: dict[str, object] | None = None,
) -> httpx2.Response:
access_token = await self._access_token()
access_token, refresh_generation = await self._access_token()
response = await self._send(
method,
path,
@ -127,18 +128,27 @@ class SpotifyClient:
params=params,
json=json,
)
response = await self._retry_once_if_unauthorized(
response, refresh_generation = await self._retry_once_if_unauthorized(
response,
method,
path,
access_token,
refresh_generation,
params=params,
json=json,
)
response = await self._retry_once_if_rate_limited(
response, refresh_generation = await self._retry_once_if_rate_limited(
response,
method,
path,
refresh_generation,
params=params,
json=json,
)
response, _ = await self._retry_once_if_unauthorized(
response,
method,
path,
refresh_generation,
params=params,
json=json,
)
@ -154,34 +164,41 @@ class SpotifyClient:
json: dict[str, object] | None,
) -> httpx2.Response:
increment_spotify_calls()
return await self.http.request(
method,
f"{self.settings.spotify_api_base_url.rstrip('/')}{path}",
params=params,
json=json,
headers={"Authorization": f"Bearer {access_token}"},
)
try:
return await self.http.request(
method,
f"{self.settings.spotify_api_base_url.rstrip('/')}{path}",
params=params,
json=json,
headers={"Authorization": f"Bearer {access_token}"},
)
except httpx2.HTTPError as error:
raise SpotifyUnavailableError(504, "Spotify request failed") from error
async def _retry_once_if_unauthorized(
self,
response: httpx2.Response,
method: str,
path: str,
access_token: str,
refresh_generation: int,
*,
params: dict[str, str | int] | None,
json: dict[str, object] | None,
) -> httpx2.Response:
) -> tuple[httpx2.Response, int]:
if response.status_code != 401:
return response
return response, refresh_generation
await self._refresh_if_current(access_token)
return await self._send(
method,
path,
self.session.tokens.access_token,
params=params,
json=json,
await self._refresh_if_current(refresh_generation)
access_token, retry_generation = self._token_snapshot()
return (
await self._send(
method,
path,
access_token,
params=params,
json=json,
),
retry_generation,
)
async def _retry_once_if_rate_limited(
@ -189,12 +206,13 @@ class SpotifyClient:
response: httpx2.Response,
method: str,
path: str,
refresh_generation: int,
*,
params: dict[str, str | int] | None,
json: dict[str, object] | None,
) -> httpx2.Response:
) -> tuple[httpx2.Response, int]:
if response.status_code != 429:
return response
return response, refresh_generation
retry_after_seconds = _parse_retry_after(response)
_, reason = _parse_error_details(response)
@ -204,15 +222,19 @@ class SpotifyClient:
or retry_after_seconds is None
or retry_after_seconds > self.settings.spotify_retry_after_cap_seconds
):
return response
return response, refresh_generation
await asyncio.sleep(retry_after_seconds)
return await self._send(
method,
path,
self.session.tokens.access_token,
params=params,
json=json,
access_token, retry_generation = self._token_snapshot()
return (
await self._send(
method,
path,
access_token,
params=params,
json=json,
),
retry_generation,
)
def _raise_for_error(self, response: httpx2.Response) -> httpx2.Response:
@ -223,26 +245,33 @@ class SpotifyClient:
if response.status_code == 401:
raise SpotifyAuthenticationError("Spotify rejected refreshed authentication")
if response.status_code == 429:
if reason == "QUOTA_EXCEEDED":
raise SpotifyQuotaExhaustedError(_parse_retry_after(response), reason)
raise SpotifyRateLimitedError(_parse_retry_after(response), reason)
if response.status_code >= 500:
raise SpotifyUnavailableError(response.status_code, message)
raise SpotifyRequestError(response.status_code, message)
async def _access_token(self) -> str:
access_token = self.session.tokens.access_token
if self.session.tokens.is_expired:
await self._refresh_if_current(access_token)
return self.session.tokens.access_token
async def _access_token(self) -> tuple[str, int]:
tokens = self.session.tokens
refresh_generation = self.session.refresh_generation
if tokens.is_expired:
await self._refresh_if_current(refresh_generation)
return self._token_snapshot()
async def _refresh_if_current(self, access_token: str) -> None:
async def _refresh_if_current(self, refresh_generation: int) -> None:
async with self.session.refresh_lock:
if self.session.tokens.access_token != access_token:
if self.session.refresh_generation != refresh_generation:
return
self.session.tokens = await refresh_access_token(
self.http,
client_id=self.settings.spotify_client_id,
tokens=self.session.tokens,
)
self.session.refresh_generation += 1
def _token_snapshot(self) -> tuple[str, int]:
return self.session.tokens.access_token, self.session.refresh_generation
def _parse_retry_after(response: httpx2.Response) -> float | None:

6
backend/app/adapters/spotify/errors.py
View file

@ -11,7 +11,7 @@ class SpotifyAuthenticationError(SpotifyError):
"""Spotify rejected authentication or token refresh."""
class SpotifyRateLimitedError(SpotifyError, CatalogQuotaExhaustedError):
class SpotifyRateLimitedError(SpotifyError):
"""Spotify rate limited a request that could not be retried."""
def __init__(self, retry_after_seconds: float | None, reason: str | None = None) -> None:
@ -21,6 +21,10 @@ class SpotifyRateLimitedError(SpotifyError, CatalogQuotaExhaustedError):
self.reason = reason
class SpotifyQuotaExhaustedError(SpotifyRateLimitedError, CatalogQuotaExhaustedError):
"""Spotify rejected a request because the application quota is exhausted."""
class SpotifyUnavailableError(SpotifyError):
"""Spotify returned a server-side failure."""

11
backend/app/adapters/spotify/session.py
View file

@ -17,6 +17,7 @@ class SpotifySession:
tokens: TokenSet
account_id: str
display_name: str
refresh_generation: int = 0
refresh_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
@ -51,7 +52,15 @@ class PendingLogins:
def add(self, state: str, code_verifier: str) -> None:
"""Store a PKCE verifier for a newly issued OAuth state."""
self._entries[state] = (code_verifier, time.monotonic())
now = time.monotonic()
expired_states = (
pending_state
for pending_state, (_, created_at) in self._entries.items()
if now - created_at >= PENDING_LOGIN_LIFETIME_SECONDS
)
for expired_state in tuple(expired_states):
del self._entries[expired_state]
self._entries[state] = (code_verifier, now)
def pop(self, state: str) -> str | None:
"""Consume a verifier unless its OAuth state is unknown or expired."""

20
backend/app/api/recommendations.py
View file

@ -8,7 +8,7 @@ import structlog
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import StreamingResponse
from app.adapters.spotify.errors import SpotifyError
from app.adapters.spotify.errors import SpotifyAuthenticationError, SpotifyError
from app.adapters.spotify.session import SpotifySession
from app.api.routes import resolve_session
from app.api.schemas import (
@ -69,7 +69,6 @@ async def recommendations(
return StreamingResponse(
_stream_lines(
request,
pipeline,
resolved.session_id,
request_id,
@ -101,6 +100,9 @@ async def create_playlist(
"Music discovery selected by the listener.",
)
await writer.add_tracks_to_playlist(playlist.id, payload.track_uris)
except SpotifyAuthenticationError as error:
structlog.get_logger().warning("playlist_write_failed", error_type=type(error).__name__)
raise HTTPException(status_code=401, detail="Spotify authentication expired") from error
except (SpotifyError, ValueError) as error:
structlog.get_logger().warning("playlist_write_failed", error_type=type(error).__name__)
raise HTTPException(status_code=502, detail="Spotify playlist creation failed") from error
@ -108,7 +110,6 @@ async def create_playlist(
async def _stream_lines(
request: Request,
pipeline: RecommendationPipeline,
session_id: str,
request_id: str,
@ -127,8 +128,6 @@ async def _stream_lines(
)
try:
async for event in event_stream:
if await request.is_disconnected():
return
yield f"{_to_wire_event(event).model_dump_json()}\n"
except asyncio.CancelledError:
raise
@ -137,12 +136,11 @@ async def _stream_lines(
"recommendation_stream_failed",
error_type=type(error).__name__,
)
if not await request.is_disconnected():
failure = ErrorEvent(
code="recommendation_failed",
message="Recommendation could not be completed.",
)
yield f"{failure.model_dump_json()}\n"
failure = ErrorEvent(
code="recommendation_failed",
message="Recommendation could not be completed.",
)
yield f"{failure.model_dump_json()}\n"
finally:
await event_stream.aclose()

2
backend/app/config.py
View file

@ -41,6 +41,8 @@ class Settings(BaseSettings):
llm_model: str = "claude-sonnet-5"
intent_effort: str = "low"
rerank_effort: str = "medium"
# Bound provider calls independently from the grounding deadline.
llm_timeout_seconds: float = 120.0
# Ceilings include adaptive thinking tokens, which is why they sit far
# above the size of the structured output itself.
intent_max_tokens: int = 16384

21
backend/app/main.py
View file

@ -5,6 +5,7 @@ from contextlib import asynccontextmanager
from pathlib import Path
import httpx2
import structlog
from anthropic import AsyncAnthropic
from fastapi import FastAPI
from fastapi.staticfiles import StaticFiles
@ -12,6 +13,7 @@ from fastapi.staticfiles import StaticFiles
from app.adapters.anthropic.llm import AnthropicRecommender
from app.adapters.spotify.auth import TokenSet, refresh_access_token
from app.adapters.spotify.client import SpotifyClient
from app.adapters.spotify.errors import SpotifyError
from app.adapters.spotify.session import PendingLogins, SessionStore, SpotifySession
from app.api.recommendations import router as recommendations_router
from app.api.routes import router
@ -44,7 +46,8 @@ def create_app(
application.state.settings = active_settings
application.state.seed_session_id = None
anthropic_client = AsyncAnthropic(
api_key=active_settings.anthropic_api_key or "unused-demo-key"
api_key=active_settings.anthropic_api_key or "unused-demo-key",
timeout=active_settings.llm_timeout_seconds,
)
application.state.anthropic = anthropic_client
application.state.recommendation_pipeline = RecommendationPipeline(
@ -60,11 +63,17 @@ def create_app(
active_settings.app_mode is AppMode.LIVE
and active_settings.spotify_seed_refresh_token
):
application.state.seed_session_id = await _install_seed_session(
http,
application.state.session_store,
active_settings,
)
try:
application.state.seed_session_id = await _install_seed_session(
http,
application.state.session_store,
active_settings,
)
except (SpotifyError, ValueError) as error:
structlog.get_logger().warning(
"seed_session_install_failed",
error_type=type(error).__name__,
)
try:
yield
finally:

33
backend/app/pipeline/grounding.py
View file

@ -7,8 +7,10 @@ from collections.abc import Callable
from dataclasses import dataclass
from enum import StrEnum
import httpx2
import structlog
from app.adapters.spotify.errors import SpotifyError
from app.config import Settings
from app.domain.matching import candidate_key, judge_candidate_match, track_key
from app.domain.models import Familiarity, Track, TrackCandidate
@ -22,6 +24,7 @@ class ResolutionStatus(StrEnum):
RESOLVED = "resolved"
MISS = "miss"
MISMATCH = "mismatch"
FAILED = "failed"
QUOTA = "quota"
@ -32,6 +35,7 @@ class GroundingMetrics:
attempted_count: int
miss_count: int
mismatch_guard_count: int
failed_count: int
cache_hit_count: int
did_reach_deadline: bool
did_exhaust_quota: bool
@ -111,6 +115,7 @@ class Grounder:
settings.resolution_cache_ttl_seconds,
settings.resolution_cache_max_entries,
)
self._semaphore = asyncio.Semaphore(settings.grounding_concurrency)
async def ground(
self,
@ -119,6 +124,7 @@ class Grounder:
known_track_ids: frozenset[str],
familiarity: Familiarity,
pool_target: int,
deadline_at: float,
) -> GroundingResult:
"""Resolve candidates until the target, deadline, or quota boundary."""
accepted: dict[int, Track] = {}
@ -127,18 +133,18 @@ class Grounder:
metrics = _MutableMetrics()
pending: dict[asyncio.Task[_ResolutionAttempt], int] = {}
next_index = 0
deadline_at = time.monotonic() + self.settings.request_deadline_seconds
semaphore = asyncio.Semaphore(self.settings.grounding_concurrency)
try:
while next_index < len(candidates) or pending:
if len(accepted) >= pool_target:
break
if time.monotonic() >= deadline_at:
metrics.did_reach_deadline = True
break
next_index = self._launch_tasks(
catalog,
candidates,
pending,
next_index,
semaphore,
)
if not pending:
break
@ -183,12 +189,9 @@ class Grounder:
candidates: tuple[TrackCandidate, ...],
pending: dict[asyncio.Task[_ResolutionAttempt], int],
next_index: int,
semaphore: asyncio.Semaphore,
) -> int:
while next_index < len(candidates) and len(pending) < self.settings.grounding_concurrency:
task = asyncio.create_task(
self._resolve(catalog, next_index, candidates[next_index], semaphore)
)
task = asyncio.create_task(self._resolve(catalog, next_index, candidates[next_index]))
pending[task] = next_index
next_index += 1
return next_index
@ -228,9 +231,8 @@ class Grounder:
catalog: MusicCatalog,
index: int,
candidate: TrackCandidate,
semaphore: asyncio.Semaphore,
) -> _ResolutionAttempt:
async with semaphore:
async with self._semaphore:
return await self._resolve_with_slot(catalog, index, candidate)
async def _resolve_with_slot(
@ -265,6 +267,15 @@ class Grounder:
bare_results = []
except CatalogQuotaExhaustedError:
return _ResolutionAttempt(index=index, status=ResolutionStatus.QUOTA)
except (SpotifyError, httpx2.HTTPError) as error:
structlog.get_logger().info(
"candidate_unresolved",
title=candidate.title,
artist=candidate.artist,
status=ResolutionStatus.FAILED,
error_type=type(error).__name__,
)
return _ResolutionAttempt(index=index, status=ResolutionStatus.FAILED)
if matched_track is not None:
self.cache.put(key, matched_track)
@ -289,6 +300,7 @@ class _MutableMetrics:
attempted_count: int = 0
miss_count: int = 0
mismatch_guard_count: int = 0
failed_count: int = 0
cache_hit_count: int = 0
did_reach_deadline: bool = False
did_exhaust_quota: bool = False
@ -297,6 +309,7 @@ class _MutableMetrics:
self.attempted_count += 1
self.miss_count += attempt.status is ResolutionStatus.MISS
self.mismatch_guard_count += attempt.status is ResolutionStatus.MISMATCH
self.failed_count += attempt.status is ResolutionStatus.FAILED
self.cache_hit_count += attempt.is_cache_hit
self.did_exhaust_quota = self.did_exhaust_quota or attempt.status is ResolutionStatus.QUOTA
@ -305,6 +318,7 @@ class _MutableMetrics:
attempted_count=self.attempted_count,
miss_count=self.miss_count,
mismatch_guard_count=self.mismatch_guard_count,
failed_count=self.failed_count,
cache_hit_count=self.cache_hit_count,
did_reach_deadline=self.did_reach_deadline,
did_exhaust_quota=self.did_exhaust_quota,
@ -340,6 +354,7 @@ def _log_grounding(result: GroundingResult) -> None:
attempted_count=metrics.attempted_count,
miss_rate=metrics.miss_rate,
mismatch_guard_rate=metrics.mismatch_guard_rate,
failed_count=metrics.failed_count,
cache_hits=metrics.cache_hit_count,
deadline_reached=metrics.did_reach_deadline,
quota_exhausted=metrics.did_exhaust_quota,

86
backend/app/pipeline/orchestrator.py
View file

@ -7,6 +7,7 @@ from contextlib import aclosing
import structlog
from app.adapters.spotify.errors import SpotifyError
from app.config import Settings
from app.domain.models import (
CompressedTasteProfile,
@ -27,7 +28,12 @@ from app.pipeline.event import (
PipelineWarningEvent,
)
from app.pipeline.grounding import Grounder
from app.ports.protocols import MusicCatalog, Recommender, RecommenderOutputError
from app.ports.protocols import (
CatalogQuotaExhaustedError,
MusicCatalog,
Recommender,
RecommenderOutputError,
)
RERANK_FALLBACK_CODE = "rerank_fallback"
RERANK_FALLBACK_MESSAGE = "Ranking output was invalid, so grounded results are shown instead."
@ -115,21 +121,43 @@ class RecommendationPipeline:
) -> AsyncGenerator[PipelineEvent]:
"""Yield ordered events for one recommendation request."""
started_at = time.monotonic()
taste = await self.taste_cache.get(session_id, catalog)
intent = await self.recommender.create_intent(
query,
history,
previous_recommendations,
taste.text,
self.settings.candidate_count,
)
deadline_at = started_at + self.settings.request_deadline_seconds
try:
taste = await self.taste_cache.get(session_id, catalog)
intent = await self.recommender.create_intent(
query,
history,
previous_recommendations,
taste.text,
self.settings.candidate_count,
)
except RecommenderOutputError:
yield PipelineErrorEvent(
code="intent_failed",
message="Recommendation intent could not be generated from the model response.",
)
return
except CatalogQuotaExhaustedError:
yield PipelineErrorEvent(
code="quota_exhausted",
message=(
"Spotify request quota was exhausted before recommendations could be prepared."
),
)
return
except SpotifyError:
yield PipelineErrorEvent(
code="spotify_unavailable",
message="Spotify was unavailable while preparing recommendations.",
)
return
yield PipelineMetadataEvent(
request_id=request_id,
intent_summary=intent.intent_summary,
candidate_count=len(intent.candidates),
)
pool = await self._grounded_pool(session_id, catalog, intent, taste)
pool = await self._grounded_pool(session_id, catalog, intent, taste, deadline_at)
if not pool:
yield PipelineErrorEvent(
code="no_grounded_results",
@ -160,6 +188,7 @@ class RecommendationPipeline:
catalog: MusicCatalog,
intent: Intent,
taste: CompressedTasteProfile,
deadline_at: float,
) -> tuple[Track, ...]:
"""Reuse the session's pool on refinement, otherwise ground anew."""
if intent.is_refinement:
@ -172,6 +201,7 @@ class RecommendationPipeline:
taste.known_track_ids,
intent.familiarity,
self.settings.rerank_count + self.settings.rerank_pool_buffer,
deadline_at,
)
if result.tracks:
self.last_pools[session_id] = result.tracks
@ -290,19 +320,31 @@ class _TasteProfileCache:
return profile
async def _fetch(self, catalog: MusicCatalog) -> CompressedTasteProfile:
short_artists, long_artists, short_tracks, long_tracks, saved_tracks = await asyncio.gather(
catalog.fetch_top_artists("short_term", self.settings.top_items_limit),
catalog.fetch_top_artists("long_term", self.settings.top_items_limit),
catalog.fetch_top_tracks("short_term", self.settings.top_items_limit),
catalog.fetch_top_tracks("long_term", self.settings.top_items_limit),
catalog.fetch_saved_tracks(self.settings.saved_tracks_limit),
)
try:
async with asyncio.TaskGroup() as task_group:
short_artists_task = task_group.create_task(
catalog.fetch_top_artists("short_term", self.settings.top_items_limit)
)
long_artists_task = task_group.create_task(
catalog.fetch_top_artists("long_term", self.settings.top_items_limit)
)
short_tracks_task = task_group.create_task(
catalog.fetch_top_tracks("short_term", self.settings.top_items_limit)
)
long_tracks_task = task_group.create_task(
catalog.fetch_top_tracks("long_term", self.settings.top_items_limit)
)
saved_tracks_task = task_group.create_task(
catalog.fetch_saved_tracks(self.settings.saved_tracks_limit)
)
except ExceptionGroup as errors:
raise errors.exceptions[0] from None
return compress_taste_profile(
TasteProfile(
short_term_artists=tuple(short_artists),
long_term_artists=tuple(long_artists),
short_term_tracks=tuple(short_tracks),
long_term_tracks=tuple(long_tracks),
saved_tracks=tuple(saved_tracks),
short_term_artists=tuple(short_artists_task.result()),
long_term_artists=tuple(long_artists_task.result()),
short_term_tracks=tuple(short_tracks_task.result()),
long_term_tracks=tuple(long_tracks_task.result()),
saved_tracks=tuple(saved_tracks_task.result()),
)
)