From d9be57a6c1536e507c9dee5636cb7a4c61c09932 Mon Sep 17 00:00:00 2001 From: Justin Visser Date: Sun, 1 Mar 2026 23:42:15 +0100 Subject: [PATCH] fix: URL-length-aware OpenAlex batching and eager scholar metadata hydration - Replace fixed batch_size=25 with URL-byte-length-aware chunking for OpenAlex title.search queries. Long/Unicode titles caused URLs to exceed server limits, resulting in 500 errors. - Always hydrate scholar profile metadata (name, image) on creation, even when an initial scrape job is queued, so the UI shows profile info immediately. Co-Authored-By: Claude Opus 4.6 --- app/api/routers/scholars.py | 15 ++--- app/services/ingestion/enrichment.py | 97 ++++++++++++++++++++++++---- 2 files changed, 91 insertions(+), 21 deletions(-) diff --git a/app/api/routers/scholars.py b/app/api/routers/scholars.py index 7095471..7bbc642 100644 --- a/app/api/routers/scholars.py +++ b/app/api/routers/scholars.py @@ -144,18 +144,17 @@ async def create_scholar( message=str(exc), ) from exc structured_log(logger, "info", "api.scholars.created", user_id=current_user.id, scholar_profile_id=created.id) - did_queue_initial_scrape = await enqueue_initial_scrape_job_for_scholar( + await enqueue_initial_scrape_job_for_scholar( db_session, profile=created, user_id=current_user.id, ) - if not did_queue_initial_scrape: - created = await hydrate_scholar_metadata_if_needed( - db_session, - profile=created, - source=source, - user_id=current_user.id, - ) + created = await hydrate_scholar_metadata_if_needed( + db_session, + profile=created, + source=source, + user_id=current_user.id, + ) return success_payload( request, diff --git a/app/services/ingestion/enrichment.py b/app/services/ingestion/enrichment.py index 1b66f58..181ff41 100644 --- a/app/services/ingestion/enrichment.py +++ b/app/services/ingestion/enrichment.py @@ -37,6 +37,40 @@ def _sanitize_titles(publications: list) -> list[str]: return titles +# OpenAlex (and most HTTP servers) struggle with very long query strings. +# Non-ASCII characters expand significantly when percent-encoded in URLs, +# so a fixed batch count is unreliable. Instead we cap the combined +# byte-length of the pipe-joined title filter value to stay well within +# safe URL limits (~4 KiB of filter text ≈ ~6 KiB when percent-encoded). +_MAX_TITLE_FILTER_BYTES = 4_000 + + +def _chunk_titles_by_url_length( + titles: list[str], + max_bytes: int = _MAX_TITLE_FILTER_BYTES, +) -> list[list[str]]: + """Split *titles* into chunks whose pipe-joined UTF-8 size stays under *max_bytes*.""" + chunks: list[list[str]] = [] + current_chunk: list[str] = [] + current_bytes = 0 + + for title in titles: + title_bytes = len(title.encode("utf-8")) + # Account for the pipe separator between titles + separator = 3 if current_chunk else 0 # len("|".encode) but url-encoded as %7C = 3 + if current_chunk and current_bytes + separator + title_bytes > max_bytes: + chunks.append(current_chunk) + current_chunk = [title] + current_bytes = title_bytes + else: + current_chunk.append(title) + current_bytes += separator + title_bytes + + if current_chunk: + chunks.append(current_chunk) + return chunks + + class EnrichmentRunner: """Post-run OpenAlex enrichment logic. @@ -156,21 +190,34 @@ class EnrichmentRunner: resolved_key = openalex_api_key or settings.openalex_api_key client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto) - batch_size = 25 now = datetime.now(UTC) arxiv_lookup_allowed = True - for i in range(0, len(publications), batch_size): + all_titles = _sanitize_titles(publications) + title_chunks = _chunk_titles_by_url_length(all_titles) + + # Build a mapping from sanitized title → publications for batch association + title_to_pubs: dict[str, list[Publication]] = {} + for p in publications: + raw = getattr(p, "title_raw", None) or getattr(p, "title", None) + if not raw or not raw.strip(): + continue + safe = " ".join(re.sub(r"[^\w\s]", " ", raw).split()) + if safe: + title_to_pubs.setdefault(safe, []).append(p) + + for title_chunk in title_chunks: if await self.run_is_canceled(db_session, run_id=run_id): structured_log(logger, "info", "ingestion.enrichment_aborted", run_id=run_id) return - batch = publications[i : i + batch_size] - titles = _sanitize_titles(batch) - if not titles: + batch = [] + for t in title_chunk: + batch.extend(title_to_pubs.get(t, [])) + if not batch: continue try: openalex_works = await client.get_works_by_filter( - {"title.search": "|".join(titles)}, limit=batch_size * 3 + {"title.search": "|".join(title_chunk)}, limit=len(title_chunk) * 3 ) except OpenAlexBudgetExhaustedError: structured_log(logger, "warning", "ingestion.openalex_budget_exhausted", run_id=run_id) @@ -289,19 +336,37 @@ class EnrichmentRunner: client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto) - batch_size = 25 + all_titles = _sanitize_titles(publications) + title_chunks = _chunk_titles_by_url_length(all_titles) + + # Build title → publication mapping for batch association + title_to_pubs: dict[str, list[PublicationCandidate]] = {} + for p in publications: + raw = getattr(p, "title", None) + if not raw or not raw.strip(): + continue + safe = " ".join(re.sub(r"[^\w\s]", " ", raw).split()) + if safe: + title_to_pubs.setdefault(safe, []).append(p) + + # Collect publications that had no sanitizable title — pass through as-is + pubs_with_titles = set() + for titles in title_to_pubs.values(): + for p in titles: + pubs_with_titles.add(id(p)) + enriched: list[PublicationCandidate] = [] - for i in range(0, len(publications), batch_size): - batch = publications[i : i + batch_size] - titles = _sanitize_titles(batch) - if not titles: - enriched.extend(batch) + for title_chunk in title_chunks: + batch = [] + for t in title_chunk: + batch.extend(title_to_pubs.get(t, [])) + if not batch: continue try: openalex_works = await client.get_works_by_filter( - {"title.search": "|".join(titles)}, limit=batch_size * 3 + {"title.search": "|".join(title_chunk)}, limit=len(title_chunk) * 3 ) except Exception as e: structured_log( @@ -335,4 +400,10 @@ class EnrichmentRunner: enriched.append(new_p) else: enriched.append(p) + + # Append publications that had no sanitizable title (pass-through) + for p in publications: + if id(p) not in pubs_with_titles: + enriched.append(p) + return enriched -- 2.49.1