refactor: extract helpers from long functions in ingestion core

- Extract _load_unenriched_publications, _enrich_batch, _flush_and_sweep_duplicates from enrich_pending_publications
- Extract _sanitize_titles shared helper for title cleaning
- Extract _run_iteration_and_complete and _inline_enrich_and_finalize from run_for_user
- Extract _run_first_pass and _run_depth_pass from run_scholar_iteration
- Extract _finalize_successful_queue_job and _finalize_failed_queue_job from _finalize_queue_job_after_run

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app/services/ingestion/application.py

@@ -576,6 +576,46 @@ class ScholarIngestionService:
             alert_network_failure_threshold=alert_network_failure_threshold,
             alert_retry_scheduled_threshold=alert_retry_scheduled_threshold,
         )
+        progress, failure_summary, alert_summary = await self._run_iteration_and_complete(
+            db_session,
+            run=run,
+            scholars=scholars,
+            user_id=user_id,
+            start_cstart_map=start_cstart_map,
+            paging=paging,
+            thresholds=thresholds,
+            auto_queue_continuations=auto_queue_continuations,
+            queue_delay_seconds=queue_delay_seconds,
+            idempotency_key=idempotency_key,
+        )
+        user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=user_id)
+        await self._inline_enrich_and_finalize(
+            db_session, run=run, user_settings=user_settings, intended_final_status=run.status
+        )
+        _log_run_completed(
+            run=run,
+            user_id=user_id,
+            scholars=scholars,
+            progress=progress,
+            failure_summary=failure_summary,
+            alert_summary=alert_summary,
+        )
+        return run_execution_summary(run=run, scholars=scholars, progress=progress)
+
+    async def _run_iteration_and_complete(
+        self,
+        db_session: AsyncSession,
+        *,
+        run: CrawlRun,
+        scholars: list[ScholarProfile],
+        user_id: int,
+        start_cstart_map: dict[int, int],
+        paging: dict[str, Any],
+        thresholds: dict[str, Any],
+        auto_queue_continuations: bool,
+        queue_delay_seconds: int,
+        idempotency_key: str | None,
+    ) -> tuple[RunProgress, RunFailureSummary, RunAlertSummary]:
         progress = await run_scholar_iteration(
             db_session,
             pagination=self._pagination,
@@ -601,6 +641,16 @@ class ScholarIngestionService:
         if intended_final_status not in (RunStatus.CANCELED,):
             run.status = RunStatus.RESOLVING
         await db_session.commit()
+        return progress, failure_summary, alert_summary
+
+    async def _inline_enrich_and_finalize(
+        self,
+        db_session: AsyncSession,
+        *,
+        run: CrawlRun,
+        user_settings: Any,
+        intended_final_status: RunStatus,
+    ) -> None:
         try:
             await self._enrichment.enrich_pending_publications(
                 db_session,
@@ -612,15 +662,6 @@ class ScholarIngestionService:
         if run.status == RunStatus.RESOLVING:
             run.status = intended_final_status
         await db_session.commit()
-        _log_run_completed(
-            run=run,
-            user_id=user_id,
-            scholars=scholars,
-            progress=progress,
-            failure_summary=failure_summary,
-            alert_summary=alert_summary,
-        )
-        return run_execution_summary(run=run, scholars=scholars, progress=progress)
 
     async def _try_acquire_user_lock(
         self,

app/services/ingestion/enrichment.py

@@ -25,6 +25,18 @@ from app.settings import settings
 logger = logging.getLogger(__name__)
 
 
+def _sanitize_titles(publications: list) -> list[str]:
+    titles = []
+    for p in publications:
+        raw = getattr(p, "title_raw", None) or getattr(p, "title", None)
+        if not raw or not raw.strip():
+            continue
+        safe = " ".join(re.sub(r"[^\w\s]", " ", raw).split())
+        if safe:
+            titles.append(safe)
+    return titles
+
+
 class EnrichmentRunner:
     """Post-run OpenAlex enrichment logic.
 
@@ -44,31 +56,15 @@ class EnrichmentRunner:
             raise RuntimeError(f"Missing crawl_run for run_id={run_id}.")
         return status == RunStatus.CANCELED
 
-    async def enrich_pending_publications(
+    async def _load_unenriched_publications(
         self,
         db_session: AsyncSession,
         *,
         run_id: int,
-        openalex_api_key: str | None = None,
+    ) -> tuple[int, list[Publication]]:
-    ) -> None:
-        """Enrich unenriched publications with OpenAlex data.
-
-        Stops immediately on budget exhaustion (429 with $0 remaining).
-        Sleeps 60s and continues on transient rate limits.
-        """
-        from app.services.openalex.client import (
-            OpenAlexBudgetExhaustedError,
-            OpenAlexClient,
-            OpenAlexRateLimitError,
-        )
-        from app.services.openalex.matching import find_best_match
-
         run_result = await db_session.execute(select(CrawlRun.user_id).where(CrawlRun.id == run_id))
         user_id = run_result.scalar_one()
-
+        cooldown_threshold = datetime.now(UTC) - timedelta(days=7)
-        now = datetime.now(UTC)
-        cooldown_threshold = now - timedelta(days=7)
-
         stmt = (
             select(Publication)
             .join(ScholarPublication)
@@ -84,88 +80,51 @@ class EnrichmentRunner:
             .distinct()
         )
         result = await db_session.execute(stmt)
-        publications = list(result.scalars().all())
+        return user_id, list(result.scalars().all())
 
-        if not publications:
+    async def _enrich_batch(
-            return
+        self,
+        db_session: AsyncSession,
+        *,
+        batch: list[Publication],
+        run_id: int,
+        openalex_works: list,
+        now: datetime,
+        arxiv_lookup_allowed: bool,
+    ) -> tuple[bool, bool]:
+        from app.services.openalex.matching import find_best_match
 
-        resolved_key = openalex_api_key or settings.openalex_api_key
+        for p in batch:
-        client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto)
-        batch_size = 25
-        arxiv_lookup_allowed = True
-
-        for i in range(0, len(publications), batch_size):
             if await self.run_is_canceled(db_session, run_id=run_id):
                 logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id})
-                return
+                return False, arxiv_lookup_allowed
-            batch = publications[i : i + batch_size]
+            p.openalex_last_attempt_at = now
-            titles = [
+            arxiv_lookup_allowed = await self._discover_identifiers_for_enrichment(
-                " ".join(re.sub(r"[^\w\s]", " ", p.title_raw).split())
+                db_session,
-                for p in batch
+                publication=p,
-                if p.title_raw and p.title_raw.strip()
+                run_id=run_id,
-            ]
+                allow_arxiv_lookup=arxiv_lookup_allowed,
-
+            )
-            if not titles:
+            match = find_best_match(
-                continue
+                target_title=p.title_raw,
-
+                target_year=p.year,
-            try:
+                target_authors=p.author_text or "",
-                openalex_works = await client.get_works_by_filter(
+                candidates=openalex_works,
-                    {"title.search": "|".join(titles)}, limit=batch_size * 3
+            )
-                )
+            if match:
-            except OpenAlexBudgetExhaustedError:
+                p.year = match.publication_year or p.year
-                structured_log(
+                p.citation_count = match.cited_by_count or p.citation_count
-                    logger,
+                p.pdf_url = match.oa_url or p.pdf_url
-                    "warning",
+                p.openalex_enriched = True
-                    "ingestion.openalex_budget_exhausted",
+        return True, arxiv_lookup_allowed
-                    run_id=run_id,
-                )
-                break
-            except OpenAlexRateLimitError:
-                structured_log(
-                    logger,
-                    "warning",
-                    "ingestion.openalex_rate_limited",
-                    run_id=run_id,
-                )
-                await asyncio.sleep(60)
-                continue
-            except Exception as e:
-                structured_log(
-                    logger,
-                    "warning",
-                    "ingestion.openalex_enrichment_failed",
-                    error=str(e),
-                    run_id=run_id,
-                )
-                continue
-
-            for p in batch:
-                if await self.run_is_canceled(db_session, run_id=run_id):
-                    logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id})
-                    return
-
-                p.openalex_last_attempt_at = now
-                arxiv_lookup_allowed = await self._discover_identifiers_for_enrichment(
-                    db_session,
-                    publication=p,
-                    run_id=run_id,
-                    allow_arxiv_lookup=arxiv_lookup_allowed,
-                )
-
-                match = find_best_match(
-                    target_title=p.title_raw,
-                    target_year=p.year,
-                    target_authors=p.author_text or "",
-                    candidates=openalex_works,
-                )
-                if match:
-                    p.year = match.publication_year or p.year
-                    p.citation_count = match.cited_by_count or p.citation_count
-                    p.pdf_url = match.oa_url or p.pdf_url
-                    p.openalex_enriched = True
 
+    async def _flush_and_sweep_duplicates(
+        self,
+        db_session: AsyncSession,
+        *,
+        run_id: int,
+    ) -> None:
         await db_session.flush()
-
         from app.services.publications.dedup import sweep_identifier_duplicates
 
         merge_count = await sweep_identifier_duplicates(db_session)
@@ -178,6 +137,64 @@ class EnrichmentRunner:
                 run_id=run_id,
             )
 
+    async def enrich_pending_publications(
+        self,
+        db_session: AsyncSession,
+        *,
+        run_id: int,
+        openalex_api_key: str | None = None,
+    ) -> None:
+        from app.services.openalex.client import (
+            OpenAlexBudgetExhaustedError,
+            OpenAlexClient,
+            OpenAlexRateLimitError,
+        )
+
+        _, publications = await self._load_unenriched_publications(db_session, run_id=run_id)
+        if not publications:
+            return
+
+        resolved_key = openalex_api_key or settings.openalex_api_key
+        client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto)
+        batch_size = 25
+        now = datetime.now(UTC)
+        arxiv_lookup_allowed = True
+
+        for i in range(0, len(publications), batch_size):
+            if await self.run_is_canceled(db_session, run_id=run_id):
+                logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id})
+                return
+            batch = publications[i : i + batch_size]
+            titles = _sanitize_titles(batch)
+            if not titles:
+                continue
+            try:
+                openalex_works = await client.get_works_by_filter(
+                    {"title.search": "|".join(titles)}, limit=batch_size * 3
+                )
+            except OpenAlexBudgetExhaustedError:
+                structured_log(logger, "warning", "ingestion.openalex_budget_exhausted", run_id=run_id)
+                break
+            except OpenAlexRateLimitError:
+                structured_log(logger, "warning", "ingestion.openalex_rate_limited", run_id=run_id)
+                await asyncio.sleep(60)
+                continue
+            except Exception as e:
+                structured_log(logger, "warning", "ingestion.openalex_enrichment_failed", error=str(e), run_id=run_id)
+                continue
+            should_continue, arxiv_lookup_allowed = await self._enrich_batch(
+                db_session,
+                batch=batch,
+                run_id=run_id,
+                openalex_works=openalex_works,
+                now=now,
+                arxiv_lookup_allowed=arxiv_lookup_allowed,
+            )
+            if not should_continue:
+                return
+
+        await self._flush_and_sweep_duplicates(db_session, run_id=run_id)
+
     async def _discover_identifiers_for_enrichment(
         self,
         db_session: AsyncSession,
@@ -275,23 +292,15 @@ class EnrichmentRunner:
 
         for i in range(0, len(publications), batch_size):
             batch = publications[i : i + batch_size]
-
+            titles = _sanitize_titles(batch)
-            titles = []
-            for p in batch:
-                if not p.title:
-                    continue
-                safe_title = re.sub(r"[^\w\s]", " ", p.title)
-                safe_title = " ".join(safe_title.split())
-                if safe_title:
-                    titles.append(safe_title)
-
             if not titles:
                 enriched.extend(batch)
                 continue
 
-            query = "|".join(t for t in titles)
             try:
-                openalex_works = await client.get_works_by_filter({"title.search": query}, limit=batch_size * 3)
+                openalex_works = await client.get_works_by_filter(
+                    {"title.search": "|".join(titles)}, limit=batch_size * 3
+                )
             except Exception as e:
                 logger.warning(
                     "ingestion.openalex_enrichment_failed",

app/services/ingestion/queue_runner.py

@@ -281,74 +281,72 @@ class QueueJobRunner:
                 await self._reschedule_queue_job_after_exception(job, exc=exc)
         return None
 
-    async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
+    async def _finalize_successful_queue_job(self, session, job, run_summary) -> None:
-        session_factory = get_session_factory()
+        queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
-        async with session_factory() as session:
+        await session.commit()
-            if int(run_summary.failed_count) <= 0:
+        if queue_item is None:
-                queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
+            structured_log(
-                await session.commit()
+                logger,
-                if queue_item is None:
+                "info",
-                    structured_log(
+                "scheduler.queue_item_resolved",
-                        logger,
+                queue_item_id=job.id,
-                        "info",
+                user_id=job.user_id,
-                        "scheduler.queue_item_resolved",
+                run_id=run_summary.crawl_run_id,
-                        queue_item_id=job.id,
+                status=run_summary.status.value,
-                        user_id=job.user_id,
-                        run_id=run_summary.crawl_run_id,
-                        status=run_summary.status.value,
-                    )
-                    return
-                structured_log(
-                    logger,
-                    "info",
-                    "scheduler.queue_item_progressed",
-                    queue_item_id=job.id,
-                    user_id=job.user_id,
-                    run_id=run_summary.crawl_run_id,
-                    status=run_summary.status.value,
-                    attempt_count=int(queue_item.attempt_count),
-                )
-                return
-            queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
-            if queue_item is None:
-                await session.commit()
-                structured_log(
-                    logger,
-                    "info",
-                    "scheduler.queue_item_resolved",
-                    queue_item_id=job.id,
-                    user_id=job.user_id,
-                    run_id=run_summary.crawl_run_id,
-                    status=run_summary.status.value,
-                )
-                return
-            if int(queue_item.attempt_count) >= self._continuation_max_attempts:
-                await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
-                await session.commit()
-                structured_log(
-                    logger,
-                    "warning",
-                    "scheduler.queue_item_dropped_max_attempts_after_run",
-                    queue_item_id=job.id,
-                    user_id=job.user_id,
-                    attempt_count=queue_item.attempt_count,
-                    run_id=run_summary.crawl_run_id,
-                    status=run_summary.status.value,
-                )
-                return
-            delay_seconds = queue_service.compute_backoff_seconds(
-                base_seconds=self._continuation_base_delay_seconds,
-                attempt_count=int(queue_item.attempt_count),
-                max_seconds=self._continuation_max_delay_seconds,
-            )
-            await queue_service.reschedule_job(
-                session,
-                job_id=job.id,
-                delay_seconds=delay_seconds,
-                reason=queue_item.reason,
-                error=queue_item.last_error,
             )
+            return
+        structured_log(
+            logger,
+            "info",
+            "scheduler.queue_item_progressed",
+            queue_item_id=job.id,
+            user_id=job.user_id,
+            run_id=run_summary.crawl_run_id,
+            status=run_summary.status.value,
+            attempt_count=int(queue_item.attempt_count),
+        )
+
+    async def _finalize_failed_queue_job(self, session, job, run_summary) -> None:
+        queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
+        if queue_item is None:
             await session.commit()
+            structured_log(
+                logger,
+                "info",
+                "scheduler.queue_item_resolved",
+                queue_item_id=job.id,
+                user_id=job.user_id,
+                run_id=run_summary.crawl_run_id,
+                status=run_summary.status.value,
+            )
+            return
+        if int(queue_item.attempt_count) >= self._continuation_max_attempts:
+            await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
+            await session.commit()
+            structured_log(
+                logger,
+                "warning",
+                "scheduler.queue_item_dropped_max_attempts_after_run",
+                queue_item_id=job.id,
+                user_id=job.user_id,
+                attempt_count=queue_item.attempt_count,
+                run_id=run_summary.crawl_run_id,
+                status=run_summary.status.value,
+            )
+            return
+        delay_seconds = queue_service.compute_backoff_seconds(
+            base_seconds=self._continuation_base_delay_seconds,
+            attempt_count=int(queue_item.attempt_count),
+            max_seconds=self._continuation_max_delay_seconds,
+        )
+        await queue_service.reschedule_job(
+            session,
+            job_id=job.id,
+            delay_seconds=delay_seconds,
+            reason=queue_item.reason,
+            error=queue_item.last_error,
+        )
+        await session.commit()
         structured_log(
             logger,
             "info",
@@ -361,6 +359,14 @@ class QueueJobRunner:
             delay_seconds=delay_seconds,
         )
 
+    async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            if int(run_summary.failed_count) <= 0:
+                await self._finalize_successful_queue_job(session, job, run_summary)
+            else:
+                await self._finalize_failed_queue_job(session, job, run_summary)
+
     async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
         if await self._drop_queue_job_if_max_attempts(job):
             return

app/services/ingestion/scholar_processing.py

@@ -528,6 +528,87 @@ def unexpected_scholar_exception_outcome(
     )
 
 
+async def _run_first_pass(
+    db_session: AsyncSession,
+    *,
+    scholars: list[ScholarProfile],
+    pagination: PaginationEngine,
+    run: CrawlRun,
+    user_id: int,
+    start_cstart_map: dict[int, int],
+    scholar_kwargs: dict[str, Any],
+    request_delay_seconds: int,
+    queue_delay_seconds: int,
+    progress: RunProgress,
+) -> dict[int, int]:
+    first_pass_cstarts: dict[int, int] = {}
+    for index, scholar in enumerate(scholars):
+        await db_session.refresh(run)
+        if run.status == RunStatus.CANCELED:
+            structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id)
+            return first_pass_cstarts
+        if index > 0 and request_delay_seconds > 0:
+            await asyncio.sleep(float(request_delay_seconds))
+        start_cstart = int(start_cstart_map.get(int(scholar.id), 0))
+        outcome = await process_scholar(
+            db_session,
+            pagination=pagination,
+            run=run,
+            scholar=scholar,
+            user_id=user_id,
+            start_cstart=start_cstart,
+            max_pages_per_scholar=1,
+            auto_queue_continuations=False,
+            queue_delay_seconds=queue_delay_seconds,
+            **scholar_kwargs,
+        )
+        apply_outcome_to_progress(progress=progress, outcome=outcome)
+        resume_cstart = outcome.result_entry.get("continuation_cstart")
+        if resume_cstart is not None and int(resume_cstart) > start_cstart:
+            first_pass_cstarts[int(scholar.id)] = int(resume_cstart)
+    return first_pass_cstarts
+
+
+async def _run_depth_pass(
+    db_session: AsyncSession,
+    *,
+    scholars: list[ScholarProfile],
+    first_pass_cstarts: dict[int, int],
+    pagination: PaginationEngine,
+    run: CrawlRun,
+    user_id: int,
+    scholar_kwargs: dict[str, Any],
+    request_delay_seconds: int,
+    remaining_max: int,
+    auto_queue_continuations: bool,
+    queue_delay_seconds: int,
+    progress: RunProgress,
+) -> None:
+    for index, scholar in enumerate(scholars):
+        resume_cstart = first_pass_cstarts.get(int(scholar.id))
+        if resume_cstart is None:
+            continue
+        await db_session.refresh(run)
+        if run.status == RunStatus.CANCELED:
+            structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id)
+            break
+        if index > 0 and request_delay_seconds > 0:
+            await asyncio.sleep(float(request_delay_seconds))
+        outcome = await process_scholar(
+            db_session,
+            pagination=pagination,
+            run=run,
+            scholar=scholar,
+            user_id=user_id,
+            start_cstart=resume_cstart,
+            max_pages_per_scholar=remaining_max,
+            auto_queue_continuations=auto_queue_continuations,
+            queue_delay_seconds=queue_delay_seconds,
+            **scholar_kwargs,
+        )
+        apply_outcome_to_progress(progress=progress, outcome=outcome)
+
+
 async def run_scholar_iteration(
     db_session: AsyncSession,
     *,
@@ -555,60 +636,33 @@ async def run_scholar_iteration(
         "rate_limit_backoff_seconds": rate_limit_backoff_seconds,
         "page_size": page_size,
     }
-
+    first_pass_cstarts = await _run_first_pass(
-    # ── Pass 1: first page of every scholar (breadth-first) ──────────
+        db_session,
-    first_pass_cstarts: dict[int, int] = {}
+        scholars=scholars,
-    for index, scholar in enumerate(scholars):
+        pagination=pagination,
-        await db_session.refresh(run)
+        run=run,
-        if run.status == RunStatus.CANCELED:
+        user_id=user_id,
-            structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id)
+        start_cstart_map=start_cstart_map,
-            return progress
+        scholar_kwargs=scholar_kwargs,
-        if index > 0 and request_delay_seconds > 0:
+        request_delay_seconds=request_delay_seconds,
-            await asyncio.sleep(float(request_delay_seconds))
+        queue_delay_seconds=queue_delay_seconds,
-        start_cstart = int(start_cstart_map.get(int(scholar.id), 0))
+        progress=progress,
-        outcome = await process_scholar(
+    )
-            db_session,
-            pagination=pagination,
-            run=run,
-            scholar=scholar,
-            user_id=user_id,
-            start_cstart=start_cstart,
-            max_pages_per_scholar=1,
-            auto_queue_continuations=False,
-            queue_delay_seconds=queue_delay_seconds,
-            **scholar_kwargs,
-        )
-        apply_outcome_to_progress(progress=progress, outcome=outcome)
-        resume_cstart = outcome.result_entry.get("continuation_cstart")
-        if resume_cstart is not None and int(resume_cstart) > start_cstart:
-            first_pass_cstarts[int(scholar.id)] = int(resume_cstart)
-
-    # ── Pass 2: remaining pages for each scholar (depth) ─────────────
     remaining_max = max(max_pages_per_scholar - 1, 0)
     if remaining_max <= 0:
         return progress
-
+    await _run_depth_pass(
-    for index, scholar in enumerate(scholars):
+        db_session,
-        resume_cstart = first_pass_cstarts.get(int(scholar.id))
+        scholars=scholars,
-        if resume_cstart is None:
+        first_pass_cstarts=first_pass_cstarts,
-            continue
+        pagination=pagination,
-        await db_session.refresh(run)
+        run=run,
-        if run.status == RunStatus.CANCELED:
+        user_id=user_id,
-            structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id)
+        scholar_kwargs=scholar_kwargs,
-            break
+        request_delay_seconds=request_delay_seconds,
-        if index > 0 and request_delay_seconds > 0:
+        remaining_max=remaining_max,
-            await asyncio.sleep(float(request_delay_seconds))
+        auto_queue_continuations=auto_queue_continuations,
-        outcome = await process_scholar(
+        queue_delay_seconds=queue_delay_seconds,
-            db_session,
+        progress=progress,
-            pagination=pagination,
+    )
-            run=run,
-            scholar=scholar,
-            user_id=user_id,
-            start_cstart=resume_cstart,
-            max_pages_per_scholar=remaining_max,
-            auto_queue_continuations=auto_queue_continuations,
-            queue_delay_seconds=queue_delay_seconds,
-            **scholar_kwargs,
-        )
-        apply_outcome_to_progress(progress=progress, outcome=outcome)
     return progress