justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

refactor: extract helpers from long functions in ingestion core

Browse source 
- Extract _load_unenriched_publications, _enrich_batch, _flush_and_sweep_duplicates from enrich_pending_publications
- Extract _sanitize_titles shared helper for title cleaning
- Extract _run_iteration_and_complete and _inline_enrich_and_finalize from run_for_user
- Extract _run_first_pass and _run_depth_pass from run_scholar_iteration
- Extract _finalize_successful_queue_job and _finalize_failed_queue_job from _finalize_queue_job_after_run

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-02-27 16:18:21 +01:00
parent 3fd9132176
commit 2a6e573a52
4 changed files with 345 additions and 235 deletions
Download patch file Download diff file Expand all files Collapse all files

59
app/services/ingestion/application.py
View file

@ -576,6 +576,46 @@ class ScholarIngestionService:
alert_network_failure_threshold=alert_network_failure_threshold,
alert_retry_scheduled_threshold=alert_retry_scheduled_threshold,
)
progress, failure_summary, alert_summary = await self._run_iteration_and_complete(
db_session,
run=run,
scholars=scholars,
user_id=user_id,
start_cstart_map=start_cstart_map,
paging=paging,
thresholds=thresholds,
auto_queue_continuations=auto_queue_continuations,
queue_delay_seconds=queue_delay_seconds,
idempotency_key=idempotency_key,
)
user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=user_id)
await self._inline_enrich_and_finalize(
db_session, run=run, user_settings=user_settings, intended_final_status=run.status
)
_log_run_completed(
run=run,
user_id=user_id,
scholars=scholars,
progress=progress,
failure_summary=failure_summary,
alert_summary=alert_summary,
)
return run_execution_summary(run=run, scholars=scholars, progress=progress)
async def _run_iteration_and_complete(
self,
db_session: AsyncSession,
*,
run: CrawlRun,
scholars: list[ScholarProfile],
user_id: int,
start_cstart_map: dict[int, int],
paging: dict[str, Any],
thresholds: dict[str, Any],
auto_queue_continuations: bool,
queue_delay_seconds: int,
idempotency_key: str | None,
) -> tuple[RunProgress, RunFailureSummary, RunAlertSummary]:
progress = await run_scholar_iteration(
db_session,
pagination=self._pagination,
@ -601,6 +641,16 @@ class ScholarIngestionService:
if intended_final_status not in (RunStatus.CANCELED,):
run.status = RunStatus.RESOLVING
await db_session.commit()
return progress, failure_summary, alert_summary
async def _inline_enrich_and_finalize(
self,
db_session: AsyncSession,
*,
run: CrawlRun,
user_settings: Any,
intended_final_status: RunStatus,
) -> None:
try:
await self._enrichment.enrich_pending_publications(
db_session,
@ -612,15 +662,6 @@ class ScholarIngestionService:
if run.status == RunStatus.RESOLVING:
run.status = intended_final_status
await db_session.commit()
_log_run_completed(
run=run,
user_id=user_id,
scholars=scholars,
progress=progress,
failure_summary=failure_summary,
alert_summary=alert_summary,
)
return run_execution_summary(run=run, scholars=scholars, progress=progress)
async def _try_acquire_user_lock(
self,