justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

refactor: extract helpers from remaining long functions

Browse source 
- Extract _start_manual_run and _spawn_background_execution from run_manual route
- Extract _upsert_page_publications to deduplicate pagination loop
- Extract _classify_attempt from fetch_and_parse_with_retry
- Extract _log_alert_threshold_warnings from complete_run_for_user
- Functions 3f-3j accepted as kwargs-exception (body ≤50 lines excluding signatures)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-02-27 16:21:49 +01:00
parent 2a6e573a52
commit 72bc046215
4 changed files with 156 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

107
app/api/routers/runs.py
View file

@ -176,6 +176,67 @@ async def cancel_run(
)
async def _start_manual_run(
db_session: AsyncSession,
*,
current_user: User,
ingest_service: ingestion_service.ScholarIngestionService,
idempotency_key: str | None,
) -> tuple[Any, Any, list, dict]:
user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=current_user.id)
run, scholars, start_cstart_map = await ingest_service.initialize_run(
db_session,
user_id=current_user.id,
trigger_type=RunTriggerType.MANUAL,
request_delay_seconds=user_settings.request_delay_seconds,
network_error_retries=settings.ingestion_network_error_retries,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
page_size=settings.ingestion_page_size,
idempotency_key=idempotency_key,
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
)
return user_settings, run, scholars, start_cstart_map
def _spawn_background_execution(
ingest_service: ingestion_service.ScholarIngestionService,
*,
run: Any,
current_user: User,
scholars: list,
start_cstart_map: dict,
user_settings: Any,
idempotency_key: str | None,
) -> None:
from app.db.session import get_session_factory
task = asyncio.create_task(
ingest_service.execute_run(
session_factory=get_session_factory(),
run_id=run.id,
user_id=current_user.id,
scholars=scholars,
start_cstart_map=start_cstart_map,
request_delay_seconds=user_settings.request_delay_seconds,
network_error_retries=settings.ingestion_network_error_retries,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
page_size=settings.ingestion_page_size,
auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
idempotency_key=idempotency_key,
)
)
_background_tasks.add(task)
task.add_done_callback(_background_tasks.discard)
@router.post(
"/manual",
response_model=ManualRunEnvelope,
@ -202,52 +263,22 @@ async def run_manual(
return reused_payload
try:
user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=current_user.id)
# Initialize run (creates the record and performs safety checks)
run, scholars, start_cstart_map = await ingest_service.initialize_run(
user_settings, run, scholars, start_cstart_map = await _start_manual_run(
db_session,
user_id=current_user.id,
trigger_type=RunTriggerType.MANUAL,
request_delay_seconds=user_settings.request_delay_seconds,
network_error_retries=settings.ingestion_network_error_retries,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
page_size=settings.ingestion_page_size,
current_user=current_user,
ingest_service=ingest_service,
idempotency_key=idempotency_key,
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
)
await db_session.commit()
# Kick off background execution
from app.db.session import get_session_factory
task = asyncio.create_task(
ingest_service.execute_run(
session_factory=get_session_factory(),
run_id=run.id,
user_id=current_user.id,
_spawn_background_execution(
ingest_service,
run=run,
current_user=current_user,
scholars=scholars,
start_cstart_map=start_cstart_map,
request_delay_seconds=user_settings.request_delay_seconds,
network_error_retries=settings.ingestion_network_error_retries,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
page_size=settings.ingestion_page_size,
auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
user_settings=user_settings,
idempotency_key=idempotency_key,
)
)
_background_tasks.add(task)
task.add_done_callback(_background_tasks.discard)
return success_payload(
request,
data={

36
app/services/ingestion/page_fetch.py
View file

@ -145,6 +145,24 @@ class PageFetcher:
if sleep_seconds > 0:
await asyncio.sleep(sleep_seconds)
@staticmethod
def _classify_attempt(
parsed_page: ParsedProfilePage,
*,
network_attempts: int,
rate_limit_attempts: int,
) -> tuple[int, int, int]:
if parsed_page.state == ParseState.NETWORK_ERROR:
network_attempts += 1
return network_attempts, rate_limit_attempts, network_attempts
if (
parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA
and parsed_page.state_reason == "blocked_http_429_rate_limited"
):
rate_limit_attempts += 1
return network_attempts, rate_limit_attempts, rate_limit_attempts
return network_attempts, rate_limit_attempts, network_attempts + rate_limit_attempts + 1
async def fetch_and_parse_with_retry(
self,
*,
@ -169,19 +187,9 @@ class PageFetcher:
page_size=page_size,
)
parsed_page = self.parse_page_or_layout_error(fetch_result=fetch_result)
if parsed_page.state == ParseState.NETWORK_ERROR:
network_attempts += 1
total_attempts = network_attempts
elif (
parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA
and parsed_page.state_reason == "blocked_http_429_rate_limited"
):
rate_limit_attempts += 1
total_attempts = rate_limit_attempts
else:
total_attempts = network_attempts + rate_limit_attempts + 1
network_attempts, rate_limit_attempts, total_attempts = self._classify_attempt(
parsed_page, network_attempts=network_attempts, rate_limit_attempts=rate_limit_attempts
)
attempt_log.append(
{
"attempt": total_attempts,
@ -192,7 +200,6 @@ class PageFetcher:
"fetch_error": fetch_result.error,
}
)
if not self._should_retry(
parsed_page=parsed_page,
network_attempt_count=network_attempts,
@ -201,7 +208,6 @@ class PageFetcher:
rate_limit_retries=rate_limit_retries,
):
break
await self._sleep_backoff(
scholar_id=scholar_id,
cstart=cstart,

48
app/services/ingestion/pagination.py
View file

@ -273,6 +273,24 @@ class PaginationEngine:
# ── Pagination loop ──────────────────────────────────────────────
@staticmethod
async def _upsert_page_publications(
db_session: Any,
*,
run: CrawlRun,
scholar: ScholarProfile,
publications: list,
seen_canonical: set[str],
state: PagedLoopState,
upsert_publications_fn: Any,
) -> None:
deduped = _dedupe_publication_candidates(list(publications), seen_canonical=seen_canonical)
if deduped:
discovered_count = await upsert_publications_fn(
db_session, run=run, scholar=scholar, publications=deduped
)
state.discovered_publication_count += discovered_count
async def _paginate_loop(
self,
*,
@ -292,14 +310,15 @@ class PaginationEngine:
seen_canonical: set[str] = set()
if state.parsed_page.publications:
deduped_first = _dedupe_publication_candidates(
list(state.parsed_page.publications), seen_canonical=seen_canonical
await self._upsert_page_publications(
db_session,
run=run,
scholar=scholar,
publications=state.parsed_page.publications,
seen_canonical=seen_canonical,
state=state,
upsert_publications_fn=upsert_publications_fn,
)
if deduped_first:
discovered_count = await upsert_publications_fn(
db_session, run=run, scholar=scholar, publications=deduped_first
)
state.discovered_publication_count += discovered_count
while state.parsed_page.has_show_more_button:
await db_session.refresh(run)
@ -337,14 +356,15 @@ class PaginationEngine:
)
if next_parsed_page.publications:
deduped_next = _dedupe_publication_candidates(
list(next_parsed_page.publications), seen_canonical=seen_canonical
await self._upsert_page_publications(
db_session,
run=run,
scholar=scholar,
publications=next_parsed_page.publications,
seen_canonical=seen_canonical,
state=state,
upsert_publications_fn=upsert_publications_fn,
)
if deduped_next:
discovered_count = await upsert_publications_fn(
db_session, run=run, scholar=scholar, publications=deduped_next
)
state.discovered_publication_count += discovered_count
if self._handle_page_state_transition(state=state):
return

46
app/services/ingestion/run_completion.py
View file

@ -262,25 +262,13 @@ def build_failure_debug_context(
return context
def complete_run_for_user(
def _log_alert_threshold_warnings(
*,
user_settings: Any,
run: CrawlRun,
scholars: list[ScholarProfile],
user_id: int,
progress: RunProgress,
idempotency_key: str | None,
alert_blocked_failure_threshold: int,
alert_network_failure_threshold: int,
alert_retry_scheduled_threshold: int,
) -> tuple[RunFailureSummary, RunAlertSummary]:
failure_summary = summarize_failures(scholar_results=progress.scholar_results)
alert_summary = build_alert_summary(
failure_summary=failure_summary,
alert_blocked_failure_threshold=alert_blocked_failure_threshold,
alert_network_failure_threshold=alert_network_failure_threshold,
alert_retry_scheduled_threshold=alert_retry_scheduled_threshold,
)
run: CrawlRun,
failure_summary: RunFailureSummary,
alert_summary: RunAlertSummary,
) -> None:
if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]:
structured_log(
logger,
@ -311,6 +299,30 @@ def complete_run_for_user(
retries_scheduled_count=failure_summary.retries_scheduled_count,
threshold=alert_summary.retry_scheduled_threshold,
)
def complete_run_for_user(
*,
user_settings: Any,
run: CrawlRun,
scholars: list[ScholarProfile],
user_id: int,
progress: RunProgress,
idempotency_key: str | None,
alert_blocked_failure_threshold: int,
alert_network_failure_threshold: int,
alert_retry_scheduled_threshold: int,
) -> tuple[RunFailureSummary, RunAlertSummary]:
failure_summary = summarize_failures(scholar_results=progress.scholar_results)
alert_summary = build_alert_summary(
failure_summary=failure_summary,
alert_blocked_failure_threshold=alert_blocked_failure_threshold,
alert_network_failure_threshold=alert_network_failure_threshold,
alert_retry_scheduled_threshold=alert_retry_scheduled_threshold,
)
_log_alert_threshold_warnings(
user_id=user_id, run=run, failure_summary=failure_summary, alert_summary=alert_summary
)
apply_safety_outcome(user_settings=user_settings, run=run, user_id=user_id, alert_summary=alert_summary)
run_status = resolve_run_status(
scholar_count=len(scholars),