refactor: extract helpers from oversized router and scheduler files
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
55315299c7
commit
f11947aad2
9 changed files with 1187 additions and 1042 deletions
|
|
@ -6,24 +6,21 @@ from dataclasses import dataclass
|
|||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
QueueItemStatus,
|
||||
RunTriggerType,
|
||||
ScholarProfile,
|
||||
User,
|
||||
UserSetting,
|
||||
)
|
||||
from app.db.session import get_session_factory
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.ingestion import queue as queue_service
|
||||
from app.services.ingestion.application import (
|
||||
RunAlreadyInProgressError,
|
||||
RunBlockedBySafetyPolicyError,
|
||||
ScholarIngestionService,
|
||||
)
|
||||
from app.services.ingestion.queue_runner import QueueJobRunner
|
||||
from app.services.scholar.source import LiveScholarSource
|
||||
from app.services.settings import application as user_settings_service
|
||||
from app.settings import settings
|
||||
|
|
@ -85,6 +82,18 @@ class SchedulerService:
|
|||
self._queue_batch_size = max(1, int(queue_batch_size))
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._source = LiveScholarSource()
|
||||
self._queue_runner = QueueJobRunner(
|
||||
tick_seconds=self._tick_seconds,
|
||||
network_error_retries=self._network_error_retries,
|
||||
retry_backoff_seconds=self._retry_backoff_seconds,
|
||||
max_pages_per_scholar=self._max_pages_per_scholar,
|
||||
page_size=self._page_size,
|
||||
continuation_queue_enabled=self._continuation_queue_enabled,
|
||||
continuation_base_delay_seconds=self._continuation_base_delay_seconds,
|
||||
continuation_max_delay_seconds=self._continuation_max_delay_seconds,
|
||||
continuation_max_attempts=self._continuation_max_attempts,
|
||||
queue_batch_size=self._queue_batch_size,
|
||||
)
|
||||
|
||||
async def start(self) -> None:
|
||||
if not self._enabled:
|
||||
|
|
@ -136,7 +145,7 @@ class SchedulerService:
|
|||
|
||||
async def _tick_once(self) -> None:
|
||||
if self._continuation_queue_enabled:
|
||||
await self._drain_continuation_queue()
|
||||
await self._queue_runner.drain_continuation_queue()
|
||||
|
||||
await self._drain_pdf_queue()
|
||||
|
||||
|
|
@ -281,313 +290,6 @@ class SchedulerService:
|
|||
new_publication_count=run_summary.new_publication_count,
|
||||
)
|
||||
|
||||
async def _drain_continuation_queue(self) -> None:
|
||||
now = datetime.now(UTC)
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
jobs = await queue_service.list_due_jobs(
|
||||
session,
|
||||
now=now,
|
||||
limit=self._queue_batch_size,
|
||||
)
|
||||
for job in jobs:
|
||||
await self._run_queue_job(job)
|
||||
|
||||
async def _drop_queue_job_if_max_attempts(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
if job.attempt_count < self._continuation_max_attempts:
|
||||
return False
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="max_attempts_reached",
|
||||
)
|
||||
await session.commit()
|
||||
if dropped is not None:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"scheduler.queue_item_dropped_max_attempts",
|
||||
queue_item_id=job.id,
|
||||
user_id=job.user_id,
|
||||
scholar_profile_id=job.scholar_profile_id,
|
||||
attempt_count=job.attempt_count,
|
||||
max_attempts=self._continuation_max_attempts,
|
||||
)
|
||||
return True
|
||||
|
||||
async def _mark_queue_job_retrying(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
queue_item = await queue_service.mark_retrying(session, job_id=job.id)
|
||||
await session.commit()
|
||||
if queue_item is None:
|
||||
return False
|
||||
return queue_item.status != QueueItemStatus.DROPPED.value
|
||||
|
||||
async def _queue_job_has_available_scholar(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
scholar_result = await session.execute(
|
||||
select(ScholarProfile.id).where(
|
||||
ScholarProfile.user_id == job.user_id,
|
||||
ScholarProfile.id == job.scholar_profile_id,
|
||||
ScholarProfile.is_enabled.is_(True),
|
||||
)
|
||||
)
|
||||
scholar_id = scholar_result.scalar_one_or_none()
|
||||
if scholar_id is not None:
|
||||
return True
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="scholar_unavailable",
|
||||
)
|
||||
await session.commit()
|
||||
if dropped is not None:
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"scheduler.queue_item_dropped_scholar_unavailable",
|
||||
queue_item_id=job.id,
|
||||
user_id=job.user_id,
|
||||
scholar_profile_id=job.scholar_profile_id,
|
||||
)
|
||||
return False
|
||||
|
||||
async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=max(self._tick_seconds, 15),
|
||||
reason="user_run_lock_active",
|
||||
error="run_already_in_progress",
|
||||
)
|
||||
await recovery_session.commit()
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"scheduler.queue_item_deferred_lock",
|
||||
queue_item_id=job.id,
|
||||
user_id=job.user_id,
|
||||
)
|
||||
|
||||
async def _reschedule_queue_job_safety_cooldown(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
exc: RunBlockedBySafetyPolicyError,
|
||||
) -> None:
|
||||
cooldown_remaining_seconds = max(
|
||||
self._tick_seconds,
|
||||
int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
|
||||
)
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
|
||||
reason="scrape_safety_cooldown",
|
||||
error=str(exc.message),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"scheduler.queue_item_deferred_safety_cooldown",
|
||||
queue_item_id=job.id,
|
||||
user_id=job.user_id,
|
||||
reason=exc.safety_state.get("cooldown_reason"),
|
||||
cooldown_remaining_seconds=cooldown_remaining_seconds,
|
||||
)
|
||||
|
||||
async def _reschedule_queue_job_after_exception(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
*,
|
||||
exc: Exception,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
|
||||
if queue_item is None:
|
||||
await recovery_session.commit()
|
||||
return
|
||||
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
|
||||
await queue_service.mark_dropped(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
reason="scheduler_exception_max_attempts",
|
||||
error=str(exc),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"scheduler.queue_item_dropped_after_exception",
|
||||
queue_item_id=job.id,
|
||||
user_id=job.user_id,
|
||||
attempt_count=queue_item.attempt_count,
|
||||
)
|
||||
return
|
||||
delay_seconds = queue_service.compute_backoff_seconds(
|
||||
base_seconds=self._continuation_base_delay_seconds,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
max_seconds=self._continuation_max_delay_seconds,
|
||||
)
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=delay_seconds,
|
||||
reason="scheduler_exception",
|
||||
error=str(exc),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.exception("scheduler.queue_item_run_failed", extra={"queue_item_id": job.id, "user_id": job.user_id})
|
||||
|
||||
async def _run_ingestion_for_queue_job(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
):
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id)
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
return await ingestion.run_for_user(
|
||||
session,
|
||||
user_id=job.user_id,
|
||||
trigger_type=RunTriggerType.SCHEDULED,
|
||||
request_delay_seconds=request_delay_seconds,
|
||||
network_error_retries=self._network_error_retries,
|
||||
retry_backoff_seconds=self._retry_backoff_seconds,
|
||||
rate_limit_retries=settings.ingestion_rate_limit_retries,
|
||||
rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds,
|
||||
max_pages_per_scholar=self._max_pages_per_scholar,
|
||||
page_size=self._page_size,
|
||||
scholar_profile_ids={job.scholar_profile_id},
|
||||
start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart},
|
||||
auto_queue_continuations=self._continuation_queue_enabled,
|
||||
queue_delay_seconds=self._continuation_base_delay_seconds,
|
||||
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
|
||||
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
|
||||
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
|
||||
)
|
||||
except RunAlreadyInProgressError:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_lock_active(job)
|
||||
except RunBlockedBySafetyPolicyError as exc:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_safety_cooldown(job, exc)
|
||||
except Exception as exc:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_after_exception(job, exc=exc)
|
||||
return None
|
||||
|
||||
async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
if int(run_summary.failed_count) <= 0:
|
||||
queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
|
||||
await session.commit()
|
||||
if queue_item is None:
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"scheduler.queue_item_resolved",
|
||||
queue_item_id=job.id,
|
||||
user_id=job.user_id,
|
||||
run_id=run_summary.crawl_run_id,
|
||||
status=run_summary.status.value,
|
||||
)
|
||||
return
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"scheduler.queue_item_progressed",
|
||||
queue_item_id=job.id,
|
||||
user_id=job.user_id,
|
||||
run_id=run_summary.crawl_run_id,
|
||||
status=run_summary.status.value,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
)
|
||||
return
|
||||
queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
|
||||
if queue_item is None:
|
||||
await session.commit()
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"scheduler.queue_item_resolved",
|
||||
queue_item_id=job.id,
|
||||
user_id=job.user_id,
|
||||
run_id=run_summary.crawl_run_id,
|
||||
status=run_summary.status.value,
|
||||
)
|
||||
return
|
||||
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
|
||||
await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
|
||||
await session.commit()
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"scheduler.queue_item_dropped_max_attempts_after_run",
|
||||
queue_item_id=job.id,
|
||||
user_id=job.user_id,
|
||||
attempt_count=queue_item.attempt_count,
|
||||
run_id=run_summary.crawl_run_id,
|
||||
status=run_summary.status.value,
|
||||
)
|
||||
return
|
||||
delay_seconds = queue_service.compute_backoff_seconds(
|
||||
base_seconds=self._continuation_base_delay_seconds,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
max_seconds=self._continuation_max_delay_seconds,
|
||||
)
|
||||
await queue_service.reschedule_job(
|
||||
session,
|
||||
job_id=job.id,
|
||||
delay_seconds=delay_seconds,
|
||||
reason=queue_item.reason,
|
||||
error=queue_item.last_error,
|
||||
)
|
||||
await session.commit()
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"scheduler.queue_item_rescheduled_failed",
|
||||
queue_item_id=job.id,
|
||||
user_id=job.user_id,
|
||||
run_id=run_summary.crawl_run_id,
|
||||
status=run_summary.status.value,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
delay_seconds=delay_seconds,
|
||||
)
|
||||
|
||||
async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
|
||||
if await self._drop_queue_job_if_max_attempts(job):
|
||||
return
|
||||
if not await self._mark_queue_job_retrying(job):
|
||||
return
|
||||
if not await self._queue_job_has_available_scholar(job):
|
||||
return
|
||||
run_summary = await self._run_ingestion_for_queue_job(job)
|
||||
if run_summary is None:
|
||||
return
|
||||
await self._finalize_queue_job_after_run(job, run_summary)
|
||||
|
||||
async def _drain_pdf_queue(self) -> None:
|
||||
from app.services.publications.pdf_queue import drain_ready_jobs
|
||||
|
||||
|
|
@ -608,15 +310,3 @@ class SchedulerService:
|
|||
)
|
||||
except Exception:
|
||||
logger.exception("scheduler.pdf_queue_drain_failed", extra={})
|
||||
|
||||
async def _load_request_delay_for_user(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int:
|
||||
result = await db_session.execute(
|
||||
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
|
||||
)
|
||||
delay = result.scalar_one_or_none()
|
||||
return _effective_request_delay_seconds(delay)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue