justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

temp commit

Browse source
This commit is contained in:
Justin Visser 2026-02-26 12:54:19 +01:00
parent 8760f27b51
commit 0e9e49df16
193 changed files with 23228 additions and 935 deletions
Download patch file Download diff file Expand all files Collapse all files

26
app/services/domains/ingestion/scheduler.py
View file

@ -146,6 +146,9 @@ class SchedulerService:
async def _tick_once(self) -> None:
if self._continuation_queue_enabled:
await self._drain_continuation_queue()
await self._drain_pdf_queue()
candidates = await self._load_candidates()
if not candidates:
return
@ -488,6 +491,8 @@ class SchedulerService:
request_delay_seconds=request_delay_seconds,
network_error_retries=self._network_error_retries,
retry_backoff_seconds=self._retry_backoff_seconds,
rate_limit_retries=settings.ingestion_rate_limit_retries,
rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds,
max_pages_per_scholar=self._max_pages_per_scholar,
page_size=self._page_size,
scholar_profile_ids={job.scholar_profile_id},
@ -583,6 +588,27 @@ class SchedulerService:
return
await self._finalize_queue_job_after_run(job, run_summary)
async def _drain_pdf_queue(self) -> None:
from app.services.domains.publications.pdf_queue import drain_ready_jobs
session_factory = get_session_factory()
async with session_factory() as session:
try:
processed = await drain_ready_jobs(
session,
limit=settings.scheduler_pdf_queue_batch_size,
max_attempts=settings.pdf_auto_retry_max_attempts,
)
if processed > 0:
logger.info("scheduler.pdf_queue_drain_completed", extra={
"event": "scheduler.pdf_queue_drain_completed",
"processed_count": processed,
})
except Exception:
logger.exception("scheduler.pdf_queue_drain_failed", extra={
"event": "scheduler.pdf_queue_drain_failed",
})
async def _load_request_delay_for_user(
self,
db_session: AsyncSession,