justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

fix(db,frontend): reserve DB connections for API and reduce frontend polling

Browse source 
Background tasks (ingestion, PDF resolution, scheduler) now acquire an
asyncio.Semaphore before checking out a DB connection, guaranteeing at
least N connections remain available for API request handlers. Removes
duplicate polling loop from PublicationsPage, debounces publication
reload on run-status changes, and increases poll interval from 5s to 15s
since SSE handles live discovery.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-03-02 19:09:06 +01:00
parent a490e14126
commit f50b609a36
11 changed files with 97 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

32
app/services/ingestion/queue_runner.py
View file

@ -5,8 +5,8 @@ from datetime import UTC, datetime
from sqlalchemy import select
from app.db.background_session import background_session
from app.db.models import QueueItemStatus, RunTriggerType, ScholarProfile, UserSetting
from app.db.session import get_session_factory
from app.logging_utils import structured_log
from app.services.ingestion import queue as queue_service
from app.services.ingestion.application import (
@ -57,8 +57,7 @@ class QueueJobRunner:
async def drain_continuation_queue(self) -> None:
now = datetime.now(UTC)
session_factory = get_session_factory()
async with session_factory() as session:
async with background_session() as session:
jobs = await queue_service.list_due_jobs(
session,
now=now,
@ -73,8 +72,7 @@ class QueueJobRunner:
) -> bool:
if job.attempt_count < self._continuation_max_attempts:
return False
session_factory = get_session_factory()
async with session_factory() as session:
async with background_session() as session:
dropped = await queue_service.mark_dropped(
session,
job_id=job.id,
@ -98,8 +96,7 @@ class QueueJobRunner:
self,
job: queue_service.ContinuationQueueJob,
) -> bool:
session_factory = get_session_factory()
async with session_factory() as session:
async with background_session() as session:
queue_item = await queue_service.mark_retrying(session, job_id=job.id)
await session.commit()
if queue_item is None:
@ -110,8 +107,7 @@ class QueueJobRunner:
self,
job: queue_service.ContinuationQueueJob,
) -> bool:
session_factory = get_session_factory()
async with session_factory() as session:
async with background_session() as session:
scholar_result = await session.execute(
select(ScholarProfile.id).where(
ScholarProfile.user_id == job.user_id,
@ -140,8 +136,7 @@ class QueueJobRunner:
return False
async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
session_factory = get_session_factory()
async with session_factory() as recovery_session:
async with background_session() as recovery_session:
await queue_service.reschedule_job(
recovery_session,
job_id=job.id,
@ -167,8 +162,7 @@ class QueueJobRunner:
self._tick_seconds,
int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
)
session_factory = get_session_factory()
async with session_factory() as recovery_session:
async with background_session() as recovery_session:
await queue_service.reschedule_job(
recovery_session,
job_id=job.id,
@ -193,8 +187,7 @@ class QueueJobRunner:
*,
exc: Exception,
) -> None:
session_factory = get_session_factory()
async with session_factory() as recovery_session:
async with background_session() as recovery_session:
queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
if queue_item is None:
await recovery_session.commit()
@ -238,8 +231,7 @@ class QueueJobRunner:
)
async def _load_request_delay_for_user(self, user_id: int, *, floor: int) -> int:
session_factory = get_session_factory()
async with session_factory() as session:
async with background_session() as session:
result = await session.execute(
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
)
@ -253,8 +245,7 @@ class QueueJobRunner:
request_delay_floor: int,
):
request_delay_seconds = await self._load_request_delay_for_user(job.user_id, floor=request_delay_floor)
session_factory = get_session_factory()
async with session_factory() as session:
async with background_session() as session:
ingestion = ScholarIngestionService(source=self._source)
try:
return await ingestion.run_for_user(
@ -366,8 +357,7 @@ class QueueJobRunner:
)
async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
session_factory = get_session_factory()
async with session_factory() as session:
async with background_session() as session:
if int(run_summary.failed_count) <= 0:
await self._finalize_successful_queue_job(session, job, run_summary)
else:

14
app/services/ingestion/scheduler.py
View file

@ -8,13 +8,13 @@ from typing import Any
from sqlalchemy import select
from app.db.background_session import background_session
from app.db.models import (
CrawlRun,
RunTriggerType,
User,
UserSetting,
)
from app.db.session import get_session_factory
from app.logging_utils import structured_log
from app.services.ingestion.application import (
RunAlreadyInProgressError,
@ -148,8 +148,7 @@ class SchedulerService:
await self._run_candidate(candidate)
async def _load_candidate_rows(self) -> list[Any]:
session_factory = get_session_factory()
async with session_factory() as session:
async with background_session() as session:
result = await session.execute(
select(
UserSetting.user_id,
@ -204,8 +203,7 @@ class SchedulerService:
return candidates
async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
session_factory = get_session_factory()
async with session_factory() as session:
async with background_session() as session:
result = await session.execute(
select(CrawlRun.start_dt)
.where(
@ -227,8 +225,7 @@ class SchedulerService:
*,
candidate: _AutoRunCandidate,
):
session_factory = get_session_factory()
async with session_factory() as session:
async with background_session() as session:
ingestion = ScholarIngestionService(source=self._source)
try:
return await ingestion.run_for_user(
@ -294,8 +291,7 @@ class SchedulerService:
if is_budget_cooldown_active():
return
session_factory = get_session_factory()
async with session_factory() as session:
async with background_session() as session:
try:
processed = await drain_ready_jobs(
session,

11
app/services/publications/pdf_queue_resolution.py
View file

@ -4,8 +4,8 @@ import asyncio
import logging
from datetime import UTC, datetime, timedelta
from app.db.background_session import background_session
from app.db.models import Publication, PublicationPdfJob
from app.db.session import get_session_factory
from app.logging_utils import structured_log
from app.services.publication_identifiers import application as identifier_service
from app.services.publications.pdf_queue_common import (
@ -51,8 +51,7 @@ async def _mark_attempt_started(
publication_id: int,
user_id: int,
) -> None:
session_factory = get_session_factory()
async with session_factory() as db_session:
async with background_session() as db_session:
job = await db_session.get(PublicationPdfJob, publication_id)
if job is None:
job = queued_job(publication_id=publication_id, user_id=user_id)
@ -138,8 +137,7 @@ async def _persist_outcome(
user_id: int,
outcome: OaResolutionOutcome,
) -> None:
session_factory = get_session_factory()
async with session_factory() as db_session:
async with background_session() as db_session:
publication = await db_session.get(Publication, publication_id)
job = await db_session.get(PublicationPdfJob, publication_id)
if publication is None or job is None:
@ -220,8 +218,7 @@ async def _run_resolution_task(
openalex_api_key: str | None = None
try:
session_factory = get_session_factory()
async with session_factory() as key_session:
async with background_session() as key_session:
user_settings = await user_settings_service.get_or_create_settings(key_session, user_id=user_id)
openalex_api_key = getattr(user_settings, "openalex_api_key", None) or settings.openalex_api_key
except Exception: