justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

Bugfixeds

Browse source
This commit is contained in:
Justin Visser 2026-03-01 22:54:23 +01:00
parent a5165dc3ba
commit 4403c9ebde
20 changed files with 199 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

57
app/services/arxiv/rate_limit.py
View file

@ -53,7 +53,7 @@ async def get_arxiv_cooldown_status(*, now_utc: datetime | None = None) -> Arxiv
result = await db_session.execute(
select(ArxivRuntimeState.cooldown_until).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
)
cooldown_until = _normalize_datetime(result.scalar_one_or_none())
cooldown_until = _normalize_datetime(result.scalar_one_or_none())
remaining_seconds = _cooldown_remaining_seconds(cooldown_until, now_utc=timestamp)
return ArxivCooldownStatus(
is_active=remaining_seconds > 0.0,
@ -68,26 +68,31 @@ async def _run_serialized_fetch(
source_path: str,
) -> tuple[httpx.Response, bool]:
session_factory = get_session_factory()
async with session_factory() as db_session, db_session.begin():
await _acquire_arxiv_lock(db_session)
runtime_state = await _load_runtime_state_for_update(db_session)
wait_seconds = await _wait_for_allowed_slot_or_raise(
runtime_state,
source_path=source_path,
)
runtime_state.next_allowed_at = datetime.now(UTC) + timedelta(seconds=_min_interval_seconds())
async with session_factory() as lock_session:
await _acquire_arxiv_lock(lock_session)
try:
async with session_factory() as db_session, db_session.begin():
runtime_state = await _load_runtime_state_for_update(db_session)
wait_seconds = await _wait_for_allowed_slot_or_raise(
runtime_state,
source_path=source_path,
)
runtime_state.next_allowed_at = datetime.now(UTC) + timedelta(seconds=_min_interval_seconds())
if wait_seconds > 0:
await asyncio.sleep(wait_seconds)
response = await fetch()
if wait_seconds > 0:
await asyncio.sleep(wait_seconds)
response = await fetch()
async with session_factory() as db_session, db_session.begin():
runtime_state = await _load_runtime_state_for_update(db_session)
hit_rate_limit = _record_post_response_state(
runtime_state,
response_status=int(response.status_code),
source_path=source_path,
)
async with session_factory() as db_session, db_session.begin():
runtime_state = await _load_runtime_state_for_update(db_session)
hit_rate_limit = _record_post_response_state(
runtime_state,
response_status=int(response.status_code),
source_path=source_path,
)
cooldown_until_value = runtime_state.cooldown_until
finally:
await _release_arxiv_lock(lock_session)
structured_log(
logger,
"info",
@ -95,7 +100,7 @@ async def _run_serialized_fetch(
status_code=int(response.status_code),
wait_seconds=wait_seconds,
cooldown_remaining_seconds=_cooldown_remaining_seconds(
runtime_state.cooldown_until, now_utc=datetime.now(UTC)
cooldown_until_value, now_utc=datetime.now(UTC)
),
source_path=source_path,
)
@ -104,7 +109,17 @@ async def _run_serialized_fetch(
async def _acquire_arxiv_lock(db_session: AsyncSession) -> None:
await db_session.execute(
text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"),
text("SELECT pg_advisory_lock(:namespace, :lock_key)"),
{
"namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
"lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,
},
)
async def _release_arxiv_lock(db_session: AsyncSession) -> None:
await db_session.execute(
text("SELECT pg_advisory_unlock(:namespace, :lock_key)"),
{
"namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
"lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,

7
app/services/crossref/application.py
View file

@ -20,7 +20,7 @@ if TYPE_CHECKING:
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
TOKEN_RE = re.compile(r"[a-z0-9]+")
NON_ALNUM_RE = re.compile(r"[^a-z0-9\\s]+")
NON_ALNUM_RE = re.compile(r"[^a-z0-9\s]+")
STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis"}
_RATE_LOCK = threading.Lock()
_LAST_REQUEST_AT = 0.0
@ -35,8 +35,9 @@ def _rate_limit_wait(min_interval_seconds: float) -> None:
with _RATE_LOCK:
elapsed = time.monotonic() - _LAST_REQUEST_AT
remaining = interval - elapsed
if remaining > 0:
time.sleep(remaining)
if remaining > 0:
time.sleep(remaining)
with _RATE_LOCK:
_LAST_REQUEST_AT = time.monotonic()

3
app/services/dbops/near_duplicate_repair.py
View file

@ -162,7 +162,10 @@ async def _complete_job(
async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
from sqlalchemy.orm import make_transient
await db_session.rollback()
make_transient(job)
job.status = REPAIR_STATUS_FAILED
job.error_text = str(error)
job.finished_at = _utcnow()

1
app/services/ingestion/application.py
View file

@ -401,6 +401,7 @@ class ScholarIngestionService:
if run_to_fail:
run_to_fail.status = RunStatus.FAILED
run_to_fail.end_dt = datetime.now(UTC)
run_to_fail.error_log = run_to_fail.error_log or {}
run_to_fail.error_log["terminal_exception"] = str(exc)
await cleanup_session.commit()
except Exception:

68
app/services/ingestion/publication_upsert.py
View file

@ -176,45 +176,51 @@ async def upsert_profile_publications(
seen_publication_ids: set[int] = set()
discovered_count = 0
for candidate in publications:
publication = await resolve_publication(db_session, candidate)
if publication.id in seen_publication_ids:
continue
seen_publication_ids.add(publication.id)
try:
for candidate in publications:
publication = await resolve_publication(db_session, candidate)
if publication.id in seen_publication_ids:
continue
seen_publication_ids.add(publication.id)
link_result = await db_session.execute(
select(ScholarPublication).where(
ScholarPublication.scholar_profile_id == scholar.id,
ScholarPublication.publication_id == publication.id,
link_result = await db_session.execute(
select(ScholarPublication).where(
ScholarPublication.scholar_profile_id == scholar.id,
ScholarPublication.publication_id == publication.id,
)
)
)
link = link_result.scalar_one_or_none()
if link is not None:
continue
link = link_result.scalar_one_or_none()
if link is not None:
continue
link = ScholarPublication(
scholar_profile_id=scholar.id,
publication_id=publication.id,
is_read=False,
first_seen_run_id=run.id,
)
db_session.add(link)
discovered_count += 1
link = ScholarPublication(
scholar_profile_id=scholar.id,
publication_id=publication.id,
is_read=False,
first_seen_run_id=run.id,
)
db_session.add(link)
discovered_count += 1
await commit_discovered_publication(
db_session,
run=run,
scholar=scholar,
publication=publication,
)
await flush_discovered_publication(
db_session,
run=run,
scholar=scholar,
publication=publication,
)
if not scholar.baseline_completed:
scholar.baseline_completed = True
if not scholar.baseline_completed:
scholar.baseline_completed = True
await db_session.commit()
except Exception:
await db_session.rollback()
raise
return discovered_count
async def commit_discovered_publication(
async def flush_discovered_publication(
db_session: AsyncSession,
*,
run: CrawlRun,
@ -222,7 +228,7 @@ async def commit_discovered_publication(
publication: Publication,
) -> None:
run.new_pub_count = int(run.new_pub_count or 0) + 1
await db_session.commit()
await db_session.flush()
await run_events.publish(
run_id=run.id,
event_type="publication_discovered",

1
app/services/ingestion/queue.py
View file

@ -196,6 +196,7 @@ async def list_due_jobs(
IngestionQueueItem.id.asc(),
)
.limit(limit)
.with_for_update(skip_locked=True)
)
rows = list(result.scalars().all())
jobs: list[ContinuationQueueJob] = []

1
app/services/publications/pdf_queue_common.py
View file

@ -47,4 +47,5 @@ def queued_job(
status=PDF_STATUS_QUEUED,
queued_at=now,
last_requested_by_user_id=user_id,
attempt_count=0,
)

22
app/services/publications/pdf_queue_resolution.py
View file

@ -46,7 +46,7 @@ async def _mark_attempt_started(
db_session.add(job)
job.status = PDF_STATUS_RUNNING
job.last_attempt_at = utcnow()
job.attempt_count = int(job.attempt_count) + 1
job.attempt_count = int(job.attempt_count or 0) + 1
db_session.add(
event_row(
publication_id=publication_id,
@ -240,6 +240,26 @@ async def _run_resolution_task(
detail="Stopping PDF resolution batch — OpenAlex daily budget exhausted",
)
break
except Exception:
structured_log(
logger,
"exception",
"pdf_queue.row_failed",
publication_id=row.publication_id,
)
try:
await _persist_outcome(
publication_id=row.publication_id,
user_id=user_id,
outcome=_failed_outcome(row=row),
)
except Exception:
structured_log(
logger,
"exception",
"pdf_queue.row_fail_persist_error",
publication_id=row.publication_id,
)
def _register_task(task: asyncio.Task[None]) -> None:

13
app/services/runs/events.py
View file

@ -8,6 +8,8 @@ from app.logging_utils import structured_log
logger = logging.getLogger(__name__)
_SUBSCRIBER_QUEUE_MAXSIZE = 256
class RunEventPublisher:
def __init__(self) -> None:
@ -17,7 +19,7 @@ class RunEventPublisher:
def subscribe(self, run_id: int) -> asyncio.Queue:
if run_id not in self._subscribers:
self._subscribers[run_id] = set()
queue: asyncio.Queue[Any] = asyncio.Queue()
queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=_SUBSCRIBER_QUEUE_MAXSIZE)
self._subscribers[run_id].add(queue)
structured_log(
logger,
@ -51,6 +53,10 @@ class RunEventPublisher:
"runs.event_subscriber_queue_full",
run_id=run_id,
)
self._subscribers[run_id].discard(queue)
async def publish_run_complete(self, run_id: int) -> None:
await self.publish(run_id, "run_complete", {})
run_events = RunEventPublisher()
@ -62,8 +68,11 @@ async def event_generator(run_id: int) -> AsyncGenerator[str, None]:
while True:
# Wait for a new event
message = await queue.get()
# Server-Sent Events format: "event: <type>\ndata: <json>\n\n"
event_type = message["type"]
if event_type == "run_complete":
yield f"event: {event_type}\ndata: {{}}\n\n"
break
# Server-Sent Events format: "event: <type>\ndata: <json>\n\n"
data_str = json.dumps(message["data"])
yield f"event: {event_type}\ndata: {data_str}\n\n"
except asyncio.CancelledError: