396 lines
12 KiB
Python
396 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Any
|
|
|
|
from sqlalchemy import Select, and_, func, literal, or_, select, union_all
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.db.models import (
|
|
Publication,
|
|
PublicationPdfJob,
|
|
ScholarProfile,
|
|
ScholarPublication,
|
|
User,
|
|
)
|
|
from app.services.publication_identifiers import application as identifier_service
|
|
from app.services.publication_identifiers.types import DisplayIdentifier
|
|
from app.services.publications.types import PublicationListItem
|
|
|
|
PDF_STATUS_UNTRACKED = "untracked"
|
|
PDF_STATUS_QUEUED = "queued"
|
|
PDF_STATUS_RUNNING = "running"
|
|
PDF_STATUS_RESOLVED = "resolved"
|
|
PDF_STATUS_FAILED = "failed"
|
|
|
|
|
|
def _bounded_limit(limit: int, *, max_value: int = 500) -> int:
|
|
return max(1, min(int(limit), max_value))
|
|
|
|
|
|
def _bounded_offset(offset: int) -> int:
|
|
return max(int(offset), 0)
|
|
|
|
|
|
def _utcnow() -> datetime:
|
|
return datetime.now(UTC)
|
|
|
|
|
|
def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str:
|
|
return str(display_name or scholar_id or "unknown")
|
|
|
|
|
|
def _retry_item_from_publication(
|
|
publication: Publication,
|
|
*,
|
|
link_row: Any | None,
|
|
) -> PublicationListItem:
|
|
if link_row is None:
|
|
scholar_profile_id = 0
|
|
scholar_label = "unknown"
|
|
is_read = True
|
|
first_seen_at = publication.created_at or _utcnow()
|
|
else:
|
|
scholar_profile_id = int(link_row[0])
|
|
scholar_label = _retry_item_label(link_row[1], link_row[2])
|
|
is_read = bool(link_row[3])
|
|
first_seen_at = link_row[4] or publication.created_at or _utcnow()
|
|
return PublicationListItem(
|
|
publication_id=int(publication.id),
|
|
scholar_profile_id=scholar_profile_id,
|
|
scholar_label=scholar_label,
|
|
title=publication.title_raw,
|
|
year=publication.year,
|
|
citation_count=int(publication.citation_count or 0),
|
|
venue_text=publication.venue_text,
|
|
pub_url=publication.pub_url,
|
|
pdf_url=publication.pdf_url,
|
|
is_read=is_read,
|
|
first_seen_at=first_seen_at,
|
|
is_new_in_latest_run=False,
|
|
)
|
|
|
|
|
|
async def retry_item_for_publication_id(
|
|
db_session: AsyncSession,
|
|
*,
|
|
publication_id: int,
|
|
) -> PublicationListItem | None:
|
|
publication = await db_session.get(Publication, publication_id)
|
|
if publication is None:
|
|
return None
|
|
result = await db_session.execute(
|
|
select(
|
|
ScholarProfile.id,
|
|
ScholarProfile.display_name,
|
|
ScholarProfile.scholar_id,
|
|
ScholarPublication.is_read,
|
|
ScholarPublication.created_at,
|
|
)
|
|
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
|
.where(ScholarPublication.publication_id == publication_id)
|
|
.order_by(ScholarPublication.created_at.asc())
|
|
.limit(1)
|
|
)
|
|
return _retry_item_from_publication(publication, link_row=result.one_or_none())
|
|
|
|
|
|
def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem:
|
|
return PublicationListItem(
|
|
publication_id=int(publication.id),
|
|
scholar_profile_id=0,
|
|
scholar_label="",
|
|
title=publication.title_raw,
|
|
year=publication.year,
|
|
citation_count=int(publication.citation_count or 0),
|
|
venue_text=publication.venue_text,
|
|
pub_url=publication.pub_url,
|
|
pdf_url=publication.pdf_url,
|
|
is_read=True,
|
|
first_seen_at=publication.created_at or _utcnow(),
|
|
is_new_in_latest_run=False,
|
|
)
|
|
|
|
|
|
async def missing_pdf_candidates(
|
|
db_session: AsyncSession,
|
|
*,
|
|
limit: int,
|
|
) -> list[PublicationListItem]:
|
|
bounded_limit = max(1, min(int(limit), 5000))
|
|
now = datetime.now(UTC)
|
|
cooldown_threshold = now - timedelta(days=7)
|
|
|
|
result = await db_session.execute(
|
|
select(Publication)
|
|
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
|
|
.where(Publication.pdf_url.is_(None))
|
|
.where(
|
|
or_(
|
|
PublicationPdfJob.publication_id.is_(None),
|
|
and_(
|
|
PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
|
|
or_(
|
|
PublicationPdfJob.last_attempt_at.is_(None),
|
|
PublicationPdfJob.last_attempt_at < cooldown_threshold,
|
|
),
|
|
),
|
|
)
|
|
)
|
|
.order_by(Publication.updated_at.desc(), Publication.id.desc())
|
|
.limit(bounded_limit)
|
|
)
|
|
return [_queue_candidate_from_publication(publication) for publication in result.scalars()]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tracked / untracked queue SQL builders
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
|
|
stmt = (
|
|
select(
|
|
PublicationPdfJob.publication_id,
|
|
Publication.title_raw,
|
|
Publication.pdf_url,
|
|
PublicationPdfJob.status,
|
|
PublicationPdfJob.attempt_count,
|
|
PublicationPdfJob.last_failure_reason,
|
|
PublicationPdfJob.last_failure_detail,
|
|
PublicationPdfJob.last_source,
|
|
PublicationPdfJob.last_requested_by_user_id,
|
|
User.email,
|
|
PublicationPdfJob.queued_at,
|
|
PublicationPdfJob.last_attempt_at,
|
|
PublicationPdfJob.resolved_at,
|
|
PublicationPdfJob.updated_at,
|
|
)
|
|
.join(Publication, Publication.id == PublicationPdfJob.publication_id)
|
|
.outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id)
|
|
)
|
|
if status:
|
|
stmt = stmt.where(PublicationPdfJob.status == status)
|
|
return stmt
|
|
|
|
|
|
def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]:
|
|
return (
|
|
_tracked_queue_select_base(status=status)
|
|
.order_by(PublicationPdfJob.updated_at.desc())
|
|
.limit(_bounded_limit(limit))
|
|
.offset(_bounded_offset(offset))
|
|
)
|
|
|
|
|
|
def _untracked_queue_select_base() -> Select[tuple]:
|
|
return (
|
|
select(
|
|
Publication.id,
|
|
Publication.title_raw,
|
|
Publication.pdf_url,
|
|
literal(PDF_STATUS_UNTRACKED),
|
|
literal(0),
|
|
literal(None),
|
|
literal(None),
|
|
literal(None),
|
|
literal(None),
|
|
literal(None),
|
|
literal(None),
|
|
literal(None),
|
|
literal(None),
|
|
Publication.updated_at,
|
|
)
|
|
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
|
|
.where(Publication.pdf_url.is_(None))
|
|
.where(PublicationPdfJob.publication_id.is_(None))
|
|
)
|
|
|
|
|
|
def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]:
|
|
return (
|
|
_untracked_queue_select_base()
|
|
.order_by(Publication.updated_at.desc(), Publication.id.desc())
|
|
.limit(_bounded_limit(limit))
|
|
.offset(_bounded_offset(offset))
|
|
)
|
|
|
|
|
|
def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]:
|
|
union_stmt = union_all(
|
|
_tracked_queue_select_base(status=None),
|
|
_untracked_queue_select_base(),
|
|
).subquery()
|
|
return (
|
|
select(union_stmt)
|
|
.order_by(union_stmt.c.updated_at.desc())
|
|
.limit(_bounded_limit(limit))
|
|
.offset(_bounded_offset(offset))
|
|
)
|
|
|
|
|
|
def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]:
|
|
stmt = select(func.count()).select_from(PublicationPdfJob)
|
|
if status:
|
|
stmt = stmt.where(PublicationPdfJob.status == status)
|
|
return stmt
|
|
|
|
|
|
def _untracked_queue_count_select() -> Select[tuple]:
|
|
return (
|
|
select(func.count())
|
|
.select_from(Publication)
|
|
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
|
|
.where(Publication.pdf_url.is_(None))
|
|
.where(PublicationPdfJob.publication_id.is_(None))
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Row hydration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PdfQueueListItem:
|
|
publication_id: int
|
|
title: str
|
|
pdf_url: str | None
|
|
status: str
|
|
attempt_count: int
|
|
last_failure_reason: str | None
|
|
last_failure_detail: str | None
|
|
last_source: str | None
|
|
requested_by_user_id: int | None
|
|
requested_by_email: str | None
|
|
queued_at: datetime | None
|
|
last_attempt_at: datetime | None
|
|
resolved_at: datetime | None
|
|
updated_at: datetime
|
|
display_identifier: DisplayIdentifier | None = None
|
|
|
|
|
|
def _queue_item_from_row(row: Any) -> PdfQueueListItem:
|
|
return PdfQueueListItem(
|
|
publication_id=int(row[0]),
|
|
title=str(row[1] or ""),
|
|
pdf_url=row[2],
|
|
status=str(row[3] or PDF_STATUS_UNTRACKED),
|
|
attempt_count=int(row[4] or 0),
|
|
last_failure_reason=row[5],
|
|
last_failure_detail=row[6],
|
|
last_source=row[7],
|
|
requested_by_user_id=int(row[8]) if row[8] is not None else None,
|
|
requested_by_email=row[9],
|
|
queued_at=row[10],
|
|
last_attempt_at=row[11],
|
|
resolved_at=row[12],
|
|
updated_at=row[13],
|
|
)
|
|
|
|
|
|
async def _hydrated_queue_items(
|
|
db_session: AsyncSession,
|
|
*,
|
|
rows: list[Any],
|
|
) -> list[PdfQueueListItem]:
|
|
items = [_queue_item_from_row(row) for row in rows]
|
|
return await identifier_service.overlay_pdf_queue_items_with_display_identifiers(
|
|
db_session,
|
|
items=items,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public listing / counting
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PdfQueuePage:
|
|
items: list[PdfQueueListItem]
|
|
total_count: int
|
|
limit: int
|
|
offset: int
|
|
|
|
|
|
async def list_pdf_queue_items(
|
|
db_session: AsyncSession,
|
|
*,
|
|
limit: int = 100,
|
|
offset: int = 0,
|
|
status: str | None = None,
|
|
) -> list[PdfQueueListItem]:
|
|
bounded_limit = _bounded_limit(limit)
|
|
bounded_offset = _bounded_offset(offset)
|
|
normalized_status = (status or "").strip().lower() or None
|
|
if normalized_status == PDF_STATUS_UNTRACKED:
|
|
result = await db_session.execute(
|
|
_untracked_queue_select(
|
|
limit=bounded_limit,
|
|
offset=bounded_offset,
|
|
)
|
|
)
|
|
return await _hydrated_queue_items(db_session, rows=list(result.all()))
|
|
if normalized_status is None:
|
|
result = await db_session.execute(
|
|
_all_queue_select(
|
|
limit=bounded_limit,
|
|
offset=bounded_offset,
|
|
)
|
|
)
|
|
return await _hydrated_queue_items(db_session, rows=list(result.all()))
|
|
result = await db_session.execute(
|
|
_tracked_queue_select(
|
|
limit=bounded_limit,
|
|
offset=bounded_offset,
|
|
status=normalized_status,
|
|
)
|
|
)
|
|
return await _hydrated_queue_items(db_session, rows=list(result.all()))
|
|
|
|
|
|
async def count_pdf_queue_items(
|
|
db_session: AsyncSession,
|
|
*,
|
|
status: str | None = None,
|
|
) -> int:
|
|
normalized_status = (status or "").strip().lower() or None
|
|
if normalized_status == PDF_STATUS_UNTRACKED:
|
|
result = await db_session.execute(_untracked_queue_count_select())
|
|
return int(result.scalar_one() or 0)
|
|
tracked_result = await db_session.execute(_tracked_queue_count_select(status=normalized_status))
|
|
tracked_count = int(tracked_result.scalar_one() or 0)
|
|
if normalized_status is not None:
|
|
return tracked_count
|
|
untracked_result = await db_session.execute(_untracked_queue_count_select())
|
|
untracked_count = int(untracked_result.scalar_one() or 0)
|
|
return tracked_count + untracked_count
|
|
|
|
|
|
async def list_pdf_queue_page(
|
|
db_session: AsyncSession,
|
|
*,
|
|
limit: int = 100,
|
|
offset: int = 0,
|
|
status: str | None = None,
|
|
) -> PdfQueuePage:
|
|
bounded_limit = _bounded_limit(limit)
|
|
bounded_offset = _bounded_offset(offset)
|
|
items = await list_pdf_queue_items(
|
|
db_session,
|
|
limit=bounded_limit,
|
|
offset=bounded_offset,
|
|
status=status,
|
|
)
|
|
total_count = await count_pdf_queue_items(
|
|
db_session,
|
|
status=status,
|
|
)
|
|
return PdfQueuePage(
|
|
items=items,
|
|
total_count=total_count,
|
|
limit=bounded_limit,
|
|
offset=bounded_offset,
|
|
)
|