Big changes
This commit is contained in:
parent
4240ad38e2
commit
e3f0d63fec
99 changed files with 8804 additions and 1731 deletions
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||
|
||||
from app.services.domains.publications.counts import (
|
||||
count_for_user,
|
||||
count_favorite_for_user,
|
||||
count_latest_for_user,
|
||||
count_unread_for_user,
|
||||
)
|
||||
|
|
@ -12,7 +13,9 @@ from app.services.domains.publications.listing import (
|
|||
list_unread_for_user,
|
||||
)
|
||||
from app.services.domains.publications.enrichment import (
|
||||
hydrate_pdf_enrichment_state,
|
||||
schedule_missing_pdf_enrichment_for_user,
|
||||
schedule_retry_pdf_enrichment_for_row,
|
||||
)
|
||||
from app.services.domains.publications.modes import (
|
||||
MODE_ALL,
|
||||
|
|
@ -30,6 +33,14 @@ from app.services.domains.publications.queries import (
|
|||
from app.services.domains.publications.read_state import (
|
||||
mark_all_unread_as_read_for_user,
|
||||
mark_selected_as_read_for_user,
|
||||
set_publication_favorite_for_user,
|
||||
)
|
||||
from app.services.domains.publications.pdf_queue import (
|
||||
count_pdf_queue_items,
|
||||
enqueue_all_missing_pdf_jobs,
|
||||
enqueue_retry_pdf_job_for_publication_id,
|
||||
list_pdf_queue_page,
|
||||
list_pdf_queue_items,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
|
|
@ -49,10 +60,19 @@ __all__ = [
|
|||
"list_unread_for_user",
|
||||
"list_new_for_latest_run_for_user",
|
||||
"retry_pdf_for_user",
|
||||
"hydrate_pdf_enrichment_state",
|
||||
"schedule_retry_pdf_enrichment_for_row",
|
||||
"list_pdf_queue_items",
|
||||
"list_pdf_queue_page",
|
||||
"count_pdf_queue_items",
|
||||
"enqueue_all_missing_pdf_jobs",
|
||||
"enqueue_retry_pdf_job_for_publication_id",
|
||||
"schedule_missing_pdf_enrichment_for_user",
|
||||
"count_for_user",
|
||||
"count_favorite_for_user",
|
||||
"count_unread_for_user",
|
||||
"count_latest_for_user",
|
||||
"mark_all_unread_as_read_for_user",
|
||||
"mark_selected_as_read_for_user",
|
||||
"set_publication_favorite_for_user",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ async def count_for_user(
|
|||
user_id: int,
|
||||
mode: str = MODE_ALL,
|
||||
scholar_profile_id: int | None = None,
|
||||
favorite_only: bool = False,
|
||||
) -> int:
|
||||
resolved_mode = resolve_publication_view_mode(mode)
|
||||
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
|
||||
|
|
@ -30,6 +31,8 @@ async def count_for_user(
|
|||
)
|
||||
if scholar_profile_id is not None:
|
||||
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
||||
if favorite_only:
|
||||
stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
|
||||
if resolved_mode == MODE_UNREAD:
|
||||
stmt = stmt.where(ScholarPublication.is_read.is_(False))
|
||||
if resolved_mode == MODE_LATEST:
|
||||
|
|
@ -45,12 +48,14 @@ async def count_unread_for_user(
|
|||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int | None = None,
|
||||
favorite_only: bool = False,
|
||||
) -> int:
|
||||
return await count_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_UNREAD,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
favorite_only=favorite_only,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -59,10 +64,27 @@ async def count_latest_for_user(
|
|||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int | None = None,
|
||||
favorite_only: bool = False,
|
||||
) -> int:
|
||||
return await count_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_LATEST,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
favorite_only=favorite_only,
|
||||
)
|
||||
|
||||
|
||||
async def count_favorite_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int | None = None,
|
||||
) -> int:
|
||||
return await count_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_ALL,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
favorite_only=True,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,135 +1,73 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.domains.publications.listing import (
|
||||
missing_pdf_items,
|
||||
resolve_and_persist_oa_metadata,
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.services.domains.publications.pdf_queue import (
|
||||
enqueue_missing_pdf_jobs,
|
||||
enqueue_retry_pdf_job,
|
||||
overlay_pdf_job_state,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_enrichment_lock = asyncio.Lock()
|
||||
_inflight_publications: set[tuple[int, int]] = set()
|
||||
_recent_attempt_seconds: dict[tuple[int, int], float] = {}
|
||||
_scheduled_tasks: set[asyncio.Task[None]] = set()
|
||||
|
||||
|
||||
def _cooldown_seconds() -> float:
|
||||
return max(float(settings.unpaywall_retry_cooldown_seconds), 1.0)
|
||||
|
||||
|
||||
def _prune_recent_attempts(now_seconds: float, *, cooldown_seconds: float) -> None:
|
||||
expiry = cooldown_seconds * 3
|
||||
stale_keys = [
|
||||
key for key, attempted_seconds in _recent_attempt_seconds.items()
|
||||
if now_seconds - attempted_seconds >= expiry
|
||||
]
|
||||
for key in stale_keys:
|
||||
_recent_attempt_seconds.pop(key, None)
|
||||
|
||||
|
||||
async def _claim_items(
|
||||
*,
|
||||
user_id: int,
|
||||
items: list[PublicationListItem],
|
||||
max_items: int,
|
||||
) -> list[PublicationListItem]:
|
||||
candidates = missing_pdf_items(items, limit=max_items)
|
||||
if not candidates:
|
||||
return []
|
||||
now_seconds = time.monotonic()
|
||||
cooldown_seconds = _cooldown_seconds()
|
||||
claimed: list[PublicationListItem] = []
|
||||
async with _enrichment_lock:
|
||||
_prune_recent_attempts(now_seconds, cooldown_seconds=cooldown_seconds)
|
||||
for item in candidates:
|
||||
key = (user_id, item.publication_id)
|
||||
attempted_seconds = _recent_attempt_seconds.get(key)
|
||||
if key in _inflight_publications:
|
||||
continue
|
||||
if attempted_seconds is not None and now_seconds - attempted_seconds < cooldown_seconds:
|
||||
continue
|
||||
_inflight_publications.add(key)
|
||||
_recent_attempt_seconds[key] = now_seconds
|
||||
claimed.append(item)
|
||||
return claimed
|
||||
|
||||
|
||||
async def _release_claims(*, user_id: int, publication_ids: list[int]) -> None:
|
||||
async with _enrichment_lock:
|
||||
for publication_id in publication_ids:
|
||||
_inflight_publications.discard((user_id, publication_id))
|
||||
|
||||
|
||||
def _on_task_done(task: asyncio.Task[None]) -> None:
|
||||
_scheduled_tasks.discard(task)
|
||||
try:
|
||||
task.result()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"publications.enrichment.task_failed",
|
||||
extra={"event": "publications.enrichment.task_failed"},
|
||||
)
|
||||
|
||||
|
||||
async def _run_enrichment(
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
items: list[PublicationListItem],
|
||||
) -> None:
|
||||
publication_ids = [item.publication_id for item in items]
|
||||
try:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as db_session:
|
||||
await resolve_and_persist_oa_metadata(
|
||||
db_session,
|
||||
rows=items,
|
||||
unpaywall_email=request_email,
|
||||
)
|
||||
finally:
|
||||
await _release_claims(user_id=user_id, publication_ids=publication_ids)
|
||||
logger.info(
|
||||
"publications.enrichment.completed",
|
||||
extra={
|
||||
"event": "publications.enrichment.completed",
|
||||
"user_id": user_id,
|
||||
"publication_count": len(items),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def schedule_missing_pdf_enrichment_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
items: list[PublicationListItem],
|
||||
max_items: int,
|
||||
) -> int:
|
||||
claimed_items = await _claim_items(user_id=user_id, items=items, max_items=max_items)
|
||||
if not claimed_items:
|
||||
return 0
|
||||
task = asyncio.create_task(
|
||||
_run_enrichment(
|
||||
user_id=user_id,
|
||||
request_email=request_email,
|
||||
items=claimed_items,
|
||||
)
|
||||
queued_ids = await enqueue_missing_pdf_jobs(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
request_email=request_email,
|
||||
rows=items,
|
||||
max_items=max_items,
|
||||
)
|
||||
_scheduled_tasks.add(task)
|
||||
task.add_done_callback(_on_task_done)
|
||||
logger.info(
|
||||
"publications.enrichment.scheduled",
|
||||
extra={
|
||||
"event": "publications.enrichment.scheduled",
|
||||
"user_id": user_id,
|
||||
"publication_count": len(claimed_items),
|
||||
"publication_count": len(queued_ids),
|
||||
},
|
||||
)
|
||||
return len(claimed_items)
|
||||
return len(queued_ids)
|
||||
|
||||
|
||||
async def schedule_retry_pdf_enrichment_for_row(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
item: PublicationListItem,
|
||||
) -> bool:
|
||||
queued = await enqueue_retry_pdf_job(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
request_email=request_email,
|
||||
row=item,
|
||||
)
|
||||
logger.info(
|
||||
"publications.enrichment.retry_scheduled",
|
||||
extra={
|
||||
"event": "publications.enrichment.retry_scheduled",
|
||||
"user_id": user_id,
|
||||
"publication_id": item.publication_id,
|
||||
"queued": queued,
|
||||
},
|
||||
)
|
||||
return queued
|
||||
|
||||
|
||||
async def hydrate_pdf_enrichment_state(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
items: list[PublicationListItem],
|
||||
) -> list[PublicationListItem]:
|
||||
return await overlay_pdf_job_state(db_session, rows=items)
|
||||
|
|
|
|||
|
|
@ -16,94 +16,6 @@ from app.services.domains.publications.queries import (
|
|||
unread_item_from_row,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
from app.services.domains.unpaywall.application import resolve_publication_oa_metadata
|
||||
from sqlalchemy import update
|
||||
from app.db.models import Publication
|
||||
|
||||
|
||||
def _with_oa_overrides(
|
||||
rows: list[PublicationListItem],
|
||||
oa_data: dict[int, tuple[str | None, str | None]],
|
||||
) -> list[PublicationListItem]:
|
||||
return [
|
||||
PublicationListItem(
|
||||
publication_id=row.publication_id,
|
||||
scholar_profile_id=row.scholar_profile_id,
|
||||
scholar_label=row.scholar_label,
|
||||
title=row.title,
|
||||
year=row.year,
|
||||
citation_count=row.citation_count,
|
||||
venue_text=row.venue_text,
|
||||
pub_url=row.pub_url,
|
||||
doi=(oa_data.get(row.publication_id) or (None, None))[0] or row.doi,
|
||||
pdf_url=(oa_data.get(row.publication_id) or (None, None))[1] or row.pdf_url,
|
||||
is_read=row.is_read,
|
||||
first_seen_at=row.first_seen_at,
|
||||
is_new_in_latest_run=row.is_new_in_latest_run,
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def _resolved_fields(
|
||||
*,
|
||||
row: PublicationListItem,
|
||||
resolved: tuple[str | None, str | None] | None,
|
||||
) -> tuple[str | None, str | None]:
|
||||
if resolved is None:
|
||||
return row.doi, row.pdf_url
|
||||
resolved_doi, resolved_pdf = resolved
|
||||
return resolved_doi or row.doi, resolved_pdf or row.pdf_url
|
||||
|
||||
|
||||
async def _persist_resolved_metadata(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
rows: list[PublicationListItem],
|
||||
oa_data: dict[int, tuple[str | None, str | None]],
|
||||
) -> None:
|
||||
by_id = {row.publication_id: row for row in rows}
|
||||
updates: list[tuple[int, str | None, str | None]] = []
|
||||
for publication_id, resolved in oa_data.items():
|
||||
existing = by_id.get(publication_id)
|
||||
if existing is None:
|
||||
continue
|
||||
next_doi, next_pdf = _resolved_fields(row=existing, resolved=resolved)
|
||||
if next_doi == existing.doi and next_pdf == existing.pdf_url:
|
||||
continue
|
||||
updates.append((publication_id, next_doi, next_pdf))
|
||||
for publication_id, doi, pdf_url in updates:
|
||||
await db_session.execute(
|
||||
update(Publication)
|
||||
.where(Publication.id == publication_id)
|
||||
.values(doi=doi, pdf_url=pdf_url)
|
||||
)
|
||||
if updates:
|
||||
await db_session.commit()
|
||||
|
||||
|
||||
def missing_pdf_items(
|
||||
rows: list[PublicationListItem],
|
||||
*,
|
||||
limit: int,
|
||||
) -> list[PublicationListItem]:
|
||||
bounded_limit = max(0, int(limit))
|
||||
if bounded_limit == 0:
|
||||
return []
|
||||
return [row for row in rows if not row.pdf_url][:bounded_limit]
|
||||
|
||||
|
||||
async def resolve_and_persist_oa_metadata(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
rows: list[PublicationListItem],
|
||||
unpaywall_email: str | None = None,
|
||||
) -> dict[int, tuple[str | None, str | None]]:
|
||||
if not rows:
|
||||
return {}
|
||||
oa_data = await resolve_publication_oa_metadata(rows, request_email=unpaywall_email)
|
||||
await _persist_resolved_metadata(db_session, rows=rows, oa_data=oa_data)
|
||||
return oa_data
|
||||
|
||||
|
||||
async def list_for_user(
|
||||
|
|
@ -112,7 +24,9 @@ async def list_for_user(
|
|||
user_id: int,
|
||||
mode: str = MODE_ALL,
|
||||
scholar_profile_id: int | None = None,
|
||||
favorite_only: bool = False,
|
||||
limit: int = 300,
|
||||
offset: int = 0,
|
||||
) -> list[PublicationListItem]:
|
||||
resolved_mode = resolve_publication_view_mode(mode)
|
||||
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
|
||||
|
|
@ -122,7 +36,9 @@ async def list_for_user(
|
|||
mode=resolved_mode,
|
||||
latest_run_id=latest_run_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
favorite_only=favorite_only,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
)
|
||||
rows = [
|
||||
|
|
@ -138,22 +54,13 @@ async def retry_pdf_for_user(
|
|||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
publication_id: int,
|
||||
unpaywall_email: str | None = None,
|
||||
) -> PublicationListItem | None:
|
||||
item = await get_publication_item_for_user(
|
||||
return await get_publication_item_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
publication_id=publication_id,
|
||||
)
|
||||
if item is None:
|
||||
return None
|
||||
oa_data = await resolve_and_persist_oa_metadata(
|
||||
db_session,
|
||||
rows=[item],
|
||||
unpaywall_email=unpaywall_email,
|
||||
)
|
||||
return _with_oa_overrides([item], oa_data)[0]
|
||||
|
||||
|
||||
async def list_unread_for_user(
|
||||
|
|
@ -168,7 +75,9 @@ async def list_unread_for_user(
|
|||
mode=MODE_UNREAD,
|
||||
latest_run_id=None,
|
||||
scholar_profile_id=None,
|
||||
favorite_only=False,
|
||||
limit=limit,
|
||||
offset=0,
|
||||
)
|
||||
)
|
||||
return [unread_item_from_row(row) for row in result.all()]
|
||||
|
|
|
|||
877
app/services/domains/publications/pdf_queue.py
Normal file
877
app/services/domains/publications/pdf_queue.py
Normal file
|
|
@ -0,0 +1,877 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import logging
|
||||
|
||||
from sqlalchemy import Select, func, literal, or_, select, union_all
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
Publication,
|
||||
PublicationPdfJob,
|
||||
PublicationPdfJobEvent,
|
||||
ScholarProfile,
|
||||
ScholarPublication,
|
||||
User,
|
||||
)
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.domains.publications.types import PublicationListItem
|
||||
from app.services.domains.unpaywall.application import (
|
||||
FAILURE_RESOLUTION_EXCEPTION,
|
||||
OaResolutionOutcome,
|
||||
resolve_publication_oa_outcomes,
|
||||
)
|
||||
from app.settings import settings
|
||||
|
||||
PDF_STATUS_UNTRACKED = "untracked"
|
||||
PDF_STATUS_QUEUED = "queued"
|
||||
PDF_STATUS_RUNNING = "running"
|
||||
PDF_STATUS_RESOLVED = "resolved"
|
||||
PDF_STATUS_FAILED = "failed"
|
||||
|
||||
PDF_EVENT_QUEUED = "queued"
|
||||
PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
|
||||
PDF_EVENT_RESOLVED = "resolved"
|
||||
PDF_EVENT_FAILED = "failed"
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_scheduled_tasks: set[asyncio.Task[None]] = set()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PdfQueueListItem:
|
||||
publication_id: int
|
||||
title: str
|
||||
doi: str | None
|
||||
pdf_url: str | None
|
||||
status: str
|
||||
attempt_count: int
|
||||
last_failure_reason: str | None
|
||||
last_failure_detail: str | None
|
||||
last_source: str | None
|
||||
requested_by_user_id: int | None
|
||||
requested_by_email: str | None
|
||||
queued_at: datetime | None
|
||||
last_attempt_at: datetime | None
|
||||
resolved_at: datetime | None
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PdfRequeueResult:
|
||||
publication_exists: bool
|
||||
queued: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PdfBulkQueueResult:
|
||||
requested_count: int
|
||||
queued_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PdfQueuePage:
|
||||
items: list[PdfQueueListItem]
|
||||
total_count: int
|
||||
limit: int
|
||||
offset: int
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _publication_ids(rows: list[PublicationListItem]) -> list[int]:
|
||||
return sorted({row.publication_id for row in rows})
|
||||
|
||||
|
||||
def _status_from_job(row: PublicationListItem, job: PublicationPdfJob | None) -> str:
|
||||
if row.pdf_url:
|
||||
return PDF_STATUS_RESOLVED
|
||||
if job is None:
|
||||
return PDF_STATUS_UNTRACKED
|
||||
return job.status
|
||||
|
||||
|
||||
def _item_from_row_and_job(
|
||||
row: PublicationListItem,
|
||||
job: PublicationPdfJob | None,
|
||||
) -> PublicationListItem:
|
||||
return PublicationListItem(
|
||||
publication_id=row.publication_id,
|
||||
scholar_profile_id=row.scholar_profile_id,
|
||||
scholar_label=row.scholar_label,
|
||||
title=row.title,
|
||||
year=row.year,
|
||||
citation_count=row.citation_count,
|
||||
venue_text=row.venue_text,
|
||||
pub_url=row.pub_url,
|
||||
doi=row.doi,
|
||||
pdf_url=row.pdf_url,
|
||||
is_read=row.is_read,
|
||||
is_favorite=row.is_favorite,
|
||||
first_seen_at=row.first_seen_at,
|
||||
is_new_in_latest_run=row.is_new_in_latest_run,
|
||||
pdf_status=_status_from_job(row, job),
|
||||
pdf_attempt_count=int(job.attempt_count) if job is not None else 0,
|
||||
pdf_failure_reason=job.last_failure_reason if job is not None else None,
|
||||
pdf_failure_detail=job.last_failure_detail if job is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def _queueable_rows(
|
||||
rows: list[PublicationListItem],
|
||||
*,
|
||||
max_items: int,
|
||||
) -> list[PublicationListItem]:
|
||||
bounded = max(0, int(max_items))
|
||||
if bounded == 0:
|
||||
return []
|
||||
candidates = [row for row in rows if not row.pdf_url]
|
||||
return candidates[:bounded]
|
||||
|
||||
|
||||
def _bounded_limit(limit: int, *, max_value: int = 500) -> int:
|
||||
return max(1, min(int(limit), max_value))
|
||||
|
||||
|
||||
def _bounded_offset(offset: int) -> int:
|
||||
return max(int(offset), 0)
|
||||
|
||||
|
||||
def _auto_retry_interval_seconds() -> int:
|
||||
return max(int(settings.pdf_auto_retry_interval_seconds), 1)
|
||||
|
||||
|
||||
def _auto_retry_first_interval_seconds() -> int:
|
||||
return max(int(settings.pdf_auto_retry_first_interval_seconds), 1)
|
||||
|
||||
|
||||
def _auto_retry_max_attempts() -> int:
|
||||
return max(int(settings.pdf_auto_retry_max_attempts), 1)
|
||||
|
||||
|
||||
def _retry_interval_seconds_for_attempt_count(attempt_count: int) -> int:
|
||||
if int(attempt_count) <= 1:
|
||||
return _auto_retry_first_interval_seconds()
|
||||
return _auto_retry_interval_seconds()
|
||||
|
||||
|
||||
def _cooldown_active(
|
||||
*,
|
||||
last_attempt_at: datetime | None,
|
||||
attempt_count: int,
|
||||
) -> bool:
|
||||
if last_attempt_at is None:
|
||||
return False
|
||||
elapsed = (_utcnow() - last_attempt_at).total_seconds()
|
||||
return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count))
|
||||
|
||||
|
||||
def _can_enqueue_job(
|
||||
job: PublicationPdfJob | None,
|
||||
*,
|
||||
force_retry: bool,
|
||||
) -> bool:
|
||||
if job is None:
|
||||
return True
|
||||
if job.status in {PDF_STATUS_QUEUED, PDF_STATUS_RUNNING}:
|
||||
return False
|
||||
if force_retry:
|
||||
return job.status in {PDF_STATUS_FAILED, PDF_STATUS_RESOLVED, PDF_STATUS_UNTRACKED}
|
||||
if job.status == PDF_STATUS_RESOLVED:
|
||||
return False
|
||||
if int(job.attempt_count) >= _auto_retry_max_attempts():
|
||||
return False
|
||||
if _cooldown_active(
|
||||
last_attempt_at=job.last_attempt_at,
|
||||
attempt_count=int(job.attempt_count),
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _event_row(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int | None,
|
||||
event_type: str,
|
||||
status: str | None,
|
||||
source: str | None = None,
|
||||
failure_reason: str | None = None,
|
||||
message: str | None = None,
|
||||
) -> PublicationPdfJobEvent:
|
||||
return PublicationPdfJobEvent(
|
||||
publication_id=publication_id,
|
||||
user_id=user_id,
|
||||
event_type=event_type,
|
||||
status=status,
|
||||
source=source,
|
||||
failure_reason=failure_reason,
|
||||
message=message,
|
||||
)
|
||||
|
||||
|
||||
def _queued_job(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int,
|
||||
) -> PublicationPdfJob:
|
||||
now = _utcnow()
|
||||
return PublicationPdfJob(
|
||||
publication_id=publication_id,
|
||||
status=PDF_STATUS_QUEUED,
|
||||
queued_at=now,
|
||||
last_requested_by_user_id=user_id,
|
||||
)
|
||||
|
||||
|
||||
def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None:
|
||||
now = _utcnow()
|
||||
job.status = PDF_STATUS_QUEUED
|
||||
job.queued_at = now
|
||||
job.last_requested_by_user_id = user_id
|
||||
job.last_failure_reason = None
|
||||
job.last_failure_detail = None
|
||||
job.last_source = None
|
||||
|
||||
|
||||
def _state_map(jobs: list[PublicationPdfJob]) -> dict[int, PublicationPdfJob]:
|
||||
return {int(job.publication_id): job for job in jobs}
|
||||
|
||||
|
||||
async def _jobs_for_publication_ids(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication_ids: list[int],
|
||||
) -> dict[int, PublicationPdfJob]:
|
||||
if not publication_ids:
|
||||
return {}
|
||||
result = await db_session.execute(
|
||||
select(PublicationPdfJob).where(PublicationPdfJob.publication_id.in_(publication_ids))
|
||||
)
|
||||
return _state_map(list(result.scalars()))
|
||||
|
||||
|
||||
async def overlay_pdf_job_state(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
rows: list[PublicationListItem],
|
||||
) -> list[PublicationListItem]:
|
||||
if not rows:
|
||||
return []
|
||||
jobs = await _jobs_for_publication_ids(
|
||||
db_session,
|
||||
publication_ids=_publication_ids(rows),
|
||||
)
|
||||
return [_item_from_row_and_job(row, jobs.get(row.publication_id)) for row in rows]
|
||||
|
||||
|
||||
async def _enqueue_rows(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
rows: list[PublicationListItem],
|
||||
force_retry: bool,
|
||||
) -> list[PublicationListItem]:
|
||||
if not rows:
|
||||
return []
|
||||
queued: list[PublicationListItem] = []
|
||||
jobs = await _jobs_for_publication_ids(
|
||||
db_session,
|
||||
publication_ids=_publication_ids(rows),
|
||||
)
|
||||
for row in rows:
|
||||
job = jobs.get(row.publication_id)
|
||||
if not _can_enqueue_job(job, force_retry=force_retry):
|
||||
continue
|
||||
if job is None:
|
||||
job = _queued_job(publication_id=row.publication_id, user_id=user_id)
|
||||
jobs[row.publication_id] = job
|
||||
db_session.add(job)
|
||||
else:
|
||||
_mark_job_queued(job, user_id=user_id)
|
||||
db_session.add(
|
||||
_event_row(
|
||||
publication_id=row.publication_id,
|
||||
user_id=user_id,
|
||||
event_type=PDF_EVENT_QUEUED,
|
||||
status=PDF_STATUS_QUEUED,
|
||||
)
|
||||
)
|
||||
queued.append(row)
|
||||
if queued:
|
||||
await db_session.commit()
|
||||
return queued
|
||||
|
||||
|
||||
def _register_task(task: asyncio.Task[None]) -> None:
|
||||
_scheduled_tasks.add(task)
|
||||
|
||||
|
||||
def _drop_finished_task(task: asyncio.Task[None]) -> None:
|
||||
_scheduled_tasks.discard(task)
|
||||
try:
|
||||
task.result()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"publications.pdf_queue.task_failed",
|
||||
extra={"event": "publications.pdf_queue.task_failed"},
|
||||
)
|
||||
|
||||
|
||||
async def _mark_attempt_started(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as db_session:
|
||||
job = await db_session.get(PublicationPdfJob, publication_id)
|
||||
if job is None:
|
||||
job = _queued_job(publication_id=publication_id, user_id=user_id)
|
||||
db_session.add(job)
|
||||
job.status = PDF_STATUS_RUNNING
|
||||
job.last_attempt_at = _utcnow()
|
||||
job.attempt_count = int(job.attempt_count) + 1
|
||||
db_session.add(
|
||||
_event_row(
|
||||
publication_id=publication_id,
|
||||
user_id=user_id,
|
||||
event_type=PDF_EVENT_ATTEMPT_STARTED,
|
||||
status=PDF_STATUS_RUNNING,
|
||||
)
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
|
||||
def _failed_outcome(
|
||||
*,
|
||||
row: PublicationListItem,
|
||||
) -> OaResolutionOutcome:
|
||||
return OaResolutionOutcome(
|
||||
publication_id=row.publication_id,
|
||||
doi=row.doi,
|
||||
pdf_url=None,
|
||||
failure_reason=FAILURE_RESOLUTION_EXCEPTION,
|
||||
source=None,
|
||||
used_crossref=False,
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_outcome_for_row(
|
||||
*,
|
||||
row: PublicationListItem,
|
||||
request_email: str | None,
|
||||
) -> OaResolutionOutcome:
|
||||
outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email)
|
||||
outcome = outcomes.get(row.publication_id)
|
||||
if outcome is not None:
|
||||
return outcome
|
||||
return _failed_outcome(row=row)
|
||||
|
||||
|
||||
def _apply_publication_update(
|
||||
publication: Publication,
|
||||
*,
|
||||
doi: str | None,
|
||||
pdf_url: str | None,
|
||||
) -> None:
|
||||
if doi and publication.doi != doi:
|
||||
publication.doi = doi
|
||||
if pdf_url and publication.pdf_url != pdf_url:
|
||||
publication.pdf_url = pdf_url
|
||||
|
||||
|
||||
def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None:
|
||||
job.last_source = outcome.source
|
||||
if outcome.pdf_url:
|
||||
job.status = PDF_STATUS_RESOLVED
|
||||
job.resolved_at = _utcnow()
|
||||
job.last_failure_reason = None
|
||||
job.last_failure_detail = None
|
||||
return
|
||||
job.status = PDF_STATUS_FAILED
|
||||
job.last_failure_reason = outcome.failure_reason
|
||||
job.last_failure_detail = outcome.failure_reason
|
||||
|
||||
|
||||
def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]:
|
||||
if outcome.pdf_url:
|
||||
return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED
|
||||
return PDF_EVENT_FAILED, PDF_STATUS_FAILED
|
||||
|
||||
|
||||
async def _persist_outcome(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int,
|
||||
outcome: OaResolutionOutcome,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as db_session:
|
||||
publication = await db_session.get(Publication, publication_id)
|
||||
job = await db_session.get(PublicationPdfJob, publication_id)
|
||||
if publication is None or job is None:
|
||||
return
|
||||
_apply_publication_update(publication, doi=outcome.doi, pdf_url=outcome.pdf_url)
|
||||
_apply_job_outcome(job, outcome=outcome)
|
||||
event_type, status = _result_event(outcome)
|
||||
db_session.add(
|
||||
_event_row(
|
||||
publication_id=publication_id,
|
||||
user_id=user_id,
|
||||
event_type=event_type,
|
||||
status=status,
|
||||
source=outcome.source,
|
||||
failure_reason=outcome.failure_reason,
|
||||
message=outcome.failure_reason,
|
||||
)
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
|
||||
async def _resolve_publication_row(
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
row: PublicationListItem,
|
||||
) -> None:
|
||||
await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
|
||||
try:
|
||||
outcome = await _fetch_outcome_for_row(row=row, request_email=request_email)
|
||||
except Exception as exc: # pragma: no cover - defensive network boundary
|
||||
logger.warning(
|
||||
"publications.pdf_queue.resolve_failed",
|
||||
extra={
|
||||
"event": "publications.pdf_queue.resolve_failed",
|
||||
"publication_id": row.publication_id,
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
outcome = _failed_outcome(row=row)
|
||||
await _persist_outcome(
|
||||
publication_id=row.publication_id,
|
||||
user_id=user_id,
|
||||
outcome=outcome,
|
||||
)
|
||||
|
||||
|
||||
async def _run_resolution_task(
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
rows: list[PublicationListItem],
|
||||
) -> None:
|
||||
for row in rows:
|
||||
await _resolve_publication_row(
|
||||
user_id=user_id,
|
||||
request_email=request_email,
|
||||
row=row,
|
||||
)
|
||||
|
||||
|
||||
def _schedule_rows(
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
rows: list[PublicationListItem],
|
||||
) -> None:
|
||||
if not rows:
|
||||
return
|
||||
task = asyncio.create_task(
|
||||
_run_resolution_task(
|
||||
user_id=user_id,
|
||||
request_email=request_email,
|
||||
rows=rows,
|
||||
)
|
||||
)
|
||||
_register_task(task)
|
||||
task.add_done_callback(_drop_finished_task)
|
||||
|
||||
|
||||
async def enqueue_missing_pdf_jobs(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
rows: list[PublicationListItem],
|
||||
max_items: int,
|
||||
) -> list[int]:
|
||||
queueable = _queueable_rows(rows, max_items=max_items)
|
||||
queued_rows = await _enqueue_rows(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
rows=queueable,
|
||||
force_retry=False,
|
||||
)
|
||||
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
|
||||
return [row.publication_id for row in queued_rows]
|
||||
|
||||
|
||||
async def enqueue_retry_pdf_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
row: PublicationListItem,
|
||||
) -> bool:
|
||||
queued_rows = await _enqueue_rows(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
rows=[row],
|
||||
force_retry=True,
|
||||
)
|
||||
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
|
||||
return bool(queued_rows)
|
||||
|
||||
|
||||
def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str:
|
||||
return str(display_name or scholar_id or "unknown")
|
||||
|
||||
|
||||
def _retry_item_from_publication(
|
||||
publication: Publication,
|
||||
*,
|
||||
link_row: tuple | None,
|
||||
) -> PublicationListItem:
|
||||
if link_row is None:
|
||||
scholar_profile_id = 0
|
||||
scholar_label = "unknown"
|
||||
is_read = True
|
||||
first_seen_at = publication.created_at or _utcnow()
|
||||
else:
|
||||
scholar_profile_id = int(link_row[0])
|
||||
scholar_label = _retry_item_label(link_row[1], link_row[2])
|
||||
is_read = bool(link_row[3])
|
||||
first_seen_at = link_row[4] or publication.created_at or _utcnow()
|
||||
return PublicationListItem(
|
||||
publication_id=int(publication.id),
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
scholar_label=scholar_label,
|
||||
title=publication.title_raw,
|
||||
year=publication.year,
|
||||
citation_count=int(publication.citation_count or 0),
|
||||
venue_text=publication.venue_text,
|
||||
pub_url=publication.pub_url,
|
||||
doi=publication.doi,
|
||||
pdf_url=publication.pdf_url,
|
||||
is_read=is_read,
|
||||
first_seen_at=first_seen_at,
|
||||
is_new_in_latest_run=False,
|
||||
)
|
||||
|
||||
|
||||
async def _retry_item_for_publication_id(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication_id: int,
|
||||
) -> PublicationListItem | None:
|
||||
publication = await db_session.get(Publication, publication_id)
|
||||
if publication is None:
|
||||
return None
|
||||
result = await db_session.execute(
|
||||
select(
|
||||
ScholarProfile.id,
|
||||
ScholarProfile.display_name,
|
||||
ScholarProfile.scholar_id,
|
||||
ScholarPublication.is_read,
|
||||
ScholarPublication.created_at,
|
||||
)
|
||||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||
.where(ScholarPublication.publication_id == publication_id)
|
||||
.order_by(ScholarPublication.created_at.asc())
|
||||
.limit(1)
|
||||
)
|
||||
return _retry_item_from_publication(publication, link_row=result.one_or_none())
|
||||
|
||||
|
||||
async def enqueue_retry_pdf_job_for_publication_id(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
publication_id: int,
|
||||
) -> PdfRequeueResult:
|
||||
row = await _retry_item_for_publication_id(
|
||||
db_session,
|
||||
publication_id=publication_id,
|
||||
)
|
||||
if row is None:
|
||||
return PdfRequeueResult(publication_exists=False, queued=False)
|
||||
queued = await enqueue_retry_pdf_job(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
request_email=request_email,
|
||||
row=row,
|
||||
)
|
||||
return PdfRequeueResult(publication_exists=True, queued=queued)
|
||||
|
||||
|
||||
def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem:
|
||||
return PublicationListItem(
|
||||
publication_id=int(publication.id),
|
||||
scholar_profile_id=0,
|
||||
scholar_label="admin",
|
||||
title=publication.title_raw,
|
||||
year=publication.year,
|
||||
citation_count=int(publication.citation_count or 0),
|
||||
venue_text=publication.venue_text,
|
||||
pub_url=publication.pub_url,
|
||||
doi=publication.doi,
|
||||
pdf_url=publication.pdf_url,
|
||||
is_read=True,
|
||||
first_seen_at=publication.created_at or _utcnow(),
|
||||
is_new_in_latest_run=False,
|
||||
)
|
||||
|
||||
|
||||
async def _missing_pdf_candidates(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
limit: int,
|
||||
) -> list[PublicationListItem]:
|
||||
bounded_limit = max(1, min(int(limit), 5000))
|
||||
result = await db_session.execute(
|
||||
select(Publication)
|
||||
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
|
||||
.where(Publication.pdf_url.is_(None))
|
||||
.where(
|
||||
or_(
|
||||
PublicationPdfJob.publication_id.is_(None),
|
||||
PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
|
||||
)
|
||||
)
|
||||
.order_by(Publication.updated_at.desc(), Publication.id.desc())
|
||||
.limit(bounded_limit)
|
||||
)
|
||||
return [
|
||||
_queue_candidate_from_publication(publication)
|
||||
for publication in result.scalars()
|
||||
]
|
||||
|
||||
|
||||
async def enqueue_all_missing_pdf_jobs(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
limit: int = 1000,
|
||||
) -> PdfBulkQueueResult:
|
||||
candidates = await _missing_pdf_candidates(db_session, limit=limit)
|
||||
queued_rows = await _enqueue_rows(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
rows=candidates,
|
||||
force_retry=True,
|
||||
)
|
||||
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
|
||||
return PdfBulkQueueResult(
|
||||
requested_count=len(candidates),
|
||||
queued_count=len(queued_rows),
|
||||
)
|
||||
|
||||
|
||||
def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
|
||||
stmt = (
|
||||
select(
|
||||
PublicationPdfJob.publication_id,
|
||||
Publication.title_raw,
|
||||
Publication.doi,
|
||||
Publication.pdf_url,
|
||||
PublicationPdfJob.status,
|
||||
PublicationPdfJob.attempt_count,
|
||||
PublicationPdfJob.last_failure_reason,
|
||||
PublicationPdfJob.last_failure_detail,
|
||||
PublicationPdfJob.last_source,
|
||||
PublicationPdfJob.last_requested_by_user_id,
|
||||
User.email,
|
||||
PublicationPdfJob.queued_at,
|
||||
PublicationPdfJob.last_attempt_at,
|
||||
PublicationPdfJob.resolved_at,
|
||||
PublicationPdfJob.updated_at,
|
||||
)
|
||||
.join(Publication, Publication.id == PublicationPdfJob.publication_id)
|
||||
.outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id)
|
||||
)
|
||||
if status:
|
||||
stmt = stmt.where(PublicationPdfJob.status == status)
|
||||
return stmt
|
||||
|
||||
|
||||
def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]:
|
||||
return (
|
||||
_tracked_queue_select_base(status=status)
|
||||
.order_by(PublicationPdfJob.updated_at.desc())
|
||||
.limit(_bounded_limit(limit))
|
||||
.offset(_bounded_offset(offset))
|
||||
)
|
||||
|
||||
|
||||
def _untracked_queue_select_base() -> Select[tuple]:
|
||||
return (
|
||||
select(
|
||||
Publication.id,
|
||||
Publication.title_raw,
|
||||
Publication.doi,
|
||||
Publication.pdf_url,
|
||||
literal(PDF_STATUS_UNTRACKED),
|
||||
literal(0),
|
||||
literal(None),
|
||||
literal(None),
|
||||
literal(None),
|
||||
literal(None),
|
||||
literal(None),
|
||||
literal(None),
|
||||
literal(None),
|
||||
literal(None),
|
||||
Publication.updated_at,
|
||||
)
|
||||
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
|
||||
.where(Publication.pdf_url.is_(None))
|
||||
.where(PublicationPdfJob.publication_id.is_(None))
|
||||
)
|
||||
|
||||
|
||||
def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]:
|
||||
return (
|
||||
_untracked_queue_select_base()
|
||||
.order_by(Publication.updated_at.desc(), Publication.id.desc())
|
||||
.limit(_bounded_limit(limit))
|
||||
.offset(_bounded_offset(offset))
|
||||
)
|
||||
|
||||
|
||||
def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]:
|
||||
union_stmt = union_all(
|
||||
_tracked_queue_select_base(status=None),
|
||||
_untracked_queue_select_base(),
|
||||
).subquery()
|
||||
return (
|
||||
select(union_stmt)
|
||||
.order_by(union_stmt.c.updated_at.desc())
|
||||
.limit(_bounded_limit(limit))
|
||||
.offset(_bounded_offset(offset))
|
||||
)
|
||||
|
||||
|
||||
def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]:
|
||||
stmt = select(func.count()).select_from(PublicationPdfJob)
|
||||
if status:
|
||||
stmt = stmt.where(PublicationPdfJob.status == status)
|
||||
return stmt
|
||||
|
||||
|
||||
def _untracked_queue_count_select() -> Select[tuple]:
|
||||
return (
|
||||
select(func.count())
|
||||
.select_from(Publication)
|
||||
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
|
||||
.where(Publication.pdf_url.is_(None))
|
||||
.where(PublicationPdfJob.publication_id.is_(None))
|
||||
)
|
||||
|
||||
|
||||
def _queue_item_from_row(row: tuple) -> PdfQueueListItem:
|
||||
return PdfQueueListItem(
|
||||
publication_id=int(row[0]),
|
||||
title=str(row[1] or ""),
|
||||
doi=row[2],
|
||||
pdf_url=row[3],
|
||||
status=str(row[4] or PDF_STATUS_UNTRACKED),
|
||||
attempt_count=int(row[5] or 0),
|
||||
last_failure_reason=row[6],
|
||||
last_failure_detail=row[7],
|
||||
last_source=row[8],
|
||||
requested_by_user_id=int(row[9]) if row[9] is not None else None,
|
||||
requested_by_email=row[10],
|
||||
queued_at=row[11],
|
||||
last_attempt_at=row[12],
|
||||
resolved_at=row[13],
|
||||
updated_at=row[14],
|
||||
)
|
||||
|
||||
|
||||
async def list_pdf_queue_items(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
status: str | None = None,
|
||||
) -> list[PdfQueueListItem]:
|
||||
bounded_limit = _bounded_limit(limit)
|
||||
bounded_offset = _bounded_offset(offset)
|
||||
normalized_status = (status or "").strip().lower() or None
|
||||
if normalized_status == PDF_STATUS_UNTRACKED:
|
||||
result = await db_session.execute(
|
||||
_untracked_queue_select(
|
||||
limit=bounded_limit,
|
||||
offset=bounded_offset,
|
||||
)
|
||||
)
|
||||
return [_queue_item_from_row(row) for row in result.all()]
|
||||
if normalized_status is None:
|
||||
result = await db_session.execute(
|
||||
_all_queue_select(
|
||||
limit=bounded_limit,
|
||||
offset=bounded_offset,
|
||||
)
|
||||
)
|
||||
return [_queue_item_from_row(row) for row in result.all()]
|
||||
result = await db_session.execute(
|
||||
_tracked_queue_select(
|
||||
limit=bounded_limit,
|
||||
offset=bounded_offset,
|
||||
status=normalized_status,
|
||||
)
|
||||
)
|
||||
return [_queue_item_from_row(row) for row in result.all()]
|
||||
|
||||
|
||||
async def count_pdf_queue_items(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
status: str | None = None,
|
||||
) -> int:
|
||||
normalized_status = (status or "").strip().lower() or None
|
||||
if normalized_status == PDF_STATUS_UNTRACKED:
|
||||
result = await db_session.execute(_untracked_queue_count_select())
|
||||
return int(result.scalar_one() or 0)
|
||||
tracked_result = await db_session.execute(
|
||||
_tracked_queue_count_select(status=normalized_status)
|
||||
)
|
||||
tracked_count = int(tracked_result.scalar_one() or 0)
|
||||
if normalized_status is not None:
|
||||
return tracked_count
|
||||
untracked_result = await db_session.execute(_untracked_queue_count_select())
|
||||
untracked_count = int(untracked_result.scalar_one() or 0)
|
||||
return tracked_count + untracked_count
|
||||
|
||||
|
||||
async def list_pdf_queue_page(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
status: str | None = None,
|
||||
) -> PdfQueuePage:
|
||||
bounded_limit = _bounded_limit(limit)
|
||||
bounded_offset = _bounded_offset(offset)
|
||||
items = await list_pdf_queue_items(
|
||||
db_session,
|
||||
limit=bounded_limit,
|
||||
offset=bounded_offset,
|
||||
status=status,
|
||||
)
|
||||
total_count = await count_pdf_queue_items(
|
||||
db_session,
|
||||
status=status,
|
||||
)
|
||||
return PdfQueuePage(
|
||||
items=items,
|
||||
total_count=total_count,
|
||||
limit=bounded_limit,
|
||||
offset=bounded_offset,
|
||||
)
|
||||
|
|
@ -36,7 +36,9 @@ def publications_query(
|
|||
mode: str,
|
||||
latest_run_id: int | None,
|
||||
scholar_profile_id: int | None,
|
||||
favorite_only: bool,
|
||||
limit: int,
|
||||
offset: int = 0,
|
||||
) -> Select[tuple]:
|
||||
scholar_label = ScholarProfile.display_name
|
||||
stmt = (
|
||||
|
|
@ -53,6 +55,7 @@ def publications_query(
|
|||
Publication.doi,
|
||||
Publication.pdf_url,
|
||||
ScholarPublication.is_read,
|
||||
ScholarPublication.is_favorite,
|
||||
ScholarPublication.first_seen_run_id,
|
||||
ScholarPublication.created_at,
|
||||
)
|
||||
|
|
@ -60,10 +63,13 @@ def publications_query(
|
|||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
|
||||
.offset(max(int(offset), 0))
|
||||
.limit(limit)
|
||||
)
|
||||
if scholar_profile_id is not None:
|
||||
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
||||
if favorite_only:
|
||||
stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
|
||||
if mode == MODE_UNREAD:
|
||||
stmt = stmt.where(ScholarPublication.is_read.is_(False))
|
||||
if mode == MODE_LATEST:
|
||||
|
|
@ -93,6 +99,7 @@ def publication_query_for_user(
|
|||
Publication.doi,
|
||||
Publication.pdf_url,
|
||||
ScholarPublication.is_read,
|
||||
ScholarPublication.is_favorite,
|
||||
ScholarPublication.first_seen_run_id,
|
||||
ScholarPublication.created_at,
|
||||
)
|
||||
|
|
@ -146,6 +153,7 @@ def publication_list_item_from_row(
|
|||
doi,
|
||||
pdf_url,
|
||||
is_read,
|
||||
is_favorite,
|
||||
first_seen_run_id,
|
||||
created_at,
|
||||
) = row
|
||||
|
|
@ -161,6 +169,7 @@ def publication_list_item_from_row(
|
|||
doi=doi,
|
||||
pdf_url=pdf_url,
|
||||
is_read=bool(is_read),
|
||||
is_favorite=bool(is_favorite),
|
||||
first_seen_at=created_at,
|
||||
is_new_in_latest_run=(
|
||||
latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
|
||||
|
|
@ -182,6 +191,7 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
|
|||
doi,
|
||||
pdf_url,
|
||||
_is_read,
|
||||
_is_favorite,
|
||||
_first_seen_run_id,
|
||||
_created_at,
|
||||
) = row
|
||||
|
|
|
|||
|
|
@ -16,16 +16,20 @@ def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[
|
|||
return pairs
|
||||
|
||||
|
||||
def _scoped_scholar_ids_query(*, user_id: int):
|
||||
return (
|
||||
select(ScholarProfile.id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
|
||||
|
||||
async def mark_all_unread_as_read_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int:
|
||||
scholar_ids = (
|
||||
select(ScholarProfile.id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
|
||||
stmt = (
|
||||
update(ScholarPublication)
|
||||
.where(
|
||||
|
|
@ -49,11 +53,7 @@ async def mark_selected_as_read_for_user(
|
|||
if not normalized_pairs:
|
||||
return 0
|
||||
|
||||
scholar_ids = (
|
||||
select(ScholarProfile.id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
|
||||
stmt = (
|
||||
update(ScholarPublication)
|
||||
.where(
|
||||
|
|
@ -69,3 +69,26 @@ async def mark_selected_as_read_for_user(
|
|||
result = await db_session.execute(stmt)
|
||||
await db_session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
async def set_publication_favorite_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
publication_id: int,
|
||||
is_favorite: bool,
|
||||
) -> int:
|
||||
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
|
||||
stmt = (
|
||||
update(ScholarPublication)
|
||||
.where(
|
||||
ScholarPublication.scholar_profile_id.in_(scholar_ids),
|
||||
ScholarPublication.scholar_profile_id == int(scholar_profile_id),
|
||||
ScholarPublication.publication_id == int(publication_id),
|
||||
)
|
||||
.values(is_favorite=bool(is_favorite))
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
await db_session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,11 @@ class PublicationListItem:
|
|||
is_read: bool
|
||||
first_seen_at: datetime
|
||||
is_new_in_latest_run: bool
|
||||
is_favorite: bool = False
|
||||
pdf_status: str = "untracked"
|
||||
pdf_attempt_count: int = 0
|
||||
pdf_failure_reason: str | None = None
|
||||
pdf_failure_detail: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue