refactor: fix naming, remove duplicate code from decomposition
- Rename _int_or_default → int_or_default (public API, was using private convention) - Extract shared pdf_queue utilities (utcnow, event_row, queued_job, status constants) to pdf_queue_common.py to avoid circular imports - Consolidate _effective_request_delay_seconds into queue_runner.py with explicit floor parameter - Update tests to match new function locations and names Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
74249fa24c
commit
3fd9132176
9 changed files with 104 additions and 128 deletions
|
|
@ -2,16 +2,24 @@ from __future__ import annotations
|
|||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
PublicationPdfJob,
|
||||
PublicationPdfJobEvent,
|
||||
User,
|
||||
)
|
||||
from app.services.publications.pdf_queue_common import (
|
||||
PDF_STATUS_FAILED,
|
||||
PDF_STATUS_QUEUED,
|
||||
PDF_STATUS_RESOLVED,
|
||||
PDF_STATUS_RUNNING,
|
||||
event_row,
|
||||
queued_job,
|
||||
utcnow,
|
||||
)
|
||||
from app.services.publications.pdf_queue_queries import (
|
||||
missing_pdf_candidates,
|
||||
retry_item_for_publication_id,
|
||||
|
|
@ -21,10 +29,6 @@ from app.services.publications.types import PublicationListItem
|
|||
from app.settings import settings
|
||||
|
||||
PDF_STATUS_UNTRACKED = "untracked"
|
||||
PDF_STATUS_QUEUED = "queued"
|
||||
PDF_STATUS_RUNNING = "running"
|
||||
PDF_STATUS_RESOLVED = "resolved"
|
||||
PDF_STATUS_FAILED = "failed"
|
||||
|
||||
PDF_EVENT_QUEUED = "queued"
|
||||
|
||||
|
|
@ -43,10 +47,6 @@ class PdfBulkQueueResult:
|
|||
queued_count: int
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _publication_ids(rows: list[PublicationListItem]) -> list[int]:
|
||||
return sorted({row.publication_id for row in rows})
|
||||
|
||||
|
|
@ -122,7 +122,7 @@ def _cooldown_active(
|
|||
) -> bool:
|
||||
if last_attempt_at is None:
|
||||
return False
|
||||
elapsed = (_utcnow() - last_attempt_at).total_seconds()
|
||||
elapsed = (utcnow() - last_attempt_at).total_seconds()
|
||||
return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count))
|
||||
|
||||
|
||||
|
|
@ -147,37 +147,8 @@ def _can_enqueue_job(
|
|||
)
|
||||
|
||||
|
||||
def _event_row(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int | None,
|
||||
event_type: str,
|
||||
status: str | None,
|
||||
) -> PublicationPdfJobEvent:
|
||||
return PublicationPdfJobEvent(
|
||||
publication_id=publication_id,
|
||||
user_id=user_id,
|
||||
event_type=event_type,
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
def _queued_job(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int,
|
||||
) -> PublicationPdfJob:
|
||||
now = _utcnow()
|
||||
return PublicationPdfJob(
|
||||
publication_id=publication_id,
|
||||
status=PDF_STATUS_QUEUED,
|
||||
queued_at=now,
|
||||
last_requested_by_user_id=user_id,
|
||||
)
|
||||
|
||||
|
||||
def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None:
|
||||
now = _utcnow()
|
||||
now = utcnow()
|
||||
job.status = PDF_STATUS_QUEUED
|
||||
job.queued_at = now
|
||||
job.last_requested_by_user_id = user_id
|
||||
|
|
@ -236,13 +207,13 @@ async def _enqueue_rows(
|
|||
if not _can_enqueue_job(job, force_retry=force_retry):
|
||||
continue
|
||||
if job is None:
|
||||
job = _queued_job(publication_id=row.publication_id, user_id=user_id)
|
||||
job = queued_job(publication_id=row.publication_id, user_id=user_id)
|
||||
jobs[row.publication_id] = job
|
||||
db_session.add(job)
|
||||
else:
|
||||
_mark_job_queued(job, user_id=user_id)
|
||||
db_session.add(
|
||||
_event_row(
|
||||
event_row(
|
||||
publication_id=row.publication_id,
|
||||
user_id=user_id,
|
||||
event_type=PDF_EVENT_QUEUED,
|
||||
|
|
|
|||
49
app/services/publications/pdf_queue_common.py
Normal file
49
app/services/publications/pdf_queue_common.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.db.models import PublicationPdfJob, PublicationPdfJobEvent
|
||||
|
||||
PDF_STATUS_QUEUED = "queued"
|
||||
PDF_STATUS_RUNNING = "running"
|
||||
PDF_STATUS_RESOLVED = "resolved"
|
||||
PDF_STATUS_FAILED = "failed"
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def event_row(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int | None,
|
||||
event_type: str,
|
||||
status: str | None,
|
||||
source: str | None = None,
|
||||
failure_reason: str | None = None,
|
||||
message: str | None = None,
|
||||
) -> PublicationPdfJobEvent:
|
||||
return PublicationPdfJobEvent(
|
||||
publication_id=publication_id,
|
||||
user_id=user_id,
|
||||
event_type=event_type,
|
||||
status=status,
|
||||
source=source,
|
||||
failure_reason=failure_reason,
|
||||
message=message,
|
||||
)
|
||||
|
||||
|
||||
def queued_job(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int,
|
||||
) -> PublicationPdfJob:
|
||||
now = utcnow()
|
||||
return PublicationPdfJob(
|
||||
publication_id=publication_id,
|
||||
status=PDF_STATUS_QUEUED,
|
||||
queued_at=now,
|
||||
last_requested_by_user_id=user_id,
|
||||
)
|
||||
|
|
@ -3,10 +3,19 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import logging
|
||||
|
||||
from app.db.models import Publication, PublicationPdfJob, PublicationPdfJobEvent
|
||||
from app.db.models import Publication, PublicationPdfJob
|
||||
from app.db.session import get_session_factory
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.publication_identifiers import application as identifier_service
|
||||
from app.services.publications.pdf_queue_common import (
|
||||
PDF_STATUS_FAILED,
|
||||
PDF_STATUS_QUEUED,
|
||||
PDF_STATUS_RESOLVED,
|
||||
PDF_STATUS_RUNNING,
|
||||
event_row,
|
||||
queued_job,
|
||||
utcnow,
|
||||
)
|
||||
from app.services.publications.pdf_resolution_pipeline import (
|
||||
resolve_publication_pdf_outcome_for_row,
|
||||
)
|
||||
|
|
@ -17,11 +26,6 @@ from app.services.unpaywall.application import (
|
|||
)
|
||||
from app.settings import settings
|
||||
|
||||
PDF_STATUS_QUEUED = "queued"
|
||||
PDF_STATUS_RUNNING = "running"
|
||||
PDF_STATUS_RESOLVED = "resolved"
|
||||
PDF_STATUS_FAILED = "failed"
|
||||
|
||||
PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
|
||||
PDF_EVENT_RESOLVED = "resolved"
|
||||
PDF_EVENT_FAILED = "failed"
|
||||
|
|
@ -30,47 +34,6 @@ logger = logging.getLogger(__name__)
|
|||
_scheduled_tasks: set[asyncio.Task[None]] = set()
|
||||
|
||||
|
||||
def _utcnow():
|
||||
from datetime import UTC, datetime
|
||||
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _event_row(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int | None,
|
||||
event_type: str,
|
||||
status: str | None,
|
||||
source: str | None = None,
|
||||
failure_reason: str | None = None,
|
||||
message: str | None = None,
|
||||
) -> PublicationPdfJobEvent:
|
||||
return PublicationPdfJobEvent(
|
||||
publication_id=publication_id,
|
||||
user_id=user_id,
|
||||
event_type=event_type,
|
||||
status=status,
|
||||
source=source,
|
||||
failure_reason=failure_reason,
|
||||
message=message,
|
||||
)
|
||||
|
||||
|
||||
def _queued_job(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int,
|
||||
) -> PublicationPdfJob:
|
||||
now = _utcnow()
|
||||
return PublicationPdfJob(
|
||||
publication_id=publication_id,
|
||||
status=PDF_STATUS_QUEUED,
|
||||
queued_at=now,
|
||||
last_requested_by_user_id=user_id,
|
||||
)
|
||||
|
||||
|
||||
async def _mark_attempt_started(
|
||||
*,
|
||||
publication_id: int,
|
||||
|
|
@ -80,13 +43,13 @@ async def _mark_attempt_started(
|
|||
async with session_factory() as db_session:
|
||||
job = await db_session.get(PublicationPdfJob, publication_id)
|
||||
if job is None:
|
||||
job = _queued_job(publication_id=publication_id, user_id=user_id)
|
||||
job = queued_job(publication_id=publication_id, user_id=user_id)
|
||||
db_session.add(job)
|
||||
job.status = PDF_STATUS_RUNNING
|
||||
job.last_attempt_at = _utcnow()
|
||||
job.last_attempt_at = utcnow()
|
||||
job.attempt_count = int(job.attempt_count) + 1
|
||||
db_session.add(
|
||||
_event_row(
|
||||
event_row(
|
||||
publication_id=publication_id,
|
||||
user_id=user_id,
|
||||
event_type=PDF_EVENT_ATTEMPT_STARTED,
|
||||
|
|
@ -142,7 +105,7 @@ def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome)
|
|||
job.last_source = outcome.source
|
||||
if outcome.pdf_url:
|
||||
job.status = PDF_STATUS_RESOLVED
|
||||
job.resolved_at = _utcnow()
|
||||
job.resolved_at = utcnow()
|
||||
job.last_failure_reason = None
|
||||
job.last_failure_detail = None
|
||||
return
|
||||
|
|
@ -178,7 +141,7 @@ async def _persist_outcome(
|
|||
_apply_job_outcome(job, outcome=outcome)
|
||||
event_type, status = _result_event(outcome)
|
||||
db_session.add(
|
||||
_event_row(
|
||||
event_row(
|
||||
publication_id=publication_id,
|
||||
user_id=user_id,
|
||||
event_type=event_type,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue