ci: add ruff linting and mypy type checking
Add ruff and mypy to dev dependencies with configuration in pyproject.toml. Add a lint CI job that runs ruff check, ruff format --check, and mypy. Auto-fix import sorting and formatting across the codebase. Exclude alembic/versions from linting (auto-generated migrations). Ignore B008 (FastAPI Depends pattern) and RUF001 (unicode in user-facing strings). 21 ruff lint errors and 50 mypy errors remain for manual review. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
399276ea69
commit
bf04c77aa9
137 changed files with 3066 additions and 1900 deletions
|
|
@ -1,9 +1,9 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import Select, and_, func, literal, or_, select, union_all
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
|
@ -17,6 +17,7 @@ from app.db.models import (
|
|||
User,
|
||||
)
|
||||
from app.db.session import get_session_factory
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.publication_identifiers import application as identifier_service
|
||||
from app.services.domains.publication_identifiers.types import DisplayIdentifier
|
||||
from app.services.domains.publications.pdf_resolution_pipeline import (
|
||||
|
|
@ -27,7 +28,6 @@ from app.services.domains.unpaywall.application import (
|
|||
FAILURE_RESOLUTION_EXCEPTION,
|
||||
OaResolutionOutcome,
|
||||
)
|
||||
from app.logging_utils import structured_log
|
||||
from app.settings import settings
|
||||
|
||||
PDF_STATUS_UNTRACKED = "untracked"
|
||||
|
|
@ -85,7 +85,7 @@ class PdfQueuePage:
|
|||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _publication_ids(rows: list[PublicationListItem]) -> list[int]:
|
||||
|
|
@ -472,7 +472,13 @@ async def _resolve_publication_row(
|
|||
# Propagate upward so the batch loop can stop immediately.
|
||||
raise
|
||||
except Exception as exc: # pragma: no cover - defensive network boundary
|
||||
structured_log(logger, "warning", "publications.pdf_queue.resolve_failed", publication_id=row.publication_id, error=str(exc))
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"publications.pdf_queue.resolve_failed",
|
||||
publication_id=row.publication_id,
|
||||
error=str(exc),
|
||||
)
|
||||
outcome = _failed_outcome(row=row)
|
||||
arxiv_rate_limited = False
|
||||
await _persist_outcome(
|
||||
|
|
@ -514,9 +520,19 @@ async def _run_resolution_task(
|
|||
)
|
||||
if arxiv_rate_limited and arxiv_lookup_allowed:
|
||||
arxiv_lookup_allowed = False
|
||||
structured_log(logger, "warning", "pdf_queue.arxiv_batch_disabled", detail="arXiv temporarily disabled for remaining batch after rate limit")
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"pdf_queue.arxiv_batch_disabled",
|
||||
detail="arXiv temporarily disabled for remaining batch after rate limit",
|
||||
)
|
||||
except OpenAlexBudgetExhaustedError:
|
||||
structured_log(logger, "warning", "pdf_queue.budget_exhausted", detail="Stopping PDF resolution batch — OpenAlex daily budget exhausted")
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"pdf_queue.budget_exhausted",
|
||||
detail="Stopping PDF resolution batch — OpenAlex daily budget exhausted",
|
||||
)
|
||||
break
|
||||
|
||||
|
||||
|
|
@ -679,7 +695,7 @@ async def _missing_pdf_candidates(
|
|||
limit: int,
|
||||
) -> list[PublicationListItem]:
|
||||
bounded_limit = max(1, min(int(limit), 5000))
|
||||
now = datetime.now(timezone.utc)
|
||||
now = datetime.now(UTC)
|
||||
cooldown_threshold = now - timedelta(days=7)
|
||||
|
||||
result = await db_session.execute(
|
||||
|
|
@ -701,10 +717,7 @@ async def _missing_pdf_candidates(
|
|||
.order_by(Publication.updated_at.desc(), Publication.id.desc())
|
||||
.limit(bounded_limit)
|
||||
)
|
||||
return [
|
||||
_queue_candidate_from_publication(publication)
|
||||
for publication in result.scalars()
|
||||
]
|
||||
return [_queue_candidate_from_publication(publication) for publication in result.scalars()]
|
||||
|
||||
|
||||
async def enqueue_all_missing_pdf_jobs(
|
||||
|
|
@ -902,9 +915,7 @@ async def count_pdf_queue_items(
|
|||
if normalized_status == PDF_STATUS_UNTRACKED:
|
||||
result = await db_session.execute(_untracked_queue_count_select())
|
||||
return int(result.scalar_one() or 0)
|
||||
tracked_result = await db_session.execute(
|
||||
_tracked_queue_count_select(status=normalized_status)
|
||||
)
|
||||
tracked_result = await db_session.execute(_tracked_queue_count_select(status=normalized_status))
|
||||
tracked_count = int(tracked_result.scalar_one() or 0)
|
||||
if normalized_status is not None:
|
||||
return tracked_count
|
||||
|
|
@ -946,9 +957,7 @@ async def drain_ready_jobs(
|
|||
limit: int,
|
||||
max_attempts: int,
|
||||
) -> int:
|
||||
result = await db_session.execute(
|
||||
select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1)
|
||||
)
|
||||
result = await db_session.execute(select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1))
|
||||
system_user_id = result.scalar_one_or_none()
|
||||
if system_user_id is None:
|
||||
return 0
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue