From 1c01b29be7b374d72a5d7b1dc811109095a113dc Mon Sep 17 00:00:00 2001 From: Justin Visser Date: Sun, 22 Feb 2026 16:23:35 +0100 Subject: [PATCH 01/38] s --- agents.md | 8 +- ...260222_0016_add_publication_identifiers.py | 104 ++++++ app/api/routers/admin_dbops.py | 13 + app/api/routers/publications.py | 21 +- app/api/schemas.py | 12 + app/db/models.py | 48 +++ app/services/domains/crossref/application.py | 127 ++++++- app/services/domains/ingestion/application.py | 12 +- .../publication_identifiers/__init__.py | 17 + .../publication_identifiers/application.py | 351 ++++++++++++++++++ .../publication_identifiers/normalize.py | 76 ++++ .../domains/publication_identifiers/types.py | 30 ++ app/services/domains/publications/listing.py | 15 +- .../domains/publications/pdf_queue.py | 39 +- .../publications/pdf_resolution_pipeline.py | 150 ++++++++ app/services/domains/publications/types.py | 3 + .../domains/scholar/publication_pdf.py | 232 ++++++++++++ app/services/domains/scholar/rate_limit.py | 18 + app/services/domains/scholar/source.py | 10 + app/services/domains/unpaywall/application.py | 36 +- frontend/src/components/ui/AppHelpHint.vue | 4 +- frontend/src/features/admin_dbops/index.ts | 9 + frontend/src/features/publications/index.ts | 9 + .../features/settings/SettingsAdminPanel.vue | 10 +- frontend/src/pages/PublicationsPage.vue | 18 +- tests/unit/test_crossref_lookup.py | 32 +- tests/unit/test_publication_identifiers.py | 38 ++ .../unit/test_publication_pdf_queue_policy.py | 63 ++++ ...est_publication_pdf_resolution_pipeline.py | 153 ++++++++ tests/unit/test_scholar_publication_pdf.py | 76 ++++ tests/unit/test_unpaywall_resolution.py | 44 ++- 31 files changed, 1725 insertions(+), 53 deletions(-) create mode 100644 alembic/versions/20260222_0016_add_publication_identifiers.py create mode 100644 app/services/domains/publication_identifiers/__init__.py create mode 100644 app/services/domains/publication_identifiers/application.py create mode 100644 app/services/domains/publication_identifiers/normalize.py create mode 100644 app/services/domains/publication_identifiers/types.py create mode 100644 app/services/domains/publications/pdf_resolution_pipeline.py create mode 100644 app/services/domains/scholar/publication_pdf.py create mode 100644 app/services/domains/scholar/rate_limit.py create mode 100644 tests/unit/test_publication_identifiers.py create mode 100644 tests/unit/test_publication_pdf_resolution_pipeline.py create mode 100644 tests/unit/test_scholar_publication_pdf.py diff --git a/agents.md b/agents.md index 07b62d2..da59b76 100644 --- a/agents.md +++ b/agents.md @@ -7,6 +7,7 @@ Adhere strictly to these constraints. * **DRY (Don't Repeat Yourself):** Abstract repetitive logic immediately. No duplicate boilerplate for database queries, API responses, or error handling. * **Negative Space Programming:** Utilize explicit assertions and constraints to define invalid states. Fail fast and early. Do not allow silent failures or cascading malformed data, especially in DOM parsing. * **Cyclomatic Complexity:** Flatten logic. Use early returns and guard clauses instead of deep nesting. +no magic numbers ## 2. Domain Architecture & Data Model * **Data Isolation:** Scholar tracking is **user-scoped**. Validate mapping/join tables; never assume global links between users and Scholar IDs. @@ -29,13 +30,6 @@ These limits prevent IP bans and are not to be optimized away. ## 5. Domain Service Boundaries * **Strict Modularity:** Flat files in the `app/services/` root are strictly prohibited. All business logic and routing must reside exclusively within `app/services/domains/`. -* **`app/services/domains/scholar/*`:** Parser contract is fail-fast. Layout drift must emit explicit exceptions and `layout_*` reasons/warnings. Never allow silent partial success. -* **`app/services/domains/ingestion/application.py`:** Orchestrates ingestion runs; validate parser outputs before persistence; enforce publication candidate constraints before upsert. -* **`app/services/domains/publications/*`:** Publication list/read-state query layer. Includes `doi` + `pdf_url` fields for UI consumption, enforces non-blocking lazy OA enrichment scheduling on list reads, and exposes per-publication PDF retry behavior. -* **`app/services/domains/crossref/*`:** DOI discovery fallback module. Must use bounded/paced lookups to avoid burst traffic and 429 responses. -* **`app/services/domains/unpaywall/*`:** OA resolver by DOI only (best OA location + PDF URL extraction). Do not use Google Scholar for PDF resolution to avoid N+1 scrape amplification. -* **`app/services/domains/portability/*`:** Handles JSON import/export for user-scoped scholars and scholar-publication link state while preserving global publication dedup rules. -* **`app/services/domains/ingestion/scheduler.py`:** Owns automatic runs and continuation queue retries/drops; do not bypass safety gate or cooldown logic. ## 6. UI rules diff --git a/alembic/versions/20260222_0016_add_publication_identifiers.py b/alembic/versions/20260222_0016_add_publication_identifiers.py new file mode 100644 index 0000000..cd824a2 --- /dev/null +++ b/alembic/versions/20260222_0016_add_publication_identifiers.py @@ -0,0 +1,104 @@ +"""Add publication identifiers table. + +Revision ID: 20260222_0016 +Revises: 20260221_0015 +Create Date: 2026-02-22 12:45:00.000000 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +revision: str = "20260222_0016" +down_revision: str | Sequence[str] | None = "20260221_0015" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _table_names() -> set[str]: + bind = op.get_bind() + inspector = sa.inspect(bind) + return set(inspector.get_table_names()) + + +def _create_publication_identifiers_table() -> None: + op.create_table( + "publication_identifiers", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("publication_id", sa.Integer(), nullable=False), + sa.Column("kind", sa.String(length=32), nullable=False), + sa.Column("value_raw", sa.Text(), nullable=False), + sa.Column("value_normalized", sa.String(length=255), nullable=False), + sa.Column("source", sa.String(length=64), nullable=False), + sa.Column( + "confidence_score", + sa.Float(), + nullable=False, + server_default=sa.text("0"), + ), + sa.Column("evidence_url", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.CheckConstraint( + "confidence_score >= 0 AND confidence_score <= 1", + name="publication_identifiers_confidence_score_range", + ), + sa.ForeignKeyConstraint( + ["publication_id"], + ["publications.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_publication_identifiers")), + sa.UniqueConstraint( + "publication_id", + "kind", + "value_normalized", + name="uq_publication_identifiers_publication_kind_value", + ), + ) + + +def _create_publication_identifiers_indexes() -> None: + op.create_index( + "ix_publication_identifiers_kind_value", + "publication_identifiers", + ["kind", "value_normalized"], + unique=False, + ) + op.create_index( + "ix_publication_identifiers_publication_id", + "publication_identifiers", + ["publication_id"], + unique=False, + ) + + +def _drop_publication_identifiers_indexes() -> None: + op.drop_index("ix_publication_identifiers_publication_id", table_name="publication_identifiers") + op.drop_index("ix_publication_identifiers_kind_value", table_name="publication_identifiers") + + +def upgrade() -> None: + if "publication_identifiers" in _table_names(): + return + _create_publication_identifiers_table() + _create_publication_identifiers_indexes() + + +def downgrade() -> None: + if "publication_identifiers" not in _table_names(): + return + _drop_publication_identifiers_indexes() + op.drop_table("publication_identifiers") diff --git a/app/api/routers/admin_dbops.py b/app/api/routers/admin_dbops.py index 16271a0..cd6d924 100644 --- a/app/api/routers/admin_dbops.py +++ b/app/api/routers/admin_dbops.py @@ -58,6 +58,7 @@ def _serialize_pdf_queue_item(item) -> dict[str, object]: "publication_id": item.publication_id, "title": item.title, "doi": item.doi, + "display_identifier": _serialize_display_identifier(item.display_identifier), "pdf_url": item.pdf_url, "status": item.status, "attempt_count": item.attempt_count, @@ -73,6 +74,18 @@ def _serialize_pdf_queue_item(item) -> dict[str, object]: } +def _serialize_display_identifier(value) -> dict[str, object] | None: + if value is None: + return None + return { + "kind": value.kind, + "value": value.value, + "label": value.label, + "url": value.url, + "confidence_score": float(value.confidence_score), + } + + def _requeue_response_state(*, queued: bool) -> tuple[str, str]: if queued: return "queued", "PDF lookup queued." diff --git a/app/api/routers/publications.py b/app/api/routers/publications.py index 5055c6c..ad8fc55 100644 --- a/app/api/routers/publications.py +++ b/app/api/routers/publications.py @@ -21,6 +21,7 @@ from app.api.schemas import ( ) from app.db.models import User from app.db.session import get_db_session +from app.services.domains.publication_identifiers import application as identifier_service from app.services.domains.publications import application as publication_service from app.services.domains.scholars import application as scholar_service from app.settings import settings @@ -62,6 +63,7 @@ def _serialize_publication_item(item) -> dict[str, object]: "venue_text": item.venue_text, "pub_url": item.pub_url, "doi": item.doi, + "display_identifier": _serialize_display_identifier(item.display_identifier), "pdf_url": item.pdf_url, "pdf_status": item.pdf_status, "pdf_attempt_count": item.pdf_attempt_count, @@ -74,6 +76,18 @@ def _serialize_publication_item(item) -> dict[str, object]: } +def _serialize_display_identifier(value) -> dict[str, object] | None: + if value is None: + return None + return { + "kind": value.kind, + "value": value.value, + "label": value.label, + "url": value.url, + "confidence_score": float(value.confidence_score), + } + + async def _publication_counts( db_session: AsyncSession, *, @@ -280,7 +294,12 @@ async def _favorite_publication_state( db_session, items=[publication], ) - return hydrated[0] if hydrated else publication + current = hydrated[0] if hydrated else publication + identifiers = await identifier_service.overlay_publication_items_with_display_identifiers( + db_session, + items=[current], + ) + return identifiers[0] if identifiers else current def _log_retry_pdf_result( diff --git a/app/api/schemas.py b/app/api/schemas.py index 3ae8876..112f672 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -581,10 +581,21 @@ class AdminDbRepairJobsEnvelope(BaseModel): model_config = ConfigDict(extra="forbid") +class DisplayIdentifierData(BaseModel): + kind: str + value: str + label: str + url: str | None + confidence_score: float = Field(ge=0.0, le=1.0) + + model_config = ConfigDict(extra="forbid") + + class AdminPdfQueueItemData(BaseModel): publication_id: int title: str doi: str | None + display_identifier: DisplayIdentifierData | None = None pdf_url: str | None status: str attempt_count: int @@ -744,6 +755,7 @@ class PublicationItemData(BaseModel): venue_text: str | None pub_url: str | None doi: str | None + display_identifier: DisplayIdentifierData | None = None pdf_url: str | None pdf_status: str = "untracked" pdf_attempt_count: int = 0 diff --git a/app/db/models.py b/app/db/models.py index 43b44c2..1f0b250 100644 --- a/app/db/models.py +++ b/app/db/models.py @@ -8,6 +8,7 @@ from sqlalchemy import ( CheckConstraint, DateTime, Enum, + Float, ForeignKey, Index, Integer, @@ -237,6 +238,53 @@ class Publication(Base): ) +class PublicationIdentifier(Base): + __tablename__ = "publication_identifiers" + __table_args__ = ( + UniqueConstraint( + "publication_id", + "kind", + "value_normalized", + name="uq_publication_identifiers_publication_kind_value", + ), + CheckConstraint( + "confidence_score >= 0 AND confidence_score <= 1", + name="publication_identifiers_confidence_score_range", + ), + Index( + "ix_publication_identifiers_kind_value", + "kind", + "value_normalized", + ), + Index( + "ix_publication_identifiers_publication_id", + "publication_id", + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + publication_id: Mapped[int] = mapped_column( + ForeignKey("publications.id", ondelete="CASCADE"), + nullable=False, + ) + kind: Mapped[str] = mapped_column(String(32), nullable=False) + value_raw: Mapped[str] = mapped_column(Text, nullable=False) + value_normalized: Mapped[str] = mapped_column(String(255), nullable=False) + source: Mapped[str] = mapped_column(String(64), nullable=False) + confidence_score: Mapped[float] = mapped_column( + Float, + nullable=False, + server_default=text("0"), + ) + evidence_url: Mapped[str | None] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + class PublicationPdfJob(Base): __tablename__ = "publication_pdf_jobs" __table_args__ = ( diff --git a/app/services/domains/crossref/application.py b/app/services/domains/crossref/application.py index 569c12d..f0e1311 100644 --- a/app/services/domains/crossref/application.py +++ b/app/services/domains/crossref/application.py @@ -21,6 +21,8 @@ STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis" _RATE_LOCK = threading.Lock() _LAST_REQUEST_AT = 0.0 logger = logging.getLogger(__name__) +STRICT_TITLE_MATCH_THRESHOLD = 0.75 +RELAXED_TITLE_MATCH_THRESHOLD = 0.85 def _rate_limit_wait(min_interval_seconds: float) -> None: @@ -132,7 +134,42 @@ def _candidate_rank(*, title: str, year: int | None, item: dict) -> tuple[float, return score, doi -def _best_candidate_doi( +def _year_delta(source_year: int | None, candidate_year: int | None) -> int | None: + if source_year is None or candidate_year is None: + return None + return abs(int(source_year) - int(candidate_year)) + + +def _candidate_rank_relaxed( + *, + title: str, + year: int | None, + item: dict, + author_surname: str | None, +) -> tuple[float, str | None]: + doi = normalize_doi(str(item.get("DOI") or "")) + if doi is None: + return 0.0, None + score = _title_match_score(title, _candidate_title(item)) + if score <= 0: + return 0.0, None + candidate_year = _candidate_year(item) + delta = _year_delta(year, candidate_year) + if delta is not None: + if delta <= 1: + score += 0.05 + elif delta <= 3: + score += 0.0 + elif delta <= 5: + score -= 0.03 + else: + score -= 0.08 + if _candidate_author_match(item, author_surname): + score += 0.03 + return score, doi + + +def _best_candidate_doi_strict( *, title: str, year: int | None, @@ -149,7 +186,7 @@ def _best_candidate_doi( continue score, doi = _candidate_rank(title=title, year=year, item=item) candidate_year = _candidate_year(item) - if doi is None or score < 0.75: + if doi is None or score < STRICT_TITLE_MATCH_THRESHOLD: continue if score > best_score: best_score = score @@ -166,6 +203,92 @@ def _best_candidate_doi( return best_doi +def _best_candidate_doi_relaxed( + *, + title: str, + year: int | None, + items: list[dict], + author_surname: str | None, +) -> str | None: + best_score = 0.0 + best_doi: str | None = None + best_author_match = False + best_delta: int | None = None + best_year: int | None = None + for item in items: + if not isinstance(item, dict): + continue + score, doi = _candidate_rank_relaxed( + title=title, + year=year, + item=item, + author_surname=author_surname, + ) + if doi is None or score < RELAXED_TITLE_MATCH_THRESHOLD: + continue + candidate_year = _candidate_year(item) + candidate_author_match = _candidate_author_match(item, author_surname) + candidate_delta = _year_delta(year, candidate_year) + if score > best_score: + best_score = score + best_doi = doi + best_author_match = candidate_author_match + best_delta = candidate_delta + best_year = candidate_year + continue + if abs(score - best_score) > 0.02: + continue + if candidate_author_match and not best_author_match: + best_doi = doi + best_author_match = True + best_delta = candidate_delta + best_year = candidate_year + continue + if best_delta is None and candidate_delta is not None: + best_doi = doi + best_author_match = candidate_author_match + best_delta = candidate_delta + best_year = candidate_year + continue + if best_delta is not None and candidate_delta is not None and candidate_delta < best_delta: + best_doi = doi + best_author_match = candidate_author_match + best_delta = candidate_delta + best_year = candidate_year + continue + if best_year is None or candidate_year is None: + continue + if candidate_year < best_year: + best_doi = doi + best_author_match = candidate_author_match + best_delta = candidate_delta + best_year = candidate_year + return best_doi + + +def _best_candidate_doi( + *, + title: str, + year: int | None, + items: list[dict], + author_surname: str | None, +) -> str | None: + strict_match = _best_candidate_doi_strict( + title=title, + year=year, + items=items, + author_surname=author_surname, + ) + if strict_match: + return strict_match + return _best_candidate_doi_relaxed( + title=title, + year=year, + items=items, + author_surname=author_surname, + ) + + def _works_client(email: str | None) -> Works: if email: etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email) diff --git a/app/services/domains/ingestion/application.py b/app/services/domains/ingestion/application.py index 54a7e25..4cb493d 100644 --- a/app/services/domains/ingestion/application.py +++ b/app/services/domains/ingestion/application.py @@ -29,6 +29,7 @@ from app.services.domains.ingestion.constants import ( RUN_LOCK_NAMESPACE, ) from app.services.domains.doi.normalize import first_doi_from_texts +from app.services.domains.publication_identifiers import application as identifier_service from app.services.domains.ingestion.fingerprints import ( _build_body_excerpt, _dedupe_publication_candidates, @@ -2071,15 +2072,24 @@ class ScholarIngestionService: fingerprint_publication=fingerprint_publication, ) if publication is None: - return await self._create_publication( + created = await self._create_publication( db_session, candidate=candidate, fingerprint=fingerprint, ) + await identifier_service.sync_identifiers_for_publication_fields( + db_session, + publication=created, + ) + return created self._update_existing_publication( publication=publication, candidate=candidate, ) + await identifier_service.sync_identifiers_for_publication_fields( + db_session, + publication=publication, + ) return publication def _resolve_run_status( diff --git a/app/services/domains/publication_identifiers/__init__.py b/app/services/domains/publication_identifiers/__init__.py new file mode 100644 index 0000000..3a35c66 --- /dev/null +++ b/app/services/domains/publication_identifiers/__init__.py @@ -0,0 +1,17 @@ +from app.services.domains.publication_identifiers.application import ( + DisplayIdentifier, + derive_display_identifier_from_values, + overlay_pdf_queue_items_with_display_identifiers, + overlay_publication_items_with_display_identifiers, + sync_identifiers_for_publication_fields, + sync_identifiers_for_publication_resolution, +) + +__all__ = [ + "DisplayIdentifier", + "derive_display_identifier_from_values", + "overlay_pdf_queue_items_with_display_identifiers", + "overlay_publication_items_with_display_identifiers", + "sync_identifiers_for_publication_fields", + "sync_identifiers_for_publication_resolution", +] diff --git a/app/services/domains/publication_identifiers/application.py b/app/services/domains/publication_identifiers/application.py new file mode 100644 index 0000000..d34e757 --- /dev/null +++ b/app/services/domains/publication_identifiers/application.py @@ -0,0 +1,351 @@ +from __future__ import annotations + +from dataclasses import replace +from typing import TYPE_CHECKING + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Publication, PublicationIdentifier +from app.services.domains.doi.normalize import normalize_doi +from app.services.domains.publication_identifiers.normalize import ( + normalize_arxiv_id, + normalize_pmcid, + normalize_pmid, +) +from app.services.domains.publication_identifiers.types import ( + DisplayIdentifier, + IdentifierCandidate, + IdentifierKind, +) + +if TYPE_CHECKING: + from app.services.domains.publications.pdf_queue import PdfQueueListItem + from app.services.domains.publications.types import PublicationListItem + +CONFIDENCE_HIGH = 0.98 +CONFIDENCE_MEDIUM = 0.9 +CONFIDENCE_LOW = 0.6 +CONFIDENCE_FALLBACK = 0.4 +PRIORITY_DOI = 400 +PRIORITY_ARXIV = 300 +PRIORITY_PMCID = 200 +PRIORITY_PMID = 100 + + +def derive_display_identifier_from_values( + *, + doi: str | None, + pub_url: str | None = None, + pdf_url: str | None = None, +) -> DisplayIdentifier | None: + candidates = _fallback_candidates_from_values(doi=doi, pub_url=pub_url, pdf_url=pdf_url) + return _best_display_identifier(candidates) + + +def _fallback_candidates_from_values( + *, + doi: str | None, + pub_url: str | None, + pdf_url: str | None, +) -> list[IdentifierCandidate]: + values = [value for value in [pub_url, pdf_url] if value] + candidates = [] + if doi: + normalized_doi = normalize_doi(doi) + if normalized_doi: + candidates.append(_candidate(IdentifierKind.DOI, doi, normalized_doi, "legacy_doi", CONFIDENCE_HIGH, pub_url)) + candidates.extend(_url_identifier_candidates(values=values, source="legacy_urls")) + return _dedup_candidates(candidates) + + +def _url_identifier_candidates(*, values: list[str], source: str) -> list[IdentifierCandidate]: + candidates: list[IdentifierCandidate] = [] + for value in values: + candidates.extend(_url_candidates_for_value(value=value, source=source)) + return candidates + + +def _url_candidates_for_value(*, value: str, source: str) -> list[IdentifierCandidate]: + candidates: list[IdentifierCandidate] = [] + arxiv = normalize_arxiv_id(value) + if arxiv: + candidates.append(_candidate(IdentifierKind.ARXIV, value, arxiv, source, CONFIDENCE_MEDIUM, value)) + pmcid = normalize_pmcid(value) + if pmcid: + candidates.append(_candidate(IdentifierKind.PMCID, value, pmcid, source, CONFIDENCE_LOW, value)) + pmid = normalize_pmid(value) + if pmid: + candidates.append(_candidate(IdentifierKind.PMID, value, pmid, source, CONFIDENCE_FALLBACK, value)) + return candidates + + +def _candidate( + kind: IdentifierKind, + value_raw: str, + value_normalized: str, + source: str, + confidence_score: float, + evidence_url: str | None, +) -> IdentifierCandidate: + return IdentifierCandidate( + kind=kind, + value_raw=value_raw, + value_normalized=value_normalized, + source=source, + confidence_score=float(confidence_score), + evidence_url=evidence_url, + ) + + +def _dedup_candidates(candidates: list[IdentifierCandidate]) -> list[IdentifierCandidate]: + deduped: dict[tuple[str, str], IdentifierCandidate] = {} + for candidate in candidates: + key = (candidate.kind.value, candidate.value_normalized) + current = deduped.get(key) + if current is None or candidate.confidence_score > current.confidence_score: + deduped[key] = candidate + return list(deduped.values()) + + +async def sync_identifiers_for_publication_fields( + db_session: AsyncSession, + *, + publication: Publication, +) -> None: + candidates = _publication_field_candidates(publication) + await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=candidates) + + +def _publication_field_candidates(publication: Publication) -> list[IdentifierCandidate]: + return _fallback_candidates_from_values( + doi=publication.doi, + pub_url=publication.pub_url, + pdf_url=publication.pdf_url, + ) + + +async def sync_identifiers_for_publication_resolution( + db_session: AsyncSession, + *, + publication: Publication, + source: str | None, +) -> None: + candidates = _publication_field_candidates(publication) + rewritten = [_candidate_with_source(candidate, source=source) for candidate in candidates] + await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=rewritten) + + +def _candidate_with_source(candidate: IdentifierCandidate, *, source: str | None) -> IdentifierCandidate: + if not source: + return candidate + return IdentifierCandidate( + kind=candidate.kind, + value_raw=candidate.value_raw, + value_normalized=candidate.value_normalized, + source=source, + confidence_score=candidate.confidence_score, + evidence_url=candidate.evidence_url, + ) + + +async def _upsert_publication_candidates( + db_session: AsyncSession, + *, + publication_id: int, + candidates: list[IdentifierCandidate], +) -> None: + for candidate in _dedup_candidates(candidates): + await _upsert_publication_candidate(db_session, publication_id=publication_id, candidate=candidate) + + +async def _upsert_publication_candidate( + db_session: AsyncSession, + *, + publication_id: int, + candidate: IdentifierCandidate, +) -> None: + existing = await _existing_identifier( + db_session, + publication_id=publication_id, + kind=candidate.kind.value, + value_normalized=candidate.value_normalized, + ) + if existing is None: + db_session.add(_new_identifier_row(publication_id=publication_id, candidate=candidate)) + return + _merge_identifier_row(existing, candidate=candidate) + + +async def _existing_identifier( + db_session: AsyncSession, + *, + publication_id: int, + kind: str, + value_normalized: str, +) -> PublicationIdentifier | None: + result = await db_session.execute( + select(PublicationIdentifier).where( + PublicationIdentifier.publication_id == publication_id, + PublicationIdentifier.kind == kind, + PublicationIdentifier.value_normalized == value_normalized, + ) + ) + return result.scalar_one_or_none() + + +def _new_identifier_row( + *, + publication_id: int, + candidate: IdentifierCandidate, +) -> PublicationIdentifier: + return PublicationIdentifier( + publication_id=publication_id, + kind=candidate.kind.value, + value_raw=candidate.value_raw, + value_normalized=candidate.value_normalized, + source=candidate.source, + confidence_score=candidate.confidence_score, + evidence_url=candidate.evidence_url, + ) + + +def _merge_identifier_row(existing: PublicationIdentifier, *, candidate: IdentifierCandidate) -> None: + if candidate.confidence_score >= float(existing.confidence_score): + existing.value_raw = candidate.value_raw + existing.source = candidate.source + existing.confidence_score = candidate.confidence_score + if candidate.evidence_url: + existing.evidence_url = candidate.evidence_url + + +async def overlay_publication_items_with_display_identifiers( + db_session: AsyncSession, + *, + items: list[PublicationListItem], +) -> list[PublicationListItem]: + if not items: + return [] + mapping = await _display_identifier_map(db_session, publication_ids=[item.publication_id for item in items]) + return [_overlay_publication_item(item, mapping.get(item.publication_id)) for item in items] + + +def _overlay_publication_item( + item: PublicationListItem, + display_identifier: DisplayIdentifier | None, +) -> PublicationListItem: + fallback = display_identifier or derive_display_identifier_from_values(doi=item.doi, pub_url=item.pub_url, pdf_url=item.pdf_url) + return replace(item, display_identifier=fallback) + + +async def overlay_pdf_queue_items_with_display_identifiers( + db_session: AsyncSession, + *, + items: list[PdfQueueListItem], +) -> list[PdfQueueListItem]: + if not items: + return [] + mapping = await _display_identifier_map(db_session, publication_ids=[item.publication_id for item in items]) + return [_overlay_queue_item(item, mapping.get(item.publication_id)) for item in items] + + +def _overlay_queue_item( + item: PdfQueueListItem, + display_identifier: DisplayIdentifier | None, +) -> PdfQueueListItem: + fallback = display_identifier or derive_display_identifier_from_values(doi=item.doi, pdf_url=item.pdf_url) + return replace(item, display_identifier=fallback) + + +async def _display_identifier_map( + db_session: AsyncSession, + *, + publication_ids: list[int], +) -> dict[int, DisplayIdentifier]: + normalized_ids = sorted({int(value) for value in publication_ids if int(value) > 0}) + if not normalized_ids: + return {} + result = await db_session.execute( + select(PublicationIdentifier).where(PublicationIdentifier.publication_id.in_(normalized_ids)) + ) + rows = list(result.scalars().all()) + return _best_display_identifier_map(rows) + + +def _best_display_identifier_map(rows: list[PublicationIdentifier]) -> dict[int, DisplayIdentifier]: + grouped: dict[int, list[IdentifierCandidate]] = {} + for row in rows: + grouped.setdefault(int(row.publication_id), []).append(_candidate_from_row(row)) + return { + publication_id: display + for publication_id, display in ( + (publication_id, _best_display_identifier(candidates)) + for publication_id, candidates in grouped.items() + ) + if display is not None + } + + +def _candidate_from_row(row: PublicationIdentifier) -> IdentifierCandidate: + return IdentifierCandidate( + kind=IdentifierKind(str(row.kind)), + value_raw=str(row.value_raw), + value_normalized=str(row.value_normalized), + source=str(row.source), + confidence_score=float(row.confidence_score), + evidence_url=row.evidence_url, + ) + + +def _best_display_identifier(candidates: list[IdentifierCandidate]) -> DisplayIdentifier | None: + if not candidates: + return None + ordered = sorted(candidates, key=_display_sort_key, reverse=True) + return _display_identifier_from_candidate(ordered[0]) + + +def _display_sort_key(candidate: IdentifierCandidate) -> tuple[int, float]: + return (_kind_priority(candidate.kind), float(candidate.confidence_score)) + + +def _kind_priority(kind: IdentifierKind) -> int: + if kind == IdentifierKind.DOI: + return PRIORITY_DOI + if kind == IdentifierKind.ARXIV: + return PRIORITY_ARXIV + if kind == IdentifierKind.PMCID: + return PRIORITY_PMCID + return PRIORITY_PMID + + +def _display_identifier_from_candidate(candidate: IdentifierCandidate) -> DisplayIdentifier: + value = candidate.value_normalized + return DisplayIdentifier( + kind=candidate.kind.value, + value=value, + label=_display_label(candidate.kind, value), + url=_identifier_url(candidate.kind, value), + confidence_score=float(candidate.confidence_score), + ) + + +def _display_label(kind: IdentifierKind, value: str) -> str: + if kind == IdentifierKind.DOI: + return f"DOI: {value}" + if kind == IdentifierKind.ARXIV: + return f"arXiv: {value}" + if kind == IdentifierKind.PMCID: + return f"PMCID: {value}" + return f"PMID: {value}" + + +def _identifier_url(kind: IdentifierKind, value: str) -> str | None: + if kind == IdentifierKind.DOI: + return f"https://doi.org/{value}" + if kind == IdentifierKind.ARXIV: + return f"https://arxiv.org/abs/{value}" + if kind == IdentifierKind.PMCID: + return f"https://pmc.ncbi.nlm.nih.gov/articles/{value}/" + if kind == IdentifierKind.PMID: + return f"https://pubmed.ncbi.nlm.nih.gov/{value}/" + return None diff --git a/app/services/domains/publication_identifiers/normalize.py b/app/services/domains/publication_identifiers/normalize.py new file mode 100644 index 0000000..50036be --- /dev/null +++ b/app/services/domains/publication_identifiers/normalize.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import re +from urllib.parse import urlparse + +from app.services.domains.doi.normalize import normalize_doi +from app.services.domains.publication_identifiers.types import IdentifierKind + +ARXIV_ABS_RE = re.compile(r"\barxiv:\s*([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?\b", re.I) +ARXIV_PATH_RE = re.compile(r"^/(?:abs|pdf)/([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?(?:\.pdf)?/?$", re.I) +PMCID_RE = re.compile(r"\b(PMC\d+)\b", re.I) +PUBMED_PATH_RE = re.compile(r"^/(\d+)/?$") + + +def normalize_identifier(kind: IdentifierKind, value: str | None) -> str | None: + if kind == IdentifierKind.DOI: + return normalize_doi(value) + if kind == IdentifierKind.ARXIV: + return normalize_arxiv_id(value) + if kind == IdentifierKind.PMCID: + return normalize_pmcid(value) + if kind == IdentifierKind.PMID: + return normalize_pmid(value) + return None + + +def normalize_arxiv_id(value: str | None) -> str | None: + text = (value or "").strip() + if not text: + return None + parsed = urlparse(text) + if parsed.scheme in {"http", "https"} and "arxiv.org" in parsed.netloc.lower(): + return _arxiv_from_path(parsed.path) + match = ARXIV_ABS_RE.search(text) + if not match: + return None + version = (match.group(2) or "").lower() + return f"{match.group(1).lower()}{version}" + + +def _arxiv_from_path(path: str) -> str | None: + match = ARXIV_PATH_RE.match(path or "") + if not match: + return None + version = (match.group(2) or "").lower() + return f"{match.group(1).lower()}{version}" + + +def normalize_pmcid(value: str | None) -> str | None: + text = (value or "").strip() + if not text: + return None + parsed = urlparse(text) + if parsed.scheme in {"http", "https"} and "ncbi.nlm.nih.gov" in parsed.netloc.lower(): + return _first_match(PMCID_RE, parsed.path) + return _first_match(PMCID_RE, text) + + +def normalize_pmid(value: str | None) -> str | None: + text = (value or "").strip() + if not text: + return None + parsed = urlparse(text) + if parsed.scheme in {"http", "https"} and "pubmed.ncbi.nlm.nih.gov" in parsed.netloc.lower(): + match = PUBMED_PATH_RE.match(parsed.path or "") + if not match: + return None + return match.group(1) + return None + + +def _first_match(pattern: re.Pattern[str], value: str) -> str | None: + match = pattern.search(value) + if not match: + return None + return match.group(1).upper() diff --git a/app/services/domains/publication_identifiers/types.py b/app/services/domains/publication_identifiers/types.py new file mode 100644 index 0000000..34b20d0 --- /dev/null +++ b/app/services/domains/publication_identifiers/types.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + + +class IdentifierKind(StrEnum): + DOI = "doi" + ARXIV = "arxiv" + PMCID = "pmcid" + PMID = "pmid" + + +@dataclass(frozen=True) +class DisplayIdentifier: + kind: str + value: str + label: str + url: str | None + confidence_score: float + + +@dataclass(frozen=True) +class IdentifierCandidate: + kind: IdentifierKind + value_raw: str + value_normalized: str + source: str + confidence_score: float + evidence_url: str | None diff --git a/app/services/domains/publications/listing.py b/app/services/domains/publications/listing.py index f4f518b..0d09d4c 100644 --- a/app/services/domains/publications/listing.py +++ b/app/services/domains/publications/listing.py @@ -2,6 +2,7 @@ from __future__ import annotations from sqlalchemy.ext.asyncio import AsyncSession +from app.services.domains.publication_identifiers import application as identifier_service from app.services.domains.publications.modes import ( MODE_ALL, MODE_UNREAD, @@ -44,7 +45,10 @@ async def list_for_user( publication_list_item_from_row(row, latest_run_id=latest_run_id) for row in result.all() ] - return rows + return await identifier_service.overlay_publication_items_with_display_identifiers( + db_session, + items=rows, + ) async def retry_pdf_for_user( @@ -54,12 +58,19 @@ async def retry_pdf_for_user( scholar_profile_id: int, publication_id: int, ) -> PublicationListItem | None: - return await get_publication_item_for_user( + item = await get_publication_item_for_user( db_session, user_id=user_id, scholar_profile_id=scholar_profile_id, publication_id=publication_id, ) + if item is None: + return None + hydrated = await identifier_service.overlay_publication_items_with_display_identifiers( + db_session, + items=[item], + ) + return hydrated[0] if hydrated else item async def list_unread_for_user( diff --git a/app/services/domains/publications/pdf_queue.py b/app/services/domains/publications/pdf_queue.py index b420d46..3972f7d 100644 --- a/app/services/domains/publications/pdf_queue.py +++ b/app/services/domains/publications/pdf_queue.py @@ -17,11 +17,16 @@ from app.db.models import ( User, ) from app.db.session import get_session_factory +from app.services.domains.publication_identifiers import application as identifier_service +from app.services.domains.publication_identifiers.types import DisplayIdentifier +from app.services.domains.publications.pdf_resolution_pipeline import ( + PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE, + resolve_publication_pdf_outcome_for_row, +) from app.services.domains.publications.types import PublicationListItem from app.services.domains.unpaywall.application import ( FAILURE_RESOLUTION_EXCEPTION, OaResolutionOutcome, - resolve_publication_oa_outcomes, ) from app.settings import settings @@ -57,6 +62,7 @@ class PdfQueueListItem: last_attempt_at: datetime | None resolved_at: datetime | None updated_at: datetime + display_identifier: DisplayIdentifier | None = None @dataclass(frozen=True) @@ -118,6 +124,7 @@ def _item_from_row_and_job( pdf_attempt_count=int(job.attempt_count) if job is not None else 0, pdf_failure_reason=job.last_failure_reason if job is not None else None, pdf_failure_detail=job.last_failure_detail if job is not None else None, + display_identifier=row.display_identifier, ) @@ -366,8 +373,11 @@ async def _fetch_outcome_for_row( row: PublicationListItem, request_email: str | None, ) -> OaResolutionOutcome: - outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email) - outcome = outcomes.get(row.publication_id) + pipeline_result = await resolve_publication_pdf_outcome_for_row( + row=row, + request_email=request_email, + ) + outcome = pipeline_result.outcome if outcome is not None: return outcome return _failed_outcome(row=row) @@ -417,6 +427,11 @@ async def _persist_outcome( if publication is None or job is None: return _apply_publication_update(publication, doi=outcome.doi, pdf_url=outcome.pdf_url) + await identifier_service.sync_identifiers_for_publication_resolution( + db_session, + publication=publication, + source=outcome.source, + ) _apply_job_outcome(job, outcome=outcome) event_type, status = _result_event(outcome) db_session.add( @@ -794,6 +809,18 @@ def _queue_item_from_row(row: tuple) -> PdfQueueListItem: ) +async def _hydrated_queue_items( + db_session: AsyncSession, + *, + rows: list[tuple], +) -> list[PdfQueueListItem]: + items = [_queue_item_from_row(row) for row in rows] + return await identifier_service.overlay_pdf_queue_items_with_display_identifiers( + db_session, + items=items, + ) + + async def list_pdf_queue_items( db_session: AsyncSession, *, @@ -811,7 +838,7 @@ async def list_pdf_queue_items( offset=bounded_offset, ) ) - return [_queue_item_from_row(row) for row in result.all()] + return await _hydrated_queue_items(db_session, rows=list(result.all())) if normalized_status is None: result = await db_session.execute( _all_queue_select( @@ -819,7 +846,7 @@ async def list_pdf_queue_items( offset=bounded_offset, ) ) - return [_queue_item_from_row(row) for row in result.all()] + return await _hydrated_queue_items(db_session, rows=list(result.all())) result = await db_session.execute( _tracked_queue_select( limit=bounded_limit, @@ -827,7 +854,7 @@ async def list_pdf_queue_items( status=normalized_status, ) ) - return [_queue_item_from_row(row) for row in result.all()] + return await _hydrated_queue_items(db_session, rows=list(result.all())) async def count_pdf_queue_items( diff --git a/app/services/domains/publications/pdf_resolution_pipeline.py b/app/services/domains/publications/pdf_resolution_pipeline.py new file mode 100644 index 0000000..bc41257 --- /dev/null +++ b/app/services/domains/publications/pdf_resolution_pipeline.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +from dataclasses import dataclass +import logging +from urllib.parse import urlparse + +import httpx + +from app.services.domains.publications.types import PublicationListItem +from app.services.domains.scholar.publication_pdf import ( + ScholarPublicationLinkCandidate, + ScholarPublicationLinkCandidates, + fetch_link_candidates_from_scholar_publication_page, +) +from app.services.domains.unpaywall import pdf_discovery as pdf_discovery_service +from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes +from app.settings import settings + +logger = logging.getLogger(__name__) + +PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE = "scholar_publication_page" +PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED = "scholar_publication_page_unlabeled_fallback" +PDF_PATH_TOKEN = "/pdf/" +HTTP_TIMEOUT_FLOOR_SECONDS = 0.5 + + +@dataclass(frozen=True) +class PipelineOutcome: + outcome: OaResolutionOutcome | None + scholar_candidates: ScholarPublicationLinkCandidates | None + + +async def resolve_publication_pdf_outcome_for_row( + *, + row: PublicationListItem, + request_email: str | None, +) -> PipelineOutcome: + candidates = await _safe_scholar_candidates(row.pub_url) + labeled = _labeled_candidate(candidates) + if labeled is not None: + return PipelineOutcome(_scholar_outcome(row=row, candidate=labeled), candidates) + oa_outcome = await _oa_outcome(row=row, request_email=request_email) + if _oa_has_pdf(oa_outcome): + return PipelineOutcome(oa_outcome, candidates) + unlabeled = _unlabeled_candidate(candidates) + if unlabeled is None: + return PipelineOutcome(oa_outcome, candidates) + fallback_outcome = await _unlabeled_fallback_outcome(row=row, candidate=unlabeled) + if fallback_outcome is not None: + return PipelineOutcome(fallback_outcome, candidates) + return PipelineOutcome(oa_outcome, candidates) + + +async def _safe_scholar_candidates(pub_url: str | None) -> ScholarPublicationLinkCandidates | None: + try: + return await fetch_link_candidates_from_scholar_publication_page(pub_url) + except Exception as exc: # pragma: no cover - defensive boundary + logger.warning( + "publications.pdf_resolution.scholar_candidates_failed", + extra={"event": "publications.pdf_resolution.scholar_candidates_failed", "error": str(exc)}, + ) + return None + + +def _labeled_candidate( + candidates: ScholarPublicationLinkCandidates | None, +) -> ScholarPublicationLinkCandidate | None: + if candidates is None: + return None + return candidates.labeled_candidate + + +def _unlabeled_candidate( + candidates: ScholarPublicationLinkCandidates | None, +) -> ScholarPublicationLinkCandidate | None: + if candidates is None: + return None + return candidates.fallback_candidate + + +def _scholar_outcome( + *, + row: PublicationListItem, + candidate: ScholarPublicationLinkCandidate, +) -> OaResolutionOutcome: + source = ( + PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE + if candidate.label_present + else PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED + ) + return OaResolutionOutcome( + publication_id=row.publication_id, + doi=row.doi, + pdf_url=candidate.url, + failure_reason=None, + source=source, + used_crossref=False, + ) + + +async def _oa_outcome( + *, + row: PublicationListItem, + request_email: str | None, +) -> OaResolutionOutcome | None: + outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email) + return outcomes.get(row.publication_id) + + +def _oa_has_pdf(outcome: OaResolutionOutcome | None) -> bool: + return bool(outcome and outcome.pdf_url) + + +async def _unlabeled_fallback_outcome( + *, + row: PublicationListItem, + candidate: ScholarPublicationLinkCandidate, +) -> OaResolutionOutcome | None: + pdf_url = await _validated_pdf_url(candidate.url) + if pdf_url is None: + return None + return _scholar_outcome(row=row, candidate=ScholarPublicationLinkCandidate( + url=pdf_url, + confidence_score=candidate.confidence_score, + label_present=False, + reason=candidate.reason, + )) + + +async def _validated_pdf_url(candidate_url: str) -> str | None: + if _looks_direct_pdf(candidate_url): + return candidate_url + timeout_seconds = _discovery_timeout_seconds() + async with httpx.AsyncClient(timeout=timeout_seconds) as client: + if await pdf_discovery_service._candidate_is_pdf(client, candidate_url=candidate_url): + return candidate_url + return await pdf_discovery_service.resolve_pdf_from_landing_page(client, page_url=candidate_url) + + +def _looks_direct_pdf(url: str | None) -> bool: + if pdf_discovery_service.looks_like_pdf_url(url): + return True + if not isinstance(url, str): + return False + path = (urlparse(url).path or "").lower() + return PDF_PATH_TOKEN in path + + +def _discovery_timeout_seconds() -> float: + return max(float(settings.unpaywall_timeout_seconds), HTTP_TIMEOUT_FLOOR_SECONDS) diff --git a/app/services/domains/publications/types.py b/app/services/domains/publications/types.py index d1d4bad..3bde82d 100644 --- a/app/services/domains/publications/types.py +++ b/app/services/domains/publications/types.py @@ -3,6 +3,8 @@ from __future__ import annotations from dataclasses import dataclass from datetime import datetime +from app.services.domains.publication_identifiers.types import DisplayIdentifier + @dataclass(frozen=True) class PublicationListItem: @@ -24,6 +26,7 @@ class PublicationListItem: pdf_attempt_count: int = 0 pdf_failure_reason: str | None = None pdf_failure_detail: str | None = None + display_identifier: DisplayIdentifier | None = None @dataclass(frozen=True) diff --git a/app/services/domains/scholar/publication_pdf.py b/app/services/domains/scholar/publication_pdf.py new file mode 100644 index 0000000..6f8b6c1 --- /dev/null +++ b/app/services/domains/scholar/publication_pdf.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +from dataclasses import dataclass +from html.parser import HTMLParser +from urllib.parse import parse_qs, urlparse + +from app.services.domains.scholar.parser_types import ScholarDomInvariantError +from app.services.domains.scholar.parser_utils import attr_href, normalize_space +from app.services.domains.scholar.rate_limit import wait_for_scholar_slot +from app.services.domains.scholar.source import LiveScholarSource +from app.services.domains.settings.application import resolve_request_delay_minimum +from app.settings import settings + +CONTAINER_ID = "gsc_oci_title_gg" +PDF_LABEL_TOKEN = "[pdf]" +SCHOLAR_PDF_LABELED_CONFIDENCE = 0.98 +SCHOLAR_PDF_UNLABELED_CONFIDENCE = 0.2 +ALLOWED_URL_SCHEMES = frozenset({"http", "https"}) + + +@dataclass(frozen=True) +class ScholarPublicationLinkCandidate: + url: str + confidence_score: float + label_present: bool + reason: str + + +@dataclass(frozen=True) +class ScholarPublicationLinkCandidates: + container_seen: bool + labeled_candidate: ScholarPublicationLinkCandidate | None + fallback_candidate: ScholarPublicationLinkCandidate | None + warnings: tuple[str, ...] = () + + +@dataclass(frozen=True) +class _ParsedAnchor: + href: str | None + text: str + + +class _ScholarPublicationPdfParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.container_seen = False + self.anchors: list[_ParsedAnchor] = [] + self._container_depth = 0 + self._anchor_depth = 0 + self._anchor_href: str | None = None + self._anchor_parts: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + self._increment_depths() + if self._starts_container(tag, attrs): + self.container_seen = True + self._container_depth = 1 + return + if self._container_depth <= 0 or tag != "a": + return + if self._anchor_depth > 0: + return + self._anchor_href = attr_href(attrs) + self._anchor_parts = [] + self._anchor_depth = 1 + + def handle_data(self, data: str) -> None: + if self._anchor_depth > 0: + self._anchor_parts.append(data) + + def handle_endtag(self, tag: str) -> None: + if self._anchor_depth > 0: + self._anchor_depth -= 1 + if self._anchor_depth == 0: + self._finish_anchor() + if self._container_depth > 0: + self._container_depth -= 1 + + def _increment_depths(self) -> None: + if self._container_depth > 0: + self._container_depth += 1 + if self._anchor_depth > 0: + self._anchor_depth += 1 + + def _starts_container(self, tag: str, attrs: list[tuple[str, str | None]]) -> bool: + if tag != "div": + return False + attrs_map = {name.lower(): (value or "") for name, value in attrs} + return attrs_map.get("id") == CONTAINER_ID + + def _finish_anchor(self) -> None: + self.anchors.append( + _ParsedAnchor( + href=self._anchor_href, + text=normalize_space("".join(self._anchor_parts)), + ) + ) + self._anchor_href = None + self._anchor_parts = [] + + +def is_scholar_publication_detail_url(url: str | None) -> bool: + if not isinstance(url, str) or not url.strip(): + return False + parsed = urlparse(url) + if parsed.scheme not in ALLOWED_URL_SCHEMES: + return False + if parsed.netloc.lower() != "scholar.google.com": + return False + query = parse_qs(parsed.query) + return _has_view_citation_params(query) + + +def _has_view_citation_params(query: dict[str, list[str]]) -> bool: + view_op = (query.get("view_op") or [""])[0] + citation = (query.get("citation_for_view") or [""])[0].strip() + return view_op == "view_citation" and bool(citation) + + +def extract_link_candidates_from_publication_detail_html(html: str) -> ScholarPublicationLinkCandidates: + parser = _parsed_publication_detail(html) + if not parser.container_seen: + return ScholarPublicationLinkCandidates(False, None, None) + anchors = _validated_container_anchors(parser.anchors) + labeled = _select_labeled_candidate(anchors) + fallback = _select_fallback_candidate(anchors, labeled=labeled) + warnings = _candidate_warnings(labeled=labeled, fallback=fallback) + return ScholarPublicationLinkCandidates(True, labeled, fallback, warnings) + + +def _parsed_publication_detail(html: str) -> _ScholarPublicationPdfParser: + parser = _ScholarPublicationPdfParser() + parser.feed(html) + parser.close() + return parser + + +def _validated_container_anchors(anchors: list[_ParsedAnchor]) -> list[_ParsedAnchor]: + if not anchors: + raise ScholarDomInvariantError( + code="layout_publication_link_container_missing_anchor", + message="Scholar publication link container was present without an anchor.", + ) + validated: list[_ParsedAnchor] = [] + for anchor in anchors: + validated.append(_validated_anchor(anchor)) + return validated + + +def _validated_anchor(anchor: _ParsedAnchor) -> _ParsedAnchor: + href = (anchor.href or "").strip() + if not href: + raise ScholarDomInvariantError( + code="layout_publication_link_missing_href", + message="Scholar publication link container anchor was missing href.", + ) + parsed = urlparse(href) + if parsed.scheme not in ALLOWED_URL_SCHEMES: + raise ScholarDomInvariantError( + code="layout_publication_link_invalid_scheme", + message="Scholar publication link used a non-http URL.", + ) + return _ParsedAnchor(href=href, text=anchor.text) + + +def _select_labeled_candidate(anchors: list[_ParsedAnchor]) -> ScholarPublicationLinkCandidate | None: + for anchor in anchors: + if PDF_LABEL_TOKEN in anchor.text.lower(): + return ScholarPublicationLinkCandidate( + url=str(anchor.href), + confidence_score=SCHOLAR_PDF_LABELED_CONFIDENCE, + label_present=True, + reason="scholar_link_labeled_pdf", + ) + return None + + +def _select_fallback_candidate( + anchors: list[_ParsedAnchor], + *, + labeled: ScholarPublicationLinkCandidate | None, +) -> ScholarPublicationLinkCandidate | None: + for anchor in anchors: + if labeled and anchor.href == labeled.url: + continue + return ScholarPublicationLinkCandidate( + url=str(anchor.href), + confidence_score=SCHOLAR_PDF_UNLABELED_CONFIDENCE, + label_present=False, + reason="scholar_link_unlabeled_fallback", + ) + if labeled is None and anchors: + anchor = anchors[0] + return ScholarPublicationLinkCandidate( + url=str(anchor.href), + confidence_score=SCHOLAR_PDF_UNLABELED_CONFIDENCE, + label_present=False, + reason="scholar_link_unlabeled_fallback", + ) + return None + + +def _candidate_warnings( + *, + labeled: ScholarPublicationLinkCandidate | None, + fallback: ScholarPublicationLinkCandidate | None, +) -> tuple[str, ...]: + warnings: list[str] = [] + if labeled is None and fallback is not None: + warnings.append("scholar_publication_link_unlabeled_only") + return tuple(warnings) + + +def _scholar_request_delay_seconds() -> int: + return resolve_request_delay_minimum(settings.ingestion_min_request_delay_seconds) + + +def _fetch_succeeded(fetch_result) -> bool: + return int(fetch_result.status_code or 0) == 200 and not fetch_result.error + + +async def fetch_link_candidates_from_scholar_publication_page( + publication_url: str | None, +) -> ScholarPublicationLinkCandidates | None: + if not is_scholar_publication_detail_url(publication_url): + return None + await wait_for_scholar_slot(min_interval_seconds=float(_scholar_request_delay_seconds())) + source = LiveScholarSource() + fetch_result = await source.fetch_publication_html(str(publication_url)) + if not _fetch_succeeded(fetch_result): + return None + return extract_link_candidates_from_publication_detail_html(fetch_result.body) diff --git a/app/services/domains/scholar/rate_limit.py b/app/services/domains/scholar/rate_limit.py new file mode 100644 index 0000000..947c91f --- /dev/null +++ b/app/services/domains/scholar/rate_limit.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import asyncio +import time + +_REQUEST_LOCK = asyncio.Lock() +_LAST_REQUEST_AT = 0.0 + + +async def wait_for_scholar_slot(*, min_interval_seconds: float) -> None: + global _LAST_REQUEST_AT + interval = max(float(min_interval_seconds), 0.0) + async with _REQUEST_LOCK: + elapsed = time.monotonic() - _LAST_REQUEST_AT + remaining = interval - elapsed + if remaining > 0: + await asyncio.sleep(remaining) + _LAST_REQUEST_AT = time.monotonic() diff --git a/app/services/domains/scholar/source.py b/app/services/domains/scholar/source.py index 83da165..92c6702 100644 --- a/app/services/domains/scholar/source.py +++ b/app/services/domains/scholar/source.py @@ -123,6 +123,16 @@ class LiveScholarSource: ) return await asyncio.to_thread(self._fetch_sync, requested_url) + async def fetch_publication_html(self, publication_url: str) -> FetchResult: + logger.debug( + "scholar_source.publication_fetch_started", + extra={ + "event": "scholar_source.publication_fetch_started", + "requested_url": publication_url, + }, + ) + return await asyncio.to_thread(self._fetch_sync, publication_url) + def _build_request(self, requested_url: str) -> Request: return Request( requested_url, diff --git a/app/services/domains/unpaywall/application.py b/app/services/domains/unpaywall/application.py index 44c7560..a658818 100644 --- a/app/services/domains/unpaywall/application.py +++ b/app/services/domains/unpaywall/application.py @@ -64,20 +64,16 @@ def _extract_explicit_doi(text: str | None) -> str | None: def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str | None: stored = normalize_doi(item.doi) - if stored: - in_metadata = any( - normalize_doi(_extract_explicit_doi(value)) == stored - for value in (item.pub_url, item.venue_text) - ) - if in_metadata: - return stored - pub_url_doi = _extract_doi_candidate(item.pub_url) - if pub_url_doi: - return normalize_doi(pub_url_doi) - return ( + explicit_doi = ( _extract_explicit_doi(item.pub_url) or _extract_explicit_doi(item.venue_text) ) + if explicit_doi: + return explicit_doi + pub_url_doi = _extract_doi_candidate(item.pub_url) + if pub_url_doi: + return normalize_doi(pub_url_doi) + return stored def _payload_locations(payload: dict) -> list[dict]: @@ -217,30 +213,30 @@ async def _resolve_item_payload( item: PublicationListItem, email: str, allow_crossref: bool, -) -> tuple[dict | None, bool]: +) -> tuple[dict | None, bool, str | None]: doi = _publication_doi(item) payload: dict | None = None if doi: payload = await _fetch_unpaywall_payload_by_doi(client=client, doi=doi, email=email) if payload is not None and _has_direct_payload_pdf(payload): - return payload, False + return payload, False, doi if not allow_crossref or not settings.crossref_enabled: - return payload, False + return payload, False, doi crossref_doi = await discover_doi_for_publication( item=item, max_rows=settings.crossref_max_rows, email=email, ) if crossref_doi is None or crossref_doi == doi: - return payload, crossref_doi is not None + return payload, crossref_doi is not None, doi or crossref_doi crossref_payload = await _fetch_unpaywall_payload_by_doi( client=client, doi=crossref_doi, email=email, ) if crossref_payload is not None: - return crossref_payload, True - return payload, True + return crossref_payload, True, crossref_doi + return payload, True, crossref_doi async def _doi_and_pdf_from_payload( @@ -265,10 +261,11 @@ def _outcome_with_failure( item: PublicationListItem, failure_reason: str, used_crossref: bool, + doi_override: str | None = None, ) -> OaResolutionOutcome: return OaResolutionOutcome( publication_id=item.publication_id, - doi=_publication_doi(item), + doi=normalize_doi(doi_override) if doi_override is not None else _publication_doi(item), pdf_url=None, failure_reason=failure_reason, source=None, @@ -323,7 +320,7 @@ async def _resolve_outcome_for_item( email: str, allow_crossref: bool, ) -> OaResolutionOutcome: - payload, used_crossref = await _resolve_item_payload( + payload, used_crossref, resolved_doi = await _resolve_item_payload( client=client, item=item, email=email, @@ -334,6 +331,7 @@ async def _resolve_outcome_for_item( item=item, failure_reason=_missing_payload_failure_reason(item, used_crossref=used_crossref), used_crossref=used_crossref, + doi_override=resolved_doi, ) doi, pdf_url = await _doi_and_pdf_from_payload(payload, client=client) return _outcome_from_payload( diff --git a/frontend/src/components/ui/AppHelpHint.vue b/frontend/src/components/ui/AppHelpHint.vue index 6736158..50d5b52 100644 --- a/frontend/src/components/ui/AppHelpHint.vue +++ b/frontend/src/components/ui/AppHelpHint.vue @@ -182,7 +182,7 @@ onBeforeUnmount(() => { :class=" hasTriggerSlot ? 'cursor-help rounded-sm' - : 'h-5 w-5 justify-center rounded-full border border-stroke-interactive/80 bg-surface-card-muted/80 text-[11px] font-semibold text-ink-secondary transition hover:bg-action-secondary-hover-bg' + : 'h-5 w-5 justify-center rounded-full border border-stroke-interactive/80 bg-surface-card-muted/80 text-xs font-semibold text-ink-secondary transition hover:bg-action-secondary-hover-bg' " :aria-label="text" @mouseenter="openTooltip" @@ -200,7 +200,7 @@ onBeforeUnmount(() => { role="tooltip" :style="tooltipStyle" :class="sideClass" - class="pointer-events-none fixed z-[90] w-64 max-w-[calc(100vw-2rem)] rounded-lg border border-stroke-default bg-surface-card/95 px-2.5 py-2 text-xs leading-relaxed text-ink-secondary shadow-lg" + class="pointer-events-none fixed z-50 w-auto max-w-xs rounded-lg border border-stroke-default bg-surface-card/95 px-2.5 py-2 text-xs leading-relaxed text-ink-secondary shadow-lg" > {{ text }} diff --git a/frontend/src/features/admin_dbops/index.ts b/frontend/src/features/admin_dbops/index.ts index e3269e1..2f042f5 100644 --- a/frontend/src/features/admin_dbops/index.ts +++ b/frontend/src/features/admin_dbops/index.ts @@ -1,5 +1,13 @@ import { apiRequest } from "@/lib/api/client"; +export interface DisplayIdentifier { + kind: string; + value: string; + label: string; + url: string | null; + confidence_score: number; +} + export interface AdminDbIntegrityCheck { name: string; count: number; @@ -34,6 +42,7 @@ export interface AdminPdfQueueItem { publication_id: number; title: string; doi: string | null; + display_identifier: DisplayIdentifier | null; pdf_url: string | null; status: string; attempt_count: number; diff --git a/frontend/src/features/publications/index.ts b/frontend/src/features/publications/index.ts index 2eddfbb..f238a0e 100644 --- a/frontend/src/features/publications/index.ts +++ b/frontend/src/features/publications/index.ts @@ -2,6 +2,14 @@ import { apiRequest } from "@/lib/api/client"; export type PublicationMode = "all" | "unread" | "latest"; +export interface DisplayIdentifier { + kind: string; + value: string; + label: string; + url: string | null; + confidence_score: number; +} + export interface PublicationItem { publication_id: number; scholar_profile_id: number; @@ -12,6 +20,7 @@ export interface PublicationItem { venue_text: string | null; pub_url: string | null; doi: string | null; + display_identifier: DisplayIdentifier | null; pdf_url: string | null; pdf_status: "untracked" | "queued" | "running" | "resolved" | "failed"; pdf_attempt_count: number; diff --git a/frontend/src/features/settings/SettingsAdminPanel.vue b/frontend/src/features/settings/SettingsAdminPanel.vue index 7f5788e..7edf67b 100644 --- a/frontend/src/features/settings/SettingsAdminPanel.vue +++ b/frontend/src/features/settings/SettingsAdminPanel.vue @@ -748,8 +748,14 @@ watch(
{{ item.title }} - - DOI: {{ item.doi }} + + {{ item.display_identifier.label }}
diff --git a/frontend/src/pages/PublicationsPage.vue b/frontend/src/pages/PublicationsPage.vue index fbecad9..beeedc7 100644 --- a/frontend/src/pages/PublicationsPage.vue +++ b/frontend/src/pages/PublicationsPage.vue @@ -169,11 +169,15 @@ function publicationPrimaryUrl(item: PublicationItem): string | null { return item.pub_url || item.pdf_url; } -function publicationDoiUrl(item: PublicationItem): string | null { - if (!item.doi) { +function publicationIdentifierUrl(item: PublicationItem): string | null { + if (!item.display_identifier?.url) { return null; } - return `https://doi.org/${item.doi}`; + return item.display_identifier.url; +} + +function publicationIdentifierLabel(item: PublicationItem): string | null { + return item.display_identifier?.label ?? null; } const selectedScholarName = computed(() => { @@ -1043,14 +1047,14 @@ watch( {{ item.title }} - DOI: {{ item.doi }} + {{ publicationIdentifierLabel(item) }} diff --git a/tests/unit/test_crossref_lookup.py b/tests/unit/test_crossref_lookup.py index 5d71281..67712fa 100644 --- a/tests/unit/test_crossref_lookup.py +++ b/tests/unit/test_crossref_lookup.py @@ -52,7 +52,9 @@ async def test_crossref_discovers_doi_from_best_title_match(monkeypatch: pytest. @pytest.mark.asyncio -async def test_crossref_rejects_large_year_mismatch(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_crossref_relaxed_fallback_allows_large_year_mismatch_for_strong_title_match( + monkeypatch: pytest.MonkeyPatch, +) -> None: async def _fake_fetch_items(**_kwargs): return [ { @@ -72,4 +74,30 @@ async def test_crossref_rejects_large_year_mismatch(monkeypatch: pytest.MonkeyPa max_rows=10, email=None, ) - assert doi is None + assert doi == "10.1000/wrong-year" + + +@pytest.mark.asyncio +async def test_crossref_relaxed_fallback_allows_author_mismatch_for_strong_title_match( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def _fake_fetch_items(**_kwargs): + return [ + { + "DOI": "10.1000/author-fallback", + "title": ["Induction of Pluripotent Stem Cells from Adult Human Fibroblasts"], + "issued": {"date-parts": [[2007]]}, + "author": [{"family": "SomeoneElse"}], + } + ] + + monkeypatch.setattr(crossref_app, "_fetch_items", _fake_fetch_items) + doi = await crossref_app.discover_doi_for_publication( + item=_item( + title="Induction of Pluripotent Stem Cells from Adult Human Fibroblasts", + year=2007, + ), + max_rows=10, + email=None, + ) + assert doi == "10.1000/author-fallback" diff --git a/tests/unit/test_publication_identifiers.py b/tests/unit/test_publication_identifiers.py new file mode 100644 index 0000000..feab56b --- /dev/null +++ b/tests/unit/test_publication_identifiers.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from app.services.domains.publication_identifiers import application as identifier_service + + +def test_derive_display_identifier_prefers_doi_over_arxiv() -> None: + display = identifier_service.derive_display_identifier_from_values( + doi="10.1000/example", + pub_url="https://arxiv.org/abs/1504.08025", + pdf_url=None, + ) + assert display is not None + assert display.kind == "doi" + assert display.value == "10.1000/example" + assert display.url == "https://doi.org/10.1000/example" + + +def test_derive_display_identifier_uses_arxiv_when_doi_missing() -> None: + display = identifier_service.derive_display_identifier_from_values( + doi=None, + pub_url="https://arxiv.org/pdf/1504.08025v2", + pdf_url=None, + ) + assert display is not None + assert display.kind == "arxiv" + assert display.value == "1504.08025v2" + assert display.label == "arXiv: 1504.08025v2" + + +def test_derive_display_identifier_uses_pmcid_when_present() -> None: + display = identifier_service.derive_display_identifier_from_values( + doi=None, + pub_url=None, + pdf_url="https://pmc.ncbi.nlm.nih.gov/articles/PMC2175868/pdf/file.pdf", + ) + assert display is not None + assert display.kind == "pmcid" + assert display.value == "PMC2175868" diff --git a/tests/unit/test_publication_pdf_queue_policy.py b/tests/unit/test_publication_pdf_queue_policy.py index 3d4fe7f..0cc3a7c 100644 --- a/tests/unit/test_publication_pdf_queue_policy.py +++ b/tests/unit/test_publication_pdf_queue_policy.py @@ -1,11 +1,14 @@ from __future__ import annotations from datetime import datetime, timedelta, timezone +from types import SimpleNamespace import pytest from app.db.models import PublicationPdfJob from app.services.domains.publications import pdf_queue +from app.services.domains.publications.pdf_resolution_pipeline import PipelineOutcome +from app.services.domains.unpaywall.application import OaResolutionOutcome def _job( @@ -22,6 +25,25 @@ def _job( ) +def _row(*, pub_url: str | None = "https://scholar.google.com/citations?view_op=view_citation&citation_for_view=abc:xyz") -> SimpleNamespace: + return SimpleNamespace( + publication_id=1, + scholar_profile_id=1, + scholar_label="Ada Lovelace", + title="A paper", + year=2024, + citation_count=0, + venue_text=None, + pub_url=pub_url, + doi=None, + pdf_url=None, + is_read=False, + is_favorite=False, + first_seen_at=datetime(2026, 2, 22, 12, 0, tzinfo=timezone.utc), + is_new_in_latest_run=True, + ) + + def test_pdf_queue_auto_enqueue_blocks_recent_attempt(monkeypatch: pytest.MonkeyPatch) -> None: now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc) monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now) @@ -107,3 +129,44 @@ def test_pdf_queue_manual_requeue_still_blocks_when_inflight() -> None: ) assert pdf_queue._can_enqueue_job(running, force_retry=True) is False assert pdf_queue._can_enqueue_job(queued, force_retry=True) is False + + +@pytest.mark.asyncio +async def test_fetch_outcome_for_row_uses_pipeline_outcome(monkeypatch: pytest.MonkeyPatch) -> None: + async def _fake_pipeline(*, row, request_email=None): + assert request_email == "user@example.com" + return PipelineOutcome( + outcome=OaResolutionOutcome( + publication_id=row.publication_id, + doi=None, + pdf_url="https://arxiv.org/pdf/1703.06103", + failure_reason=None, + source=pdf_queue.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE, + used_crossref=False, + ), + scholar_candidates=None, + ) + + monkeypatch.setattr(pdf_queue, "resolve_publication_pdf_outcome_for_row", _fake_pipeline) + + outcome = await pdf_queue._fetch_outcome_for_row(row=_row(), request_email="user@example.com") + + assert outcome.pdf_url == "https://arxiv.org/pdf/1703.06103" + assert outcome.source == pdf_queue.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE + assert outcome.used_crossref is False + + +@pytest.mark.asyncio +async def test_fetch_outcome_for_row_returns_failed_outcome_when_pipeline_returns_none( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def _fake_pipeline(*, row, request_email=None): + assert request_email == "user@example.com" + return PipelineOutcome(outcome=None, scholar_candidates=None) + + monkeypatch.setattr(pdf_queue, "resolve_publication_pdf_outcome_for_row", _fake_pipeline) + + outcome = await pdf_queue._fetch_outcome_for_row(row=_row(), request_email="user@example.com") + + assert outcome.pdf_url is None + assert outcome.failure_reason == pdf_queue.FAILURE_RESOLUTION_EXCEPTION diff --git a/tests/unit/test_publication_pdf_resolution_pipeline.py b/tests/unit/test_publication_pdf_resolution_pipeline.py new file mode 100644 index 0000000..bc7bb5c --- /dev/null +++ b/tests/unit/test_publication_pdf_resolution_pipeline.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest + +from app.services.domains.publications import pdf_resolution_pipeline as pipeline +from app.services.domains.unpaywall.application import OaResolutionOutcome + + +@dataclass(frozen=True) +class _Candidate: + url: str + confidence_score: float + label_present: bool + reason: str + + +@dataclass(frozen=True) +class _Candidates: + container_seen: bool + labeled_candidate: _Candidate | None + fallback_candidate: _Candidate | None + warnings: tuple[str, ...] = () + + +def _row(*, doi: str | None = None) -> SimpleNamespace: + return SimpleNamespace( + publication_id=1, + scholar_profile_id=1, + scholar_label="Ada Lovelace", + title="A paper", + year=2024, + citation_count=0, + venue_text=None, + pub_url="https://scholar.google.com/citations?view_op=view_citation&citation_for_view=abc:xyz", + doi=doi, + pdf_url=None, + is_read=False, + is_favorite=False, + first_seen_at=datetime(2026, 2, 22, 12, 0, tzinfo=timezone.utc), + is_new_in_latest_run=True, + ) + + +def _oa_outcome(*, pdf_url: str | None, source: str = "unpaywall") -> OaResolutionOutcome: + return OaResolutionOutcome( + publication_id=1, + doi="10.1000/example", + pdf_url=pdf_url, + failure_reason=None if pdf_url else "no_pdf_found", + source=source, + used_crossref=False, + ) + + +@pytest.mark.asyncio +async def test_pipeline_prefers_labeled_scholar_candidate_before_oa(monkeypatch: pytest.MonkeyPatch) -> None: + async def _fake_candidates(_url): + return _Candidates( + container_seen=True, + labeled_candidate=_Candidate( + url="https://arxiv.org/pdf/1703.06103", + confidence_score=0.98, + label_present=True, + reason="scholar_link_labeled_pdf", + ), + fallback_candidate=None, + ) + + async def _fail_oa(*, row, request_email): + raise AssertionError(f"OA should not run when labeled Scholar candidate exists: {row.publication_id} {request_email}") + + monkeypatch.setattr(pipeline, "fetch_link_candidates_from_scholar_publication_page", _fake_candidates) + monkeypatch.setattr(pipeline, "_oa_outcome", _fail_oa) + + result = await pipeline.resolve_publication_pdf_outcome_for_row(row=_row(), request_email="user@example.com") + + assert result.outcome is not None + assert result.outcome.pdf_url == "https://arxiv.org/pdf/1703.06103" + assert result.outcome.source == pipeline.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE + + +@pytest.mark.asyncio +async def test_pipeline_uses_oa_result_before_unlabeled_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + async def _fake_candidates(_url): + return _Candidates( + container_seen=True, + labeled_candidate=None, + fallback_candidate=_Candidate( + url="https://example.org/download/42", + confidence_score=0.2, + label_present=False, + reason="scholar_link_unlabeled_fallback", + ), + ) + + async def _fake_oa(*, row, request_email): + assert request_email == "user@example.com" + return _oa_outcome(pdf_url="https://oa.example.org/found.pdf") + + async def _fail_fallback(*, row, candidate): + raise AssertionError(f"Unlabeled fallback should not run when OA returns PDF: {row.publication_id} {candidate.url}") + + monkeypatch.setattr(pipeline, "fetch_link_candidates_from_scholar_publication_page", _fake_candidates) + monkeypatch.setattr(pipeline, "_oa_outcome", _fake_oa) + monkeypatch.setattr(pipeline, "_unlabeled_fallback_outcome", _fail_fallback) + + result = await pipeline.resolve_publication_pdf_outcome_for_row(row=_row(), request_email="user@example.com") + + assert result.outcome is not None + assert result.outcome.pdf_url == "https://oa.example.org/found.pdf" + assert result.outcome.source == "unpaywall" + + +@pytest.mark.asyncio +async def test_pipeline_uses_unlabeled_fallback_after_oa_failure(monkeypatch: pytest.MonkeyPatch) -> None: + fallback_candidate = _Candidate( + url="https://example.org/download/42", + confidence_score=0.2, + label_present=False, + reason="scholar_link_unlabeled_fallback", + ) + + async def _fake_candidates(_url): + return _Candidates(container_seen=True, labeled_candidate=None, fallback_candidate=fallback_candidate) + + async def _fake_oa(*, row, request_email): + assert request_email == "user@example.com" + return _oa_outcome(pdf_url=None) + + async def _fake_fallback(*, row, candidate): + assert candidate == fallback_candidate + return OaResolutionOutcome( + publication_id=row.publication_id, + doi=row.doi, + pdf_url="https://example.org/fallback.pdf", + failure_reason=None, + source=pipeline.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED, + used_crossref=False, + ) + + monkeypatch.setattr(pipeline, "fetch_link_candidates_from_scholar_publication_page", _fake_candidates) + monkeypatch.setattr(pipeline, "_oa_outcome", _fake_oa) + monkeypatch.setattr(pipeline, "_unlabeled_fallback_outcome", _fake_fallback) + + result = await pipeline.resolve_publication_pdf_outcome_for_row(row=_row(), request_email="user@example.com") + + assert result.outcome is not None + assert result.outcome.pdf_url == "https://example.org/fallback.pdf" + assert result.outcome.source == pipeline.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED diff --git a/tests/unit/test_scholar_publication_pdf.py b/tests/unit/test_scholar_publication_pdf.py new file mode 100644 index 0000000..7382b77 --- /dev/null +++ b/tests/unit/test_scholar_publication_pdf.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import pytest + +from app.services.domains.scholar.parser_types import ScholarDomInvariantError +from app.services.domains.scholar.publication_pdf import ( + extract_link_candidates_from_publication_detail_html, + is_scholar_publication_detail_url, +) + + +def test_extract_link_candidates_from_publication_detail_html_reads_gsc_oci_pdf_link() -> None: + html = """ + +
+ +
+ + """ + candidates = extract_link_candidates_from_publication_detail_html(html) + assert candidates.labeled_candidate is not None + assert candidates.labeled_candidate.url == "https://arxiv.org/pdf/1703.06103" + + +def test_extract_link_candidates_from_publication_detail_html_returns_no_candidates_when_container_missing() -> None: + html = "
No PDF section
" + candidates = extract_link_candidates_from_publication_detail_html(html) + assert candidates.container_seen is False + assert candidates.labeled_candidate is None + assert candidates.fallback_candidate is None + + +def test_extract_pdf_url_from_publication_detail_html_fails_fast_on_malformed_pdf_container() -> None: + html = """ + +
+ +
+ + """ + with pytest.raises(ScholarDomInvariantError) as exc: + extract_link_candidates_from_publication_detail_html(html) + assert exc.value.code == "layout_publication_link_missing_href" + + +def test_extract_link_candidates_from_publication_detail_html_keeps_unlabeled_fallback() -> None: + html = """ + +
+ +
+ + """ + candidates = extract_link_candidates_from_publication_detail_html(html) + assert candidates.container_seen is True + assert candidates.labeled_candidate is None + assert candidates.fallback_candidate is not None + assert candidates.fallback_candidate.url == "https://example.org/download?id=42" + assert candidates.fallback_candidate.label_present is False + assert "scholar_publication_link_unlabeled_only" in candidates.warnings + assert candidates.labeled_candidate is None + + +def test_is_scholar_publication_detail_url_matches_view_citation_links() -> None: + assert is_scholar_publication_detail_url( + "https://scholar.google.com/citations?view_op=view_citation&hl=en&user=8200InoAAAAJ&citation_for_view=8200InoAAAAJ:gsN89kCJA0AC" + ) is True + assert is_scholar_publication_detail_url("https://example.org/paper") is False diff --git a/tests/unit/test_unpaywall_resolution.py b/tests/unit/test_unpaywall_resolution.py index 5cc34ce..651b33d 100644 --- a/tests/unit/test_unpaywall_resolution.py +++ b/tests/unit/test_unpaywall_resolution.py @@ -1,5 +1,6 @@ from __future__ import annotations +from dataclasses import replace from datetime import datetime, timezone import pytest @@ -38,6 +39,26 @@ def _item(publication_id: int) -> PublicationListItem: ) +def test_publication_doi_uses_stored_value_when_metadata_has_no_doi() -> None: + item = replace( + _item(99), + pub_url="https://scholar.google.com/citations?view_op=view_citation&citation_for_view=abc:123", + venue_text="Cell 130 (5), 2007", + doi="10.1016/j.cell.2007.11.019", + ) + assert unpaywall_app._publication_doi(item) == "10.1016/j.cell.2007.11.019" + + +def test_publication_doi_prefers_explicit_metadata_doi_over_stored_value() -> None: + item = replace( + _item(100), + pub_url="https://doi.org/10.2000/fresh-value", + venue_text="Cell", + doi="10.1000/stale-value", + ) + assert unpaywall_app._publication_doi(item) == "10.2000/fresh-value" + + @pytest.mark.asyncio async def test_unpaywall_resolve_prefers_direct_pdf_without_landing_crawl(monkeypatch: pytest.MonkeyPatch) -> None: payload = { @@ -49,7 +70,7 @@ async def test_unpaywall_resolve_prefers_direct_pdf_without_landing_crawl(monkey } async def _fake_resolve_item_payload(**_kwargs): - return payload, False + return payload, False, "10.1016/j.cell.2007.11.019" async def _fail_crawl(_client, *, page_url: str): raise AssertionError(f"unexpected landing crawl: {page_url}") @@ -75,7 +96,7 @@ async def test_unpaywall_resolve_crawls_landing_page_when_pdf_url_is_not_direct( crawled_pages: list[str] = [] async def _fake_resolve_item_payload(**_kwargs): - return payload, False + return payload, False, "10.1016/j.cell.2007.11.019" async def _fake_crawl(_client, *, page_url: str): crawled_pages.append(page_url) @@ -87,3 +108,22 @@ async def test_unpaywall_resolve_crawls_landing_page_when_pdf_url_is_not_direct( resolved = await unpaywall_app.resolve_publication_oa_metadata([_item(2)], request_email="user@example.com") assert resolved == {2: ("10.1016/j.cell.2007.11.019", "https://oa.example.org/files/paper-42.pdf")} assert "https://oa.example.org/landing/42" in crawled_pages + + +@pytest.mark.asyncio +async def test_unpaywall_preserves_crossref_doi_when_unpaywall_has_no_record( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def _fake_resolve_item_payload(**_kwargs): + return None, True, "10.2000/crossref-only" + + monkeypatch.setattr(unpaywall_app, "_resolve_item_payload", _fake_resolve_item_payload) + monkeypatch.setattr("httpx.AsyncClient", _DummyAsyncClient) + + outcomes = await unpaywall_app.resolve_publication_oa_outcomes([_item(3)], request_email="user@example.com") + + outcome = outcomes[3] + assert outcome.doi == "10.2000/crossref-only" + assert outcome.pdf_url is None + assert outcome.failure_reason == unpaywall_app.FAILURE_NO_RECORD + assert outcome.used_crossref is True -- 2.49.1 From 8760f27b51417e676a040bede3650958cbd1f935 Mon Sep 17 00:00:00 2001 From: Justin Visser Date: Sun, 22 Feb 2026 16:35:39 +0100 Subject: [PATCH 02/38] small ui update --- frontend/src/pages/PublicationsPage.vue | 10 ++++++++-- frontend/src/pages/ScholarsPage.vue | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/frontend/src/pages/PublicationsPage.vue b/frontend/src/pages/PublicationsPage.vue index beeedc7..21afc03 100644 --- a/frontend/src/pages/PublicationsPage.vue +++ b/frontend/src/pages/PublicationsPage.vue @@ -802,8 +802,13 @@ watch( diff --git a/frontend/src/pages/ScholarsPage.vue b/frontend/src/pages/ScholarsPage.vue index 4c8fb57..a8794f2 100644 --- a/frontend/src/pages/ScholarsPage.vue +++ b/frontend/src/pages/ScholarsPage.vue @@ -869,7 +869,7 @@ onMounted(() => { />
-
    +
    • Date: Thu, 26 Feb 2026 12:54:19 +0100 Subject: [PATCH 03/38] temp commit --- README.md | 41 +- agents.md | 48 +- ...8_add_openalex_enriched_to_publications.py | 30 + ...0260224_0019_add_resolving_to_runstatus.py | 31 + ...60224_0020_add_openalex_last_attempt_at.py | 26 + ...0021_enforce_single_active_run_per_user.py | 70 + ...dd_canonical_title_hash_to_publications.py | 35 + ...ef777_remove_doi_from_publications_and_.py | 27 + ...add_api_integration_keys_to_usersetting.py | 39 + app/api/routers/admin_dbops.py | 62 +- app/api/routers/publications.py | 53 +- app/api/routers/runs.py | 194 +- app/api/routers/settings.py | 7 + app/api/schemas.py | 12 +- app/db/models.py | 25 +- app/main.py | 12 + app/services/domains/arxiv/application.py | 110 + app/services/domains/ingestion/application.py | 862 +++++- .../domains/ingestion/fingerprints.py | 141 +- app/services/domains/ingestion/scheduler.py | 26 + app/services/domains/ingestion/types.py | 2 + app/services/domains/openalex/__init__.py | 5 + app/services/domains/openalex/client.py | 151 ++ app/services/domains/openalex/matching.py | 108 + app/services/domains/openalex/types.py | 71 + app/services/domains/portability/exporting.py | 3 - .../domains/portability/publication_import.py | 8 - .../publication_identifiers/application.py | 90 +- .../publication_identifiers/normalize.py | 2 +- .../domains/publications/application.py | 4 +- app/services/domains/publications/counts.py | 19 +- app/services/domains/publications/dedup.py | 103 + app/services/domains/publications/listing.py | 16 +- .../domains/publications/pdf_queue.py | 140 +- .../publications/pdf_resolution_pipeline.py | 201 +- app/services/domains/publications/queries.py | 56 +- app/services/domains/publications/types.py | 2 - app/services/domains/runs/events.py | 59 + .../domains/scholar/publication_pdf.py | 232 -- app/services/domains/settings/application.py | 15 +- app/services/domains/unpaywall/application.py | 10 +- app/settings.py | 14 +- build/lib/app/__init__.py | 1 + build/lib/app/api/__init__.py | 2 + build/lib/app/api/deps.py | 35 + build/lib/app/api/errors.py | 85 + build/lib/app/api/media.py | 63 + build/lib/app/api/responses.py | 64 + build/lib/app/api/router.py | 14 + build/lib/app/api/routers/__init__.py | 2 + build/lib/app/api/routers/admin.py | 202 ++ build/lib/app/api/routers/admin_dbops.py | 349 +++ build/lib/app/api/routers/auth.py | 237 ++ build/lib/app/api/routers/publications.py | 528 ++++ build/lib/app/api/routers/runs.py | 845 ++++++ build/lib/app/api/routers/scholars.py | 557 ++++ build/lib/app/api/routers/settings.py | 189 ++ build/lib/app/api/runtime_deps.py | 16 + build/lib/app/api/schemas.py | 884 ++++++ build/lib/app/auth/__init__.py | 2 + build/lib/app/auth/deps.py | 27 + build/lib/app/auth/rate_limit.py | 69 + build/lib/app/auth/runtime.py | 55 + build/lib/app/auth/security.py | 19 + build/lib/app/auth/service.py | 38 + build/lib/app/auth/session.py | 50 + build/lib/app/db/__init__.py | 1 + build/lib/app/db/base.py | 27 + build/lib/app/db/models.py | 546 ++++ build/lib/app/db/session.py | 99 + build/lib/app/http/__init__.py | 1 + build/lib/app/http/middleware.py | 205 ++ build/lib/app/logging_config.py | 183 ++ build/lib/app/logging_context.py | 15 + build/lib/app/main.py | 168 ++ build/lib/app/security/__init__.py | 2 + build/lib/app/security/csrf.py | 133 + build/lib/app/services/__init__.py | 2 + build/lib/app/services/domains/__init__.py | 1 + .../app/services/domains/arxiv/application.py | 84 + .../app/services/domains/crossref/__init__.py | 5 + .../services/domains/crossref/application.py | 382 +++ .../app/services/domains/dbops/__init__.py | 5 + .../app/services/domains/dbops/application.py | 364 +++ .../app/services/domains/dbops/integrity.py | 175 ++ build/lib/app/services/domains/dbops/query.py | 22 + .../lib/app/services/domains/doi/normalize.py | 23 + .../services/domains/ingestion/__init__.py | 1 + .../services/domains/ingestion/application.py | 2413 +++++++++++++++++ .../services/domains/ingestion/constants.py | 31 + .../domains/ingestion/fingerprints.py | 175 ++ .../app/services/domains/ingestion/queue.py | 348 +++ .../app/services/domains/ingestion/safety.py | 282 ++ .../services/domains/ingestion/scheduler.py | 622 +++++ .../app/services/domains/ingestion/types.py | 110 + .../app/services/domains/openalex/__init__.py | 5 + .../app/services/domains/openalex/client.py | 135 + .../app/services/domains/openalex/matching.py | 108 + .../app/services/domains/openalex/types.py | 71 + .../services/domains/portability/__init__.py | 1 + .../domains/portability/application.py | 67 + .../services/domains/portability/constants.py | 9 + .../services/domains/portability/exporting.py | 91 + .../services/domains/portability/normalize.py | 109 + .../domains/portability/publication_import.py | 370 +++ .../domains/portability/scholar_import.py | 107 + .../app/services/domains/portability/types.py | 25 + .../publication_identifiers/__init__.py | 17 + .../publication_identifiers/application.py | 433 +++ .../publication_identifiers/normalize.py | 76 + .../domains/publication_identifiers/types.py | 30 + .../services/domains/publications/__init__.py | 1 + .../domains/publications/application.py | 74 + .../services/domains/publications/counts.py | 90 + .../domains/publications/enrichment.py | 73 + .../services/domains/publications/listing.py | 93 + .../services/domains/publications/modes.py | 14 + .../domains/publications/pdf_queue.py | 915 +++++++ .../publications/pdf_resolution_pipeline.py | 108 + .../services/domains/publications/queries.py | 203 ++ .../domains/publications/read_state.py | 94 + .../services/domains/publications/types.py | 41 + .../lib/app/services/domains/runs/__init__.py | 1 + .../app/services/domains/runs/application.py | 45 + build/lib/app/services/domains/runs/events.py | 59 + .../services/domains/runs/queue_queries.py | 70 + .../services/domains/runs/queue_service.py | 182 ++ .../app/services/domains/runs/runs_service.py | 79 + .../lib/app/services/domains/runs/summary.py | 76 + build/lib/app/services/domains/runs/types.py | 38 + .../app/services/domains/scholar/__init__.py | 1 + .../services/domains/scholar/author_rows.py | 202 ++ .../app/services/domains/scholar/parser.py | 165 ++ .../domains/scholar/parser_constants.py | 87 + .../services/domains/scholar/parser_types.py | 75 + .../services/domains/scholar/parser_utils.py | 42 + .../services/domains/scholar/profile_rows.py | 270 ++ .../services/domains/scholar/rate_limit.py | 18 + .../app/services/domains/scholar/source.py | 235 ++ .../domains/scholar/state_detection.py | 164 ++ .../app/services/domains/scholars/__init__.py | 1 + .../services/domains/scholars/application.py | 974 +++++++ .../services/domains/scholars/constants.py | 70 + .../services/domains/scholars/exceptions.py | 5 + .../services/domains/scholars/search_hints.py | 67 + .../app/services/domains/scholars/uploads.py | 39 + .../services/domains/scholars/validators.py | 38 + .../app/services/domains/settings/__init__.py | 1 + .../services/domains/settings/application.py | 159 ++ .../services/domains/unpaywall/__init__.py | 10 + .../services/domains/unpaywall/application.py | 458 ++++ .../domains/unpaywall/pdf_discovery.py | 156 ++ .../services/domains/unpaywall/rate_limit.py | 18 + .../app/services/domains/users/__init__.py | 1 + .../app/services/domains/users/application.py | 98 + build/lib/app/settings.py | 267 ++ diag_pubs.py | 39 + docs/developer/api-contract.md | 12 + docs/developer/architecture.md | 32 +- docs/developer/ingestion.md | 26 + docs/developer/overview.md | 2 + frontend/src/components/layout/AppNav.vue | 4 + .../components/patterns/RunStatusBadge.vue | 6 + frontend/src/features/admin_dbops/index.ts | 17 +- frontend/src/features/publications/index.ts | 18 +- frontend/src/features/runs/index.ts | 7 + frontend/src/features/scholars/index.ts | 3 +- .../features/settings/SettingsAdminPanel.vue | 64 + frontend/src/features/settings/index.ts | 6 + frontend/src/pages/DashboardPage.vue | 63 +- frontend/src/pages/PublicationsPage.vue | 154 +- frontend/src/pages/RunsPage.vue | 43 +- frontend/src/pages/SettingsPage.vue | 56 + frontend/src/stores/run_status.test.ts | 88 +- frontend/src/stores/run_status.ts | 157 +- frontend/src/stores/user_settings.test.ts | 3 + pyproject.toml | 2 + test_enrich.py | 42 + tests/integration/test_api_v1.py | 238 +- tests/integration/test_deferred_enrichment.py | 103 + tests/integration/test_fixture_probe_runs.py | 4 +- tests/integration/test_migrations.py | 24 +- .../test_run_lifecycle_consistency.py | 514 ++++ .../services/domains/openalex/test_client.py | 64 + .../domains/openalex/test_matching.py | 55 + .../domains/publications/test_dedup.py | 163 ++ tests/unit/test_fingerprints.py | 242 ++ tests/unit/test_publication_identifiers.py | 69 + .../unit/test_publication_pdf_queue_policy.py | 72 +- ...est_publication_pdf_resolution_pipeline.py | 149 +- tests/unit/test_scholar_publication_pdf.py | 76 - tests/unit/test_unpaywall_resolution.py | 19 +- uv.lock | 76 + 193 files changed, 23228 insertions(+), 935 deletions(-) create mode 100644 alembic/versions/20260224_0018_add_openalex_enriched_to_publications.py create mode 100644 alembic/versions/20260224_0019_add_resolving_to_runstatus.py create mode 100644 alembic/versions/20260224_0020_add_openalex_last_attempt_at.py create mode 100644 alembic/versions/20260225_0021_enforce_single_active_run_per_user.py create mode 100644 alembic/versions/20260225_0022_add_canonical_title_hash_to_publications.py create mode 100644 alembic/versions/44f7e10ef777_remove_doi_from_publications_and_.py create mode 100644 alembic/versions/fa0c5e3262d5_add_api_integration_keys_to_usersetting.py create mode 100644 app/services/domains/arxiv/application.py create mode 100644 app/services/domains/openalex/__init__.py create mode 100644 app/services/domains/openalex/client.py create mode 100644 app/services/domains/openalex/matching.py create mode 100644 app/services/domains/openalex/types.py create mode 100644 app/services/domains/publications/dedup.py create mode 100644 app/services/domains/runs/events.py delete mode 100644 app/services/domains/scholar/publication_pdf.py create mode 100644 build/lib/app/__init__.py create mode 100644 build/lib/app/api/__init__.py create mode 100644 build/lib/app/api/deps.py create mode 100644 build/lib/app/api/errors.py create mode 100644 build/lib/app/api/media.py create mode 100644 build/lib/app/api/responses.py create mode 100644 build/lib/app/api/router.py create mode 100644 build/lib/app/api/routers/__init__.py create mode 100644 build/lib/app/api/routers/admin.py create mode 100644 build/lib/app/api/routers/admin_dbops.py create mode 100644 build/lib/app/api/routers/auth.py create mode 100644 build/lib/app/api/routers/publications.py create mode 100644 build/lib/app/api/routers/runs.py create mode 100644 build/lib/app/api/routers/scholars.py create mode 100644 build/lib/app/api/routers/settings.py create mode 100644 build/lib/app/api/runtime_deps.py create mode 100644 build/lib/app/api/schemas.py create mode 100644 build/lib/app/auth/__init__.py create mode 100644 build/lib/app/auth/deps.py create mode 100644 build/lib/app/auth/rate_limit.py create mode 100644 build/lib/app/auth/runtime.py create mode 100644 build/lib/app/auth/security.py create mode 100644 build/lib/app/auth/service.py create mode 100644 build/lib/app/auth/session.py create mode 100644 build/lib/app/db/__init__.py create mode 100644 build/lib/app/db/base.py create mode 100644 build/lib/app/db/models.py create mode 100644 build/lib/app/db/session.py create mode 100644 build/lib/app/http/__init__.py create mode 100644 build/lib/app/http/middleware.py create mode 100644 build/lib/app/logging_config.py create mode 100644 build/lib/app/logging_context.py create mode 100644 build/lib/app/main.py create mode 100644 build/lib/app/security/__init__.py create mode 100644 build/lib/app/security/csrf.py create mode 100644 build/lib/app/services/__init__.py create mode 100644 build/lib/app/services/domains/__init__.py create mode 100644 build/lib/app/services/domains/arxiv/application.py create mode 100644 build/lib/app/services/domains/crossref/__init__.py create mode 100644 build/lib/app/services/domains/crossref/application.py create mode 100644 build/lib/app/services/domains/dbops/__init__.py create mode 100644 build/lib/app/services/domains/dbops/application.py create mode 100644 build/lib/app/services/domains/dbops/integrity.py create mode 100644 build/lib/app/services/domains/dbops/query.py create mode 100644 build/lib/app/services/domains/doi/normalize.py create mode 100644 build/lib/app/services/domains/ingestion/__init__.py create mode 100644 build/lib/app/services/domains/ingestion/application.py create mode 100644 build/lib/app/services/domains/ingestion/constants.py create mode 100644 build/lib/app/services/domains/ingestion/fingerprints.py create mode 100644 build/lib/app/services/domains/ingestion/queue.py create mode 100644 build/lib/app/services/domains/ingestion/safety.py create mode 100644 build/lib/app/services/domains/ingestion/scheduler.py create mode 100644 build/lib/app/services/domains/ingestion/types.py create mode 100644 build/lib/app/services/domains/openalex/__init__.py create mode 100644 build/lib/app/services/domains/openalex/client.py create mode 100644 build/lib/app/services/domains/openalex/matching.py create mode 100644 build/lib/app/services/domains/openalex/types.py create mode 100644 build/lib/app/services/domains/portability/__init__.py create mode 100644 build/lib/app/services/domains/portability/application.py create mode 100644 build/lib/app/services/domains/portability/constants.py create mode 100644 build/lib/app/services/domains/portability/exporting.py create mode 100644 build/lib/app/services/domains/portability/normalize.py create mode 100644 build/lib/app/services/domains/portability/publication_import.py create mode 100644 build/lib/app/services/domains/portability/scholar_import.py create mode 100644 build/lib/app/services/domains/portability/types.py create mode 100644 build/lib/app/services/domains/publication_identifiers/__init__.py create mode 100644 build/lib/app/services/domains/publication_identifiers/application.py create mode 100644 build/lib/app/services/domains/publication_identifiers/normalize.py create mode 100644 build/lib/app/services/domains/publication_identifiers/types.py create mode 100644 build/lib/app/services/domains/publications/__init__.py create mode 100644 build/lib/app/services/domains/publications/application.py create mode 100644 build/lib/app/services/domains/publications/counts.py create mode 100644 build/lib/app/services/domains/publications/enrichment.py create mode 100644 build/lib/app/services/domains/publications/listing.py create mode 100644 build/lib/app/services/domains/publications/modes.py create mode 100644 build/lib/app/services/domains/publications/pdf_queue.py create mode 100644 build/lib/app/services/domains/publications/pdf_resolution_pipeline.py create mode 100644 build/lib/app/services/domains/publications/queries.py create mode 100644 build/lib/app/services/domains/publications/read_state.py create mode 100644 build/lib/app/services/domains/publications/types.py create mode 100644 build/lib/app/services/domains/runs/__init__.py create mode 100644 build/lib/app/services/domains/runs/application.py create mode 100644 build/lib/app/services/domains/runs/events.py create mode 100644 build/lib/app/services/domains/runs/queue_queries.py create mode 100644 build/lib/app/services/domains/runs/queue_service.py create mode 100644 build/lib/app/services/domains/runs/runs_service.py create mode 100644 build/lib/app/services/domains/runs/summary.py create mode 100644 build/lib/app/services/domains/runs/types.py create mode 100644 build/lib/app/services/domains/scholar/__init__.py create mode 100644 build/lib/app/services/domains/scholar/author_rows.py create mode 100644 build/lib/app/services/domains/scholar/parser.py create mode 100644 build/lib/app/services/domains/scholar/parser_constants.py create mode 100644 build/lib/app/services/domains/scholar/parser_types.py create mode 100644 build/lib/app/services/domains/scholar/parser_utils.py create mode 100644 build/lib/app/services/domains/scholar/profile_rows.py create mode 100644 build/lib/app/services/domains/scholar/rate_limit.py create mode 100644 build/lib/app/services/domains/scholar/source.py create mode 100644 build/lib/app/services/domains/scholar/state_detection.py create mode 100644 build/lib/app/services/domains/scholars/__init__.py create mode 100644 build/lib/app/services/domains/scholars/application.py create mode 100644 build/lib/app/services/domains/scholars/constants.py create mode 100644 build/lib/app/services/domains/scholars/exceptions.py create mode 100644 build/lib/app/services/domains/scholars/search_hints.py create mode 100644 build/lib/app/services/domains/scholars/uploads.py create mode 100644 build/lib/app/services/domains/scholars/validators.py create mode 100644 build/lib/app/services/domains/settings/__init__.py create mode 100644 build/lib/app/services/domains/settings/application.py create mode 100644 build/lib/app/services/domains/unpaywall/__init__.py create mode 100644 build/lib/app/services/domains/unpaywall/application.py create mode 100644 build/lib/app/services/domains/unpaywall/pdf_discovery.py create mode 100644 build/lib/app/services/domains/unpaywall/rate_limit.py create mode 100644 build/lib/app/services/domains/users/__init__.py create mode 100644 build/lib/app/services/domains/users/application.py create mode 100644 build/lib/app/settings.py create mode 100644 diag_pubs.py create mode 100644 docs/developer/api-contract.md create mode 100644 docs/developer/ingestion.md create mode 100644 test_enrich.py create mode 100644 tests/integration/test_deferred_enrichment.py create mode 100644 tests/integration/test_run_lifecycle_consistency.py create mode 100644 tests/unit/services/domains/openalex/test_client.py create mode 100644 tests/unit/services/domains/openalex/test_matching.py create mode 100644 tests/unit/services/domains/publications/test_dedup.py create mode 100644 tests/unit/test_fingerprints.py delete mode 100644 tests/unit/test_scholar_publication_pdf.py diff --git a/README.md b/README.md index 5d5f84f..c05beaa 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@
      -Self-hosted scholar tracking with a single app image (API + frontend). +Self-hosted scholar tracking with a single app image (API + frontend). It automates publication discovery, parses Google Scholar, gathers external metadata via Crossref & arXiv APIs, and autonomously downloads Open Access PDFs. [![CI](https://img.shields.io/github/actions/workflow/status/justinzeus/scholarr/ci.yml?style=for-the-badge)](https://github.com/JustinZeus/scholarr/actions/workflows/ci.yml) [![Docker Pulls](https://img.shields.io/docker/pulls/justinzeus/scholarr?style=for-the-badge&logo=docker)](https://hub.docker.com/r/justinzeus/scholarr) @@ -10,6 +10,25 @@ Self-hosted scholar tracking with a single app image (API + frontend).
      +## Key Features +- **Automated Ingestion**: Background tasks regularly scrape author profiles to pull the latest publications. +- **Resilient Parsing**: Custom DOM parsing logic falls back heavily across changes, backed by a dynamic `429 Too Many Requests` backoff cycle. +- **Identifier Subsystem**: Isolates DOIs, PMIDs, and arXiv IDs independently of PDFs mapping fallback external APIs sequentially. +- **Unpaywall PDF Search**: Leverages a fully automated resolution queue to grab public PDFs via the Unpaywall API on identified entries. +- **Beautiful UI**: Polished Vue 3 frontend that ships natively inside the same Docker container as the FastAPI backend. + +## Architecture Data Flow +```mermaid +graph LR + HTML(Google Scholar) --> |Scraper| Ingestion + Ingestion --> |Extract DOI| DB[(Database)] + Ingestion -.-> |Fallback API Search| Crossref/arXiv + Crossref/arXiv --> DB + + DB --> |Identified DOIs| PDFQueue(Unpaywall/PDF Workers) + PDFQueue --> |Download & Store File| Storage +``` + ## Quick Start 1. Copy env template: @@ -36,10 +55,12 @@ Open: ## Documentation Complete documentation is published at: - - https://justinzeus.github.io/scholarr/ -Source markdown and docs tooling live under `docs/`. +You can also read the developer/architecture breakdowns directly within the repository under: +- `docs/developer/architecture.md` +- `docs/developer/ingestion.md` +- `docs/developer/api-contract.md` ## Quality Gates @@ -53,11 +74,14 @@ docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv r Frontend: ```bash -cd frontend -npm install -npm run typecheck -npm run test:run -npm run build +# ALL tests/builds MUST be run within the docker container. Do not run raw npm commands on the host. +# For automated agents checking outputs, redirect stdout/stderr to files (e.g. > /tmp/npm_typecheck.log 2>&1) +# The temp files /tmp/npm_test.log, /tmp/npm_lint.log, /tmp/npm_typecheck.log, and /tmp/pytest_eval.log are pre-authorized for overwrite. + +docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm install +docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run typecheck +docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run test:run +docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run build ``` Contract and env checks: @@ -67,3 +91,4 @@ python3 scripts/check_frontend_api_contract.py python3 scripts/check_env_contract.py ./scripts/check_no_generated_artifacts.sh ``` +thanks arxiv \ No newline at end of file diff --git a/agents.md b/agents.md index da59b76..a114a45 100644 --- a/agents.md +++ b/agents.md @@ -3,35 +3,45 @@ Adhere strictly to these constraints. ## 1. Coding Standards (Strict Enforcement) -* **Function Length:** Maximum 50 lines of code per function. Break down complex logic into small, testable, single-responsibility functions. -* **DRY (Don't Repeat Yourself):** Abstract repetitive logic immediately. No duplicate boilerplate for database queries, API responses, or error handling. -* **Negative Space Programming:** Utilize explicit assertions and constraints to define invalid states. Fail fast and early. Do not allow silent failures or cascading malformed data, especially in DOM parsing. -* **Cyclomatic Complexity:** Flatten logic. Use early returns and guard clauses instead of deep nesting. -no magic numbers + +- **Function Length:** Maximum 50 lines of code per function. Break down complex logic into small, testable, single-responsibility functions. +- **DRY (Don't Repeat Yourself):** Abstract repetitive logic immediately. No duplicate boilerplate for database queries, API responses, or error handling. +- **Negative Space Programming:** Utilize explicit assertions and constraints to define invalid states. Fail fast and early. Do not allow silent failures or cascading malformed data, especially in DOM parsing. +- **Cyclomatic Complexity:** Flatten logic. Use early returns and guard clauses instead of deep nesting. + no magic numbers ## 2. Domain Architecture & Data Model -* **Data Isolation:** Scholar tracking is **user-scoped**. Validate mapping/join tables; never assume global links between users and Scholar IDs. -* **Data Deduplication:** Publications are **global records**. Deduplicate via Scholar cluster ID and normalized fingerprinting prior to database insertion. -* **State Management:** Visibility and "read/unread" states exist exclusively on the scholar-publication link table, not the global publication table. -* **API Contract:** Exact envelope format required: - * Success: `{"data": ..., "meta": {"request_id": "..."}}` - * Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}` + +- **Data Isolation:** Scholar tracking is **user-scoped**. Validate mapping/join tables; never assume global links between users and Scholar IDs. +- **Data Deduplication:** Publications are **global records**. Deduplicate via Scholar cluster ID and normalized fingerprinting prior to database insertion. +- **State Management:** Visibility and "read/unread" states exist exclusively on the scholar-publication link table, not the global publication table. +- **API Contract:** Exact envelope format required: + - Success: `{"data": ..., "meta": {"request_id": "..."}}` + - Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}` ## 3. Scrape Safety & Rate Limiting (Immutable) + These limits prevent IP bans and are not to be optimized away. -* **Minimum Delay:** Enforce `INGESTION_MIN_REQUEST_DELAY_SECONDS` (default 2s) between all external requests. -* **Anti-Detection:** Default to direct ID or profile URL ingestion. Name searches trigger CAPTCHAs. -* **Cooldowns:** Respect `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS` (1800s) and `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS` (900s) upon threshold breaches. + +- **Minimum Delay:** Enforce `INGESTION_MIN_REQUEST_DELAY_SECONDS` (default 2s) between all external requests. +- **Anti-Detection:** Default to direct ID or profile URL ingestion. Name searches trigger CAPTCHAs. +- **Cooldowns:** Respect `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS` (1800s) and `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS` (900s) upon threshold breaches. ## 4. Current Environment & Stack -* **Backend:** Python 3.12+, FastAPI, SQLAlchemy (Async/asyncpg), Alembic. -* **Frontend:** TypeScript, Vue 3, Vite. -* **Infrastructure:** Multi-stage Docker. + +- **Backend:** Python 3.12+, FastAPI, SQLAlchemy (Async/asyncpg), Alembic. +- **Frontend:** TypeScript, Vue 3, Vite. +- **Infrastructure:** Multi-stage Docker. ## 5. Domain Service Boundaries -* **Strict Modularity:** Flat files in the `app/services/` root are strictly prohibited. All business logic and routing must reside exclusively within `app/services/domains/`. +- **Strict Modularity:** Flat files in the `app/services/` root are strictly prohibited. All business logic and routing must reside exclusively within `app/services/domains/`. ## 6. UI rules + Make sure to properly integrate tailwind in combination with the preset theming -Clarity through both styling and language are a priority. all UI elements need to have a proper reason for existing. \ No newline at end of file +Clarity through both styling and language are a priority. all UI elements need to have a proper reason for existing. + +## 7. Testing + +All tests need to be ran using containers. `docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app` diff --git a/alembic/versions/20260224_0018_add_openalex_enriched_to_publications.py b/alembic/versions/20260224_0018_add_openalex_enriched_to_publications.py new file mode 100644 index 0000000..1d7ecad --- /dev/null +++ b/alembic/versions/20260224_0018_add_openalex_enriched_to_publications.py @@ -0,0 +1,30 @@ +"""Add openalex_enriched to publications + +Revision ID: 20260224_0018 +Revises: fa0c5e3262d5 +Create Date: 2026-02-24 19:10:00.000000 + +""" +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '20260224_0018' +down_revision: str | Sequence[str] | None = 'fa0c5e3262d5' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('publications', sa.Column('openalex_enriched', sa.Boolean(), server_default=sa.text('false'), nullable=False)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('publications', 'openalex_enriched') + # ### end Alembic commands ### diff --git a/alembic/versions/20260224_0019_add_resolving_to_runstatus.py b/alembic/versions/20260224_0019_add_resolving_to_runstatus.py new file mode 100644 index 0000000..645a32b --- /dev/null +++ b/alembic/versions/20260224_0019_add_resolving_to_runstatus.py @@ -0,0 +1,31 @@ +"""Add resolving to RunStatus enum + +Revision ID: 20260224_0019 +Revises: 20260224_0018 +Create Date: 2026-02-24 22:20:00.000000 + +""" +from collections.abc import Sequence + +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = '20260224_0019' +down_revision: str | Sequence[str] | None = '20260224_0018' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # Use COMMIT to break out of the transaction block that Alembic creates in env.py. + # Postgres restricts ALTER TYPE ... ADD VALUE inside transactions when the type is in use. + # This is the robust, non-patchwork way to ensure the schema evolves correctly. + op.execute("COMMIT") + op.execute("ALTER TYPE run_status ADD VALUE IF NOT EXISTS 'resolving' AFTER 'running'") + + +def downgrade() -> None: + # Enum values cannot be easily removed in Postgres without recreating the type, + # which is risky and usually avoided in simple migrations. + pass diff --git a/alembic/versions/20260224_0020_add_openalex_last_attempt_at.py b/alembic/versions/20260224_0020_add_openalex_last_attempt_at.py new file mode 100644 index 0000000..024f547 --- /dev/null +++ b/alembic/versions/20260224_0020_add_openalex_last_attempt_at.py @@ -0,0 +1,26 @@ +"""Add openalex_last_attempt_at to publications + +Revision ID: 20260224_0020 +Revises: 20260224_0019 +Create Date: 2026-02-24 22:50:00.000000 + +""" +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '20260224_0020' +down_revision: str | Sequence[str] | None = '20260224_0019' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column('publications', sa.Column('openalex_last_attempt_at', sa.DateTime(timezone=True), nullable=True)) + + +def downgrade() -> None: + op.drop_column('publications', 'openalex_last_attempt_at') diff --git a/alembic/versions/20260225_0021_enforce_single_active_run_per_user.py b/alembic/versions/20260225_0021_enforce_single_active_run_per_user.py new file mode 100644 index 0000000..daaa2a6 --- /dev/null +++ b/alembic/versions/20260225_0021_enforce_single_active_run_per_user.py @@ -0,0 +1,70 @@ +"""Enforce one active crawl run per user. + +Revision ID: 20260225_0021 +Revises: 20260224_0020 +Create Date: 2026-02-25 09:10:00.000000 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "20260225_0021" +down_revision: str | Sequence[str] | None = "20260224_0020" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +INDEX_NAME = "uq_crawl_runs_user_active" +ACTIVE_STATUSES = ("running", "resolving") + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + indexes = {index["name"] for index in inspector.get_indexes("crawl_runs")} + + op.execute("ALTER TYPE run_status ADD VALUE IF NOT EXISTS 'canceled'") + + op.execute( + """ + WITH ranked AS ( + SELECT + id, + ROW_NUMBER() OVER ( + PARTITION BY user_id + ORDER BY start_dt DESC, id DESC + ) AS rn + FROM crawl_runs + WHERE status IN ('running', 'resolving') + ) + UPDATE crawl_runs AS runs + SET + status = 'failed', + end_dt = COALESCE(runs.end_dt, NOW()) + FROM ranked + WHERE runs.id = ranked.id + AND ranked.rn > 1 + """ + ) + + if INDEX_NAME not in indexes: + op.create_index( + INDEX_NAME, + "crawl_runs", + ["user_id"], + unique=True, + postgresql_where=sa.text( + "status IN ('running'::run_status, 'resolving'::run_status)" + ), + ) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + indexes = {index["name"] for index in inspector.get_indexes("crawl_runs")} + if INDEX_NAME in indexes: + op.drop_index(INDEX_NAME, table_name="crawl_runs") diff --git a/alembic/versions/20260225_0022_add_canonical_title_hash_to_publications.py b/alembic/versions/20260225_0022_add_canonical_title_hash_to_publications.py new file mode 100644 index 0000000..f73d280 --- /dev/null +++ b/alembic/versions/20260225_0022_add_canonical_title_hash_to_publications.py @@ -0,0 +1,35 @@ +"""Add canonical_title_hash to publications for cross-scholar dedup. + +Revision ID: 20260225_0022 +Revises: 20260225_0021 +Create Date: 2026-02-25 10:00:00.000000 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "20260225_0022" +down_revision: str | Sequence[str] | None = "20260225_0021" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column( + "publications", + sa.Column("canonical_title_hash", sa.String(64), nullable=True), + ) + op.create_index( + "ix_publications_canonical_title_hash", + "publications", + ["canonical_title_hash"], + ) + + +def downgrade() -> None: + op.drop_index("ix_publications_canonical_title_hash", table_name="publications") + op.drop_column("publications", "canonical_title_hash") diff --git a/alembic/versions/44f7e10ef777_remove_doi_from_publications_and_.py b/alembic/versions/44f7e10ef777_remove_doi_from_publications_and_.py new file mode 100644 index 0000000..f47adbd --- /dev/null +++ b/alembic/versions/44f7e10ef777_remove_doi_from_publications_and_.py @@ -0,0 +1,27 @@ +"""remove_doi_from_publications_and_migrate_identifiers + +Revision ID: 44f7e10ef777 +Revises: 20260222_0016 +Create Date: 2026-02-22 17:03:56.261936 + +""" +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '44f7e10ef777' +down_revision: str | Sequence[str] | None = '20260222_0016' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass + diff --git a/alembic/versions/fa0c5e3262d5_add_api_integration_keys_to_usersetting.py b/alembic/versions/fa0c5e3262d5_add_api_integration_keys_to_usersetting.py new file mode 100644 index 0000000..627a18a --- /dev/null +++ b/alembic/versions/fa0c5e3262d5_add_api_integration_keys_to_usersetting.py @@ -0,0 +1,39 @@ +"""Add API integration keys to UserSetting + +Revision ID: fa0c5e3262d5 +Revises: 44f7e10ef777 +Create Date: 2026-02-23 14:25:45.021512 + +""" +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'fa0c5e3262d5' +down_revision: str | Sequence[str] | None = '44f7e10ef777' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_publications_doi'), table_name='publications') + op.drop_column('publications', 'doi') + op.add_column('user_settings', sa.Column('openalex_api_key', sa.String(length=255), nullable=True)) + op.add_column('user_settings', sa.Column('crossref_api_token', sa.String(length=255), nullable=True)) + op.add_column('user_settings', sa.Column('crossref_api_mailto', sa.String(length=255), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('user_settings', 'crossref_api_mailto') + op.drop_column('user_settings', 'crossref_api_token') + op.drop_column('user_settings', 'openalex_api_key') + op.add_column('publications', sa.Column('doi', sa.VARCHAR(length=255), autoincrement=False, nullable=True)) + op.create_index(op.f('ix_publications_doi'), 'publications', ['doi'], unique=False) + # ### end Alembic commands ### + diff --git a/app/api/routers/admin_dbops.py b/app/api/routers/admin_dbops.py index cd6d924..b3c1fdf 100644 --- a/app/api/routers/admin_dbops.py +++ b/app/api/routers/admin_dbops.py @@ -57,7 +57,6 @@ def _serialize_pdf_queue_item(item) -> dict[str, object]: return { "publication_id": item.publication_id, "title": item.title, - "doi": item.doi, "display_identifier": _serialize_display_identifier(item.display_identifier), "pdf_url": item.pdf_url, "status": item.status, @@ -348,3 +347,64 @@ async def trigger_publication_link_repair( }, ) return success_payload(request, data=result) + + +DROP_PUBLICATIONS_CONFIRMATION = "DROP ALL PUBLICATIONS" + + +@router.post("/drop-all-publications") +async def drop_all_publications( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + body = await request.json() + confirmation_text = (body.get("confirmation_text") or "").strip() + if confirmation_text != DROP_PUBLICATIONS_CONFIRMATION: + raise ApiException( + status_code=400, + code="confirmation_required", + message=f"Type '{DROP_PUBLICATIONS_CONFIRMATION}' to confirm this destructive action.", + ) + + from sqlalchemy import delete, func, select, update + + from app.db.models import ( + Publication, + PublicationIdentifier, + PublicationPdfJob, + PublicationPdfJobEvent, + ScholarProfile, + ScholarPublication, + ) + + count_result = await db_session.execute(select(func.count()).select_from(Publication)) + total_publications = count_result.scalar_one() + + await db_session.execute(delete(ScholarPublication)) + await db_session.execute(delete(PublicationIdentifier)) + await db_session.execute(delete(PublicationPdfJobEvent)) + await db_session.execute(delete(PublicationPdfJob)) + await db_session.execute(delete(Publication)) + await db_session.execute( + update(ScholarProfile).values(baseline_completed=False) + ) + await db_session.commit() + + logger.warning( + "api.admin.db.all_publications_dropped", + extra={ + "event": "api.admin.db.all_publications_dropped", + "admin_user_id": int(admin_user.id), + "admin_email": admin_user.email, + "deleted_count": int(total_publications), + }, + ) + return success_payload( + request, + data={ + "deleted_count": int(total_publications), + "message": f"Dropped {total_publications} publication(s) and all related data. " + "Scholar baselines have been reset; the next run will re-discover all publications.", + }, + ) diff --git a/app/api/routers/publications.py b/app/api/routers/publications.py index ad8fc55..b4a616e 100644 --- a/app/api/routers/publications.py +++ b/app/api/routers/publications.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime, timezone import logging from typing import Literal @@ -62,7 +63,6 @@ def _serialize_publication_item(item) -> dict[str, object]: "citation_count": item.citation_count, "venue_text": item.venue_text, "pub_url": item.pub_url, - "doi": item.doi, "display_identifier": _serialize_display_identifier(item.display_identifier), "pdf_url": item.pdf_url, "pdf_status": item.pdf_status, @@ -94,23 +94,27 @@ async def _publication_counts( user_id: int, selected_scholar_id: int | None, favorite_only: bool, + snapshot_before: datetime | None, ) -> tuple[int, int, int, int]: unread_count = await publication_service.count_unread_for_user( db_session, user_id=user_id, scholar_profile_id=selected_scholar_id, favorite_only=favorite_only, + snapshot_before=snapshot_before, ) favorites_count = await publication_service.count_favorite_for_user( db_session, user_id=user_id, scholar_profile_id=selected_scholar_id, + snapshot_before=snapshot_before, ) latest_count = await publication_service.count_latest_for_user( db_session, user_id=user_id, scholar_profile_id=selected_scholar_id, favorite_only=favorite_only, + snapshot_before=snapshot_before, ) total_count = await publication_service.count_for_user( db_session, @@ -118,6 +122,7 @@ async def _publication_counts( mode=publication_service.MODE_ALL, scholar_profile_id=selected_scholar_id, favorite_only=favorite_only, + snapshot_before=snapshot_before, ) return unread_count, favorites_count, latest_count, total_count @@ -129,8 +134,12 @@ async def _list_publications_for_request( mode: Literal["all", "unread", "latest", "new"] | None, favorite_only: bool, scholar_profile_id: int | None, + search: str | None, + sort_by: str, + sort_dir: str, limit: int, offset: int, + snapshot_before: datetime | None, ) -> tuple[str, int | None, list]: resolved_mode = publication_service.resolve_publication_view_mode(mode) selected_scholar_id = scholar_profile_id @@ -145,8 +154,12 @@ async def _list_publications_for_request( mode=resolved_mode, scholar_profile_id=selected_scholar_id, favorite_only=favorite_only, + search=search, + sort_by=sort_by, + sort_dir=sort_dir, limit=limit, offset=offset, + snapshot_before=snapshot_before, ) await publication_service.schedule_missing_pdf_enrichment_for_user( db_session, @@ -162,6 +175,27 @@ async def _list_publications_for_request( return resolved_mode, selected_scholar_id, hydrated +def _resolve_publications_snapshot( + *, + snapshot: str | None, +) -> tuple[datetime, str]: + if snapshot is None: + now_utc = datetime.now(timezone.utc) + return now_utc, now_utc.isoformat() + try: + parsed = datetime.fromisoformat(snapshot) + except ValueError as exc: + raise ApiException( + status_code=400, + code="invalid_snapshot", + message="Invalid publications snapshot cursor.", + ) from exc + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + normalized = parsed.astimezone(timezone.utc) + return normalized, normalized.isoformat() + + def _resolve_publications_paging( *, page: int, @@ -188,6 +222,7 @@ def _publications_list_data( page: int, page_size: int, offset: int, + snapshot: str, ) -> dict[str, object]: return { "mode": mode, @@ -200,6 +235,7 @@ def _publications_list_data( "total_count": total_count, "page": int(page), "page_size": int(page_size), + "snapshot": snapshot, "has_prev": int(offset) > 0, "has_next": int(offset) + int(page_size) < int(total_count), "publications": [_serialize_publication_item(item) for item in publications], @@ -322,7 +358,6 @@ def _log_retry_pdf_result( "queued": queued, "resolved_pdf": resolved_pdf, "pdf_status": pdf_status, - "has_doi": bool(doi), }, ) @@ -336,10 +371,14 @@ async def list_publications( mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None), favorite_only: bool = Query(default=False), scholar_profile_id: int | None = Query(default=None, ge=1), + search: str | None = Query(default=None, min_length=1, max_length=200), + sort_by: Literal["first_seen", "title", "year", "citations", "scholar"] = Query(default="first_seen"), + sort_dir: Literal["asc", "desc"] = Query(default="desc"), page: int = Query(default=1, ge=1), page_size: int = Query(default=100, ge=1, le=500), limit: int | None = Query(default=None, ge=1, le=1000), offset: int | None = Query(default=None, ge=0), + snapshot: str | None = Query(default=None, min_length=1, max_length=64), db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): @@ -349,20 +388,27 @@ async def list_publications( limit=limit, offset=offset, ) + snapshot_before, snapshot_cursor = _resolve_publications_snapshot(snapshot=snapshot) + normalized_search = (search or "").strip() or None resolved_mode, selected_scholar_id, publications = await _list_publications_for_request( db_session, current_user=current_user, mode=mode, favorite_only=favorite_only, scholar_profile_id=scholar_profile_id, + search=normalized_search, + sort_by=sort_by, + sort_dir=sort_dir, limit=resolved_limit, offset=resolved_offset, + snapshot_before=snapshot_before, ) unread_count, favorites_count, latest_count, total_count = await _publication_counts( db_session, user_id=current_user.id, selected_scholar_id=selected_scholar_id, favorite_only=favorite_only, + snapshot_before=snapshot_before, ) data = _publications_list_data( mode=resolved_mode, @@ -376,6 +422,7 @@ async def list_publications( page=resolved_page, page_size=resolved_limit, offset=resolved_offset, + snapshot=snapshot_cursor, ) return success_payload(request, data=data) @@ -480,7 +527,7 @@ async def retry_publication_pdf( queued=queued, resolved_pdf=resolved_pdf, pdf_status=current.pdf_status, - doi=current.doi, + doi=None, ) return success_payload( request, diff --git a/app/api/routers/runs.py b/app/api/routers/runs.py index ddd5290..b03a45d 100644 --- a/app/api/routers/runs.py +++ b/app/api/routers/runs.py @@ -11,6 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_api_current_user from app.api.errors import ApiException from app.api.responses import success_payload +from fastapi.responses import StreamingResponse from app.api.schemas import ( ManualRunEnvelope, QueueClearEnvelope, @@ -24,6 +25,7 @@ from app.db.session import get_db_session from app.services.domains.ingestion import application as ingestion_service from app.services.domains.ingestion import safety as run_safety_service from app.services.domains.runs import application as run_service +from app.services.domains.runs.events import event_generator from app.services.domains.settings import application as user_settings_service from app.settings import settings from app.api.runtime_deps import get_ingestion_service @@ -31,6 +33,7 @@ from app.api.runtime_deps import get_ingestion_service logger = logging.getLogger(__name__) router = APIRouter(prefix="/runs", tags=["api-runs"]) +ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING} IDEMPOTENCY_HEADER = "Idempotency-Key" IDEMPOTENCY_MAX_LENGTH = 128 @@ -338,7 +341,7 @@ async def _reused_manual_run_payload( ) if previous_run is None: return None - if previous_run.status == RunStatus.RUNNING: + if previous_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING): raise ApiException( status_code=409, code="run_in_progress", @@ -410,7 +413,7 @@ async def _recover_integrity_error( extra={"event": "api.runs.manual_integrity_error", "user_id": user_id}, ) raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc - if existing_run.status == RunStatus.RUNNING: + if existing_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING): raise ApiException( status_code=409, code="run_in_progress", @@ -586,6 +589,61 @@ async def get_run( ) +@router.post( + "/{run_id}/cancel", + response_model=RunDetailEnvelope, +) +async def cancel_run( + run_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + run = await run_service.get_run_for_user( + db_session, + user_id=current_user.id, + run_id=run_id, + ) + if run is None: + raise ApiException( + status_code=404, + code="run_not_found", + message="Run not found.", + ) + + if run.status in ACTIVE_RUN_STATUSES: + run.status = RunStatus.CANCELED + await db_session.commit() + await db_session.refresh(run) + else: + raise ApiException( + status_code=409, + code="run_not_cancelable", + message="Run is already terminal and cannot be canceled.", + details={"run_id": int(run.id), "status": run.status.value}, + ) + + error_log = run.error_log if isinstance(run.error_log, dict) else {} + scholar_results = error_log.get("scholar_results") + if not isinstance(scholar_results, list): + scholar_results = [] + + safety_state = await _load_safety_state( + db_session, + user_id=current_user.id, + ) + + return success_payload( + request, + data={ + "run": _serialize_run(run), + "summary": run_service.extract_run_summary(error_log), + "scholar_results": [_normalize_scholar_result(item) for item in scholar_results], + "safety_state": safety_state, + }, + ) + + @router.post( "/manual", response_model=ManualRunEnvelope, @@ -596,10 +654,7 @@ async def run_manual( current_user: User = Depends(get_api_current_user), ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service), ): - safety_state = await _load_safety_state( - db_session, - user_id=current_user.id, - ) + safety_state = await _load_safety_state(db_session, user_id=current_user.id) if not settings.ingestion_manual_run_allowed: _raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state) @@ -614,28 +669,90 @@ async def run_manual( if reused_payload is not None: return reused_payload - run_summary = await _execute_manual_run( - db_session, - request=request, - ingest_service=ingest_service, - user_id=current_user.id, - idempotency_key=idempotency_key, - ) - if isinstance(run_summary, dict): - return run_summary - - current_safety_state = await _load_safety_state( - db_session, - user_id=current_user.id, - ) - return success_payload( - request, - data=_manual_run_success_payload( - run_summary=run_summary, + try: + user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=current_user.id) + + # Initialize run (creates the record and performs safety checks) + run, scholars, start_cstart_map = await ingest_service.initialize_run( + db_session, + user_id=current_user.id, + trigger_type=RunTriggerType.MANUAL, + request_delay_seconds=user_settings.request_delay_seconds, + network_error_retries=settings.ingestion_network_error_retries, + retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, + max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, + page_size=settings.ingestion_page_size, idempotency_key=idempotency_key, - safety_state=current_safety_state, - ), - ) + alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, + alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, + alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, + ) + + await db_session.commit() + + # Kick off background execution + from app.db.session import get_session_factory + import asyncio + + asyncio.create_task( + ingest_service.execute_run( + session_factory=get_session_factory(), + run_id=run.id, + user_id=current_user.id, + scholars=scholars, + start_cstart_map=start_cstart_map, + request_delay_seconds=user_settings.request_delay_seconds, + network_error_retries=settings.ingestion_network_error_retries, + retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, + max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, + page_size=settings.ingestion_page_size, + auto_queue_continuations=settings.ingestion_continuation_queue_enabled, + queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds, + alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, + alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, + alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, + idempotency_key=idempotency_key, + ) + ) + + return success_payload( + request, + data={ + "run_id": int(run.id), + "status": run.status.value, + "scholar_count": int(run.scholar_count or 0), + "succeeded_count": 0, + "failed_count": 0, + "partial_count": 0, + "new_publication_count": 0, + "reused_existing_run": False, + "idempotency_key": idempotency_key, + "safety_state": await _load_safety_state(db_session, user_id=current_user.id), + } + ) + except ingestion_service.RunBlockedBySafetyPolicyError as exc: + await db_session.rollback() + _raise_manual_blocked_safety(exc=exc, user_id=current_user.id) + except ingestion_service.RunAlreadyInProgressError as exc: + await db_session.rollback() + raise ApiException( + status_code=409, + code="run_in_progress", + message="A run is already in progress for this account.", + ) from exc + except IntegrityError as exc: + await db_session.rollback() + return await _recover_integrity_error( + db_session, + request=request, + user_id=current_user.id, + idempotency_key=idempotency_key, + original_exc=exc, + ) + except Exception as exc: + await db_session.rollback() + _raise_manual_failed(exc=exc, user_id=current_user.id) + @router.get( @@ -769,3 +886,26 @@ async def clear_queue_item( "message": "Queue item cleared.", }, ) + + +@router.get("/{run_id}/stream") +async def stream_run_events( + run_id: int, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + run = await run_service.get_run_for_user( + db_session, + user_id=current_user.id, + run_id=run_id, + ) + if run is None: + raise ApiException( + status_code=404, + code="run_not_found", + message="Run not found.", + ) + return StreamingResponse( + event_generator(run_id), + media_type="text/event-stream" + ) diff --git a/app/api/routers/settings.py b/app/api/routers/settings.py index f5e9cac..5a2ef91 100644 --- a/app/api/routers/settings.py +++ b/app/api/routers/settings.py @@ -38,6 +38,9 @@ def _serialize_settings(user_settings) -> dict[str, object]: "cooldown_network_seconds": max(60, int(settings_module.ingestion_safety_cooldown_network_seconds)), }, "safety_state": run_safety_service.get_safety_state_payload(user_settings), + "openalex_api_key": user_settings.openalex_api_key, + "crossref_api_token": user_settings.crossref_api_token, + "crossref_api_mailto": user_settings.crossref_api_mailto, } @@ -107,6 +110,7 @@ def _log_settings_update(*, user_id: int, updated) -> None: "run_interval_minutes": updated.run_interval_minutes, "request_delay_seconds": updated.request_delay_seconds, "nav_visible_pages": updated.nav_visible_pages, + "openalex_api_key": "SET" if updated.openalex_api_key else "UNSET", }, ) @@ -169,6 +173,9 @@ async def update_settings( run_interval_minutes=parsed_interval, request_delay_seconds=parsed_delay, nav_visible_pages=parsed_nav_visible_pages, + openalex_api_key=payload.openalex_api_key, + crossref_api_token=payload.crossref_api_token, + crossref_api_mailto=payload.crossref_api_mailto, ) await _clear_expired_cooldown_with_log( db_session, diff --git a/app/api/schemas.py b/app/api/schemas.py index 112f672..07ccedf 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -205,7 +205,6 @@ class PublicationExportItemData(BaseModel): author_text: str | None = None venue_text: str | None = None pub_url: str | None = None - doi: str | None = None pdf_url: str | None = None is_read: bool = False @@ -594,7 +593,6 @@ class DisplayIdentifierData(BaseModel): class AdminPdfQueueItemData(BaseModel): publication_id: int title: str - doi: str | None display_identifier: DisplayIdentifierData | None = None pdf_url: str | None status: str @@ -725,6 +723,10 @@ class SettingsData(BaseModel): nav_visible_pages: list[str] policy: SettingsPolicyData safety_state: ScrapeSafetyStateData + + openalex_api_key: str | None = None + crossref_api_token: str | None = None + crossref_api_mailto: str | None = None model_config = ConfigDict(extra="forbid") @@ -742,6 +744,10 @@ class SettingsUpdateRequest(BaseModel): request_delay_seconds: int nav_visible_pages: list[str] | None = None + openalex_api_key: str | None = None + crossref_api_token: str | None = None + crossref_api_mailto: str | None = None + model_config = ConfigDict(extra="forbid") @@ -754,7 +760,6 @@ class PublicationItemData(BaseModel): citation_count: int venue_text: str | None pub_url: str | None - doi: str | None display_identifier: DisplayIdentifierData | None = None pdf_url: str | None pdf_status: str = "untracked" @@ -780,6 +785,7 @@ class PublicationsListData(BaseModel): total_count: int page: int page_size: int + snapshot: str has_next: bool = False has_prev: bool = False publications: list[PublicationItemData] diff --git a/app/db/models.py b/app/db/models.py index 1f0b250..c687547 100644 --- a/app/db/models.py +++ b/app/db/models.py @@ -31,9 +31,11 @@ class RunTriggerType(StrEnum): class RunStatus(StrEnum): RUNNING = "running" + RESOLVING = "resolving" SUCCESS = "success" PARTIAL_FAILURE = "partial_failure" FAILED = "failed" + CANCELED = "canceled" class QueueItemStatus(StrEnum): @@ -113,6 +115,11 @@ class UserSetting(Base): ) scrape_cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) scrape_cooldown_reason: Mapped[str | None] = mapped_column(String(64)) + + openalex_api_key: Mapped[str | None] = mapped_column(String(255)) + crossref_api_token: Mapped[str | None] = mapped_column(String(255)) + crossref_api_mailto: Mapped[str | None] = mapped_column(String(255)) + created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) @@ -161,6 +168,14 @@ class CrawlRun(Base): __tablename__ = "crawl_runs" __table_args__ = ( Index("ix_crawl_runs_user_start", "user_id", "start_dt"), + Index( + "uq_crawl_runs_user_active", + "user_id", + unique=True, + postgresql_where=text( + "status IN ('running'::run_status, 'resolving'::run_status)" + ), + ), Index( "uq_crawl_runs_user_manual_idempotency_key", "user_id", @@ -228,8 +243,16 @@ class Publication(Base): author_text: Mapped[str | None] = mapped_column(Text) venue_text: Mapped[str | None] = mapped_column(Text) pub_url: Mapped[str | None] = mapped_column(Text) - doi: Mapped[str | None] = mapped_column(String(255)) pdf_url: Mapped[str | None] = mapped_column(Text) + canonical_title_hash: Mapped[str | None] = mapped_column( + String(64), nullable=True, index=True + ) + openalex_enriched: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) + openalex_last_attempt_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) diff --git a/app/main.py b/app/main.py index 254eb4a..56ddf58 100644 --- a/app/main.py +++ b/app/main.py @@ -66,6 +66,18 @@ def _log_startup_build_marker() -> None: @asynccontextmanager async def lifespan(_: FastAPI): _log_startup_build_marker() + + from app.db.session import get_session_factory + from sqlalchemy import text + try: + session_factory = get_session_factory() + async with session_factory() as session: + await session.execute(text("UPDATE crawl_runs SET status = 'failed' WHERE status::text IN ('running', 'resolving')")) + await session.commit() + logger.info("app.startup_orphaned_runs_cleaned", extra={"event": "app.startup_orphaned_runs_cleaned"}) + except Exception as e: + logger.error(f"Failed to clean orphaned runs at startup: {e}") + await scheduler_service.start() yield await scheduler_service.stop() diff --git a/app/services/domains/arxiv/application.py b/app/services/domains/arxiv/application.py new file mode 100644 index 0000000..57b1473 --- /dev/null +++ b/app/services/domains/arxiv/application.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import asyncio +import logging +from typing import TYPE_CHECKING +import xml.etree.ElementTree as ET + +import httpx + +from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id +from app.settings import settings + +if TYPE_CHECKING: + from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem + +logger = logging.getLogger(__name__) + +# arXiv API terms: max 1 request per 3 seconds, single connection at a time. +_ARXIV_LOCK = asyncio.Lock() +_ARXIV_MIN_INTERVAL_SECONDS = 4.0 +# Global cooldown: when arXiv returns 429, all batches back off for this long. +_ARXIV_RATE_LIMIT_COOLDOWN_SECONDS = 60.0 +_arxiv_rate_limited_until: float = 0.0 # asyncio monotonic time + + +class ArxivRateLimitError(Exception): + """arXiv returned 429 — stop the batch to avoid hammering.""" + pass + + +def _build_arxiv_query(title: str, author_surname: str | None) -> str | None: + parts = [] + if title: + clean_title = title.replace('"', '').replace("'", "") + parts.append(f'ti:"{clean_title}"') + if author_surname: + parts.append(f'au:"{author_surname}"') + if not parts: + return None + return " AND ".join(parts) + + +async def discover_arxiv_id_for_publication( + *, + item: PublicationListItem | UnreadPublicationItem, + request_email: str | None = None, + timeout_seconds: float = 3.0, +) -> str | None: + title = (item.title or "").strip() + if not title: + return None + + author_surname = None + if item.scholar_label: + tokens = [t for t in item.scholar_label.strip().split() if t] + if tokens: + author_surname = tokens[-1].lower() + + query = _build_arxiv_query(title, author_surname) + if not query: + return None + + url = "https://export.arxiv.org/api/query" + params = {"search_query": query, "start": 0, "max_results": 3} + headers = { + "User-Agent": ( + f"scholar-scraper/1.0 " + f"(mailto:{request_email or settings.crossref_api_mailto or 'unknown@example.com'})" + ) + } + + try: + async with _ARXIV_LOCK: + global _arxiv_rate_limited_until + now = asyncio.get_running_loop().time() + if now < _arxiv_rate_limited_until: + remaining = _arxiv_rate_limited_until - now + raise ArxivRateLimitError(f"arXiv global cooldown active ({remaining:.0f}s remaining)") + + async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True, headers=headers) as client: + response = await client.get(url, params=params) + + if response.status_code == 429: + _arxiv_rate_limited_until = asyncio.get_running_loop().time() + _ARXIV_RATE_LIMIT_COOLDOWN_SECONDS + raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch") + + await asyncio.sleep(_ARXIV_MIN_INTERVAL_SECONDS) + + response.raise_for_status() + + root = ET.fromstring(response.text) + namespace = {"atom": "http://www.w3.org/2005/Atom"} + + for entry in root.findall("atom:entry", namespace): + id_elem = entry.find("atom:id", namespace) + if id_elem is not None and id_elem.text: + candidate = str(id_elem.text) + if "/abs/" in candidate: + candidate = candidate.split("/abs/")[-1] + normalized = normalize_arxiv_id(candidate) + if normalized: + logger.debug("arxiv.id_discovered", extra={"event": "arxiv.id_discovered", "arxiv_id": normalized}) + return normalized + + except ArxivRateLimitError: + raise # propagate so the batch loop can stop + except Exception as exc: + logger.debug(f"Failed to query arXiv API: {exc}") + + return None diff --git a/app/services/domains/ingestion/application.py b/app/services/domains/ingestion/application.py index 4cb493d..a313a6f 100644 --- a/app/services/domains/ingestion/application.py +++ b/app/services/domains/ingestion/application.py @@ -1,12 +1,14 @@ from __future__ import annotations import asyncio -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone import hashlib import logging +import re from typing import Any -from sqlalchemy import select, text +from sqlalchemy import or_, select, text +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import ( @@ -37,6 +39,7 @@ from app.services.domains.ingestion.fingerprints import ( build_initial_page_fingerprint, build_publication_fingerprint, build_publication_url, + canonical_title_for_dedup, normalize_title, ) from app.services.domains.ingestion import queue as queue_service @@ -53,6 +56,7 @@ from app.services.domains.ingestion.types import ( ScholarProcessingOutcome, ) from app.services.domains.settings import application as user_settings_service +from app.services.domains.runs.events import run_events from app.services.domains.scholar.parser import ( ParseState, ParsedProfilePage, @@ -64,6 +68,21 @@ from app.services.domains.scholar.source import FetchResult, ScholarSource from app.settings import settings logger = logging.getLogger(__name__) +ACTIVE_RUN_INDEX_NAME = "uq_crawl_runs_user_active" + + +def _is_active_run_integrity_error(exc: IntegrityError) -> bool: + original_error = getattr(exc, "orig", None) + if ACTIVE_RUN_INDEX_NAME in str(exc): + return True + if original_error is None: + return False + if ACTIVE_RUN_INDEX_NAME in str(original_error): + return True + diagnostics = getattr(original_error, "diag", None) + if diagnostics is None: + return False + return getattr(diagnostics, "constraint_name", None) == ACTIVE_RUN_INDEX_NAME def _int_or_default(value: Any, default: int = 0) -> int: @@ -357,8 +376,6 @@ class ScholarIngestionService: scholar.display_name = first_page.profile_name if first_page.profile_image_url: scholar.profile_image_url = first_page.profile_image_url - if paged_parse_result.first_page_fingerprint_sha256: - scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256 scholar.last_initial_page_checked_at = run_dt @staticmethod @@ -522,12 +539,9 @@ class ScholarIngestionService: result_entry: dict[str, Any], has_partial_publication_set: bool, ) -> ScholarProcessingOutcome: - discovered_count = await self._upsert_profile_publications( - db_session, - run=run, - scholar=scholar, - publications=paged_parse_result.publications, - ) + # We no longer aggregate and upsert the publications here. + # Eager DB insertions have already saved and committed them inside `_fetch_and_parse_all_pages_with_retry` and `_paginate_loop`. + discovered_count = paged_parse_result.discovered_publication_count is_partial = ( paged_parse_result.has_more_remaining or paged_parse_result.pagination_truncated_reason is not None @@ -535,6 +549,8 @@ class ScholarIngestionService: ) scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS scholar.last_run_dt = run_dt + if not is_partial and paged_parse_result.first_page_fingerprint_sha256: + scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256 result_entry["outcome"] = "partial" if is_partial else "success" if is_partial: result_entry["debug"] = self._build_failure_debug_context( @@ -661,6 +677,8 @@ class ScholarIngestionService: request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, start_cstart: int, @@ -676,6 +694,8 @@ class ScholarIngestionService: request_delay_seconds=request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size, start_cstart=start_cstart, @@ -700,6 +720,8 @@ class ScholarIngestionService: request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, start_cstart: int, @@ -707,6 +729,7 @@ class ScholarIngestionService: queue_delay_seconds: int, ) -> ScholarProcessingOutcome: run_dt, paged_parse_result, result_entry = await self._fetch_and_prepare_scholar_result( + db_session, run=run, scholar=scholar, user_id=user_id, @@ -714,6 +737,8 @@ class ScholarIngestionService: request_delay_seconds=request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size, ) @@ -740,6 +765,7 @@ class ScholarIngestionService: async def _fetch_and_prepare_scholar_result( self, + db_session: AsyncSession, *, run: CrawlRun, scholar: ScholarProfile, @@ -748,16 +774,22 @@ class ScholarIngestionService: request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, ) -> tuple[datetime, PagedParseResult, dict[str, Any]]: run_dt = datetime.now(timezone.utc) paged_parse_result = await self._fetch_and_parse_all_pages_with_retry( - scholar_id=scholar.scholar_id, + scholar=scholar, + run=run, + db_session=db_session, start_cstart=start_cstart, request_delay_seconds=request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, max_pages=max_pages_per_scholar, page_size=page_size, previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256, @@ -808,7 +840,6 @@ class ScholarIngestionService: progress.succeeded_count += outcome.succeeded_count_delta progress.failed_count += outcome.failed_count_delta progress.partial_count += outcome.partial_count_delta - run.new_pub_count = int(run.new_pub_count or 0) + outcome.discovered_publication_count progress.scholar_results.append(outcome.result_entry) def _unexpected_scholar_exception_outcome( @@ -1049,12 +1080,13 @@ class ScholarIngestionService: idempotency_key: str | None, ) -> None: run.end_dt = datetime.now(timezone.utc) - run.status = self._resolve_run_status( - scholar_count=len(scholars), - succeeded_count=progress.succeeded_count, - failed_count=progress.failed_count, - partial_count=progress.partial_count, - ) + if run.status != RunStatus.CANCELED: + run.status = self._resolve_run_status( + scholar_count=len(scholars), + succeeded_count=progress.succeeded_count, + failed_count=progress.failed_count, + partial_count=progress.partial_count, + ) run.error_log = { "scholar_results": progress.scholar_results, "summary": { @@ -1114,6 +1146,8 @@ class ScholarIngestionService: request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, ) -> dict[str, Any]: @@ -1121,6 +1155,8 @@ class ScholarIngestionService: "request_delay_seconds": request_delay_seconds, "network_error_retries": network_error_retries, "retry_backoff_seconds": retry_backoff_seconds, + "rate_limit_retries": rate_limit_retries, + "rate_limit_backoff_seconds": rate_limit_backoff_seconds, "max_pages_per_scholar": max_pages_per_scholar, "page_size": page_size, } @@ -1155,7 +1191,7 @@ class ScholarIngestionService: new_publication_count=run.new_pub_count, ) - async def _initialize_run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, scholar_profile_ids: set[int] | None, start_cstart_by_scholar_id: dict[int, int] | None, request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, idempotency_key: str | None, alert_blocked_failure_threshold: int, alert_network_failure_threshold: int, alert_retry_scheduled_threshold: int) -> tuple[Any, CrawlRun, list[ScholarProfile], dict[int, int]]: + async def _initialize_run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, scholar_profile_ids: set[int] | None, start_cstart_by_scholar_id: dict[int, int] | None, request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, rate_limit_retries: int, rate_limit_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, idempotency_key: str | None, alert_blocked_failure_threshold: int, alert_network_failure_threshold: int, alert_retry_scheduled_threshold: int) -> tuple[Any, CrawlRun, list[ScholarProfile], dict[int, int]]: user_settings = await self._load_user_settings_for_run( db_session, user_id=user_id, @@ -1230,7 +1266,15 @@ class ScholarIngestionService: idempotency_key=idempotency_key, ) db_session.add(run) - await db_session.flush() + try: + await db_session.flush() + except IntegrityError as exc: + if _is_active_run_integrity_error(exc): + await db_session.rollback() + raise RunAlreadyInProgressError( + f"Run already in progress for user_id={user_id}." + ) from exc + raise return run async def _run_scholar_iteration( @@ -1244,13 +1288,27 @@ class ScholarIngestionService: request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, auto_queue_continuations: bool, queue_delay_seconds: int, ) -> RunProgress: progress = RunProgress() + + # ── Pass 1: first page of every scholar (breadth-first) ────────── + # This ensures all scholars have visible publications as quickly as + # possible, rather than fully paginating one scholar before the next. + first_pass_cstarts: dict[int, int] = {} for index, scholar in enumerate(scholars): + await db_session.refresh(run) + if run.status == RunStatus.CANCELED: + logger.info( + "ingestion.run_canceled", + extra={"event": "ingestion.run_canceled", "run_id": run.id, "user_id": user_id}, + ) + return progress await self._wait_between_scholars(index=index, request_delay_seconds=request_delay_seconds) start_cstart = int(start_cstart_map.get(int(scholar.id), 0)) outcome = await self._process_scholar( @@ -1261,9 +1319,50 @@ class ScholarIngestionService: request_delay_seconds=request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + max_pages_per_scholar=1, page_size=page_size, start_cstart=start_cstart, + auto_queue_continuations=False, + queue_delay_seconds=queue_delay_seconds, + ) + self._apply_outcome_to_progress(progress=progress, run=run, outcome=outcome) + # Track where to resume from for the deep pass + resume_cstart = outcome.result_entry.get("continuation_cstart") + if resume_cstart is not None and int(resume_cstart) > start_cstart: + first_pass_cstarts[int(scholar.id)] = int(resume_cstart) + + # ── Pass 2: remaining pages for each scholar (depth) ───────────── + remaining_max = max(max_pages_per_scholar - 1, 0) + if remaining_max <= 0: + return progress + + for index, scholar in enumerate(scholars): + resume_cstart = first_pass_cstarts.get(int(scholar.id)) + if resume_cstart is None: + continue # first page failed, had no continuation, or was skipped + await db_session.refresh(run) + if run.status == RunStatus.CANCELED: + logger.info( + "ingestion.run_canceled", + extra={"event": "ingestion.run_canceled", "run_id": run.id, "user_id": user_id}, + ) + break + await self._wait_between_scholars(index=index, request_delay_seconds=request_delay_seconds) + outcome = await self._process_scholar( + db_session, + run=run, + scholar=scholar, + user_id=user_id, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + max_pages_per_scholar=remaining_max, + page_size=page_size, + start_cstart=resume_cstart, auto_queue_continuations=auto_queue_continuations, queue_delay_seconds=queue_delay_seconds, ) @@ -1307,17 +1406,50 @@ class ScholarIngestionService: ) return failure_summary, alert_summary - async def run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, request_delay_seconds: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, max_pages_per_scholar: int = 30, page_size: int = 100, scholar_profile_ids: set[int] | None = None, start_cstart_by_scholar_id: dict[int, int] | None = None, auto_queue_continuations: bool = True, queue_delay_seconds: int = 60, idempotency_key: str | None = None, alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3) -> RunExecutionSummary: - effective_request_delay_seconds = self._effective_request_delay_seconds(request_delay_seconds) - if effective_request_delay_seconds != _int_or_default(request_delay_seconds, effective_request_delay_seconds): + async def initialize_run( + self, + db_session: AsyncSession, + *, + user_id: int, + trigger_type: RunTriggerType, + request_delay_seconds: int, + network_error_retries: int = 1, + retry_backoff_seconds: float = 1.0, + rate_limit_retries: int | None = None, + rate_limit_backoff_seconds: float | None = None, + max_pages_per_scholar: int = 30, + page_size: int = 100, + scholar_profile_ids: set[int] | None = None, + start_cstart_by_scholar_id: dict[int, int] | None = None, + idempotency_key: str | None = None, + alert_blocked_failure_threshold: int = 1, + alert_network_failure_threshold: int = 2, + alert_retry_scheduled_threshold: int = 3, + ) -> tuple[CrawlRun, list[ScholarProfile], dict[int, int]]: + effective_delay = self._effective_request_delay_seconds(request_delay_seconds) + if effective_delay != _int_or_default(request_delay_seconds, effective_delay): self._log_request_delay_coercion( user_id=user_id, requested_request_delay_seconds=_int_or_default(request_delay_seconds, 0), - effective_request_delay_seconds=effective_request_delay_seconds, + effective_request_delay_seconds=effective_delay, ) - paging_kwargs = self._paging_kwargs(request_delay_seconds=effective_request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size) - threshold_kwargs = self._threshold_kwargs(alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold) - user_settings, run, scholars, start_cstart_map = await self._initialize_run_for_user( + + paging_kwargs = self._paging_kwargs( + request_delay_seconds=effective_delay, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries if rate_limit_retries is not None else settings.ingestion_rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds if rate_limit_backoff_seconds is not None else settings.ingestion_rate_limit_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + ) + threshold_kwargs = self._threshold_kwargs( + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + + _, run, scholars, start_cstart_map = await self._initialize_run_for_user( db_session, user_id=user_id, trigger_type=trigger_type, @@ -1327,6 +1459,224 @@ class ScholarIngestionService: **paging_kwargs, **threshold_kwargs, ) + return run, scholars, start_cstart_map + + async def execute_run( + self, + session_factory: Any, + *, + run_id: int, + user_id: int, + scholars: list[ScholarProfile], + start_cstart_map: dict[int, int], + request_delay_seconds: int, + network_error_retries: int = 1, + retry_backoff_seconds: float = 1.0, + rate_limit_retries: int | None = None, + rate_limit_backoff_seconds: float | None = None, + max_pages_per_scholar: int = 30, + page_size: int = 100, + auto_queue_continuations: bool = True, + queue_delay_seconds: int = 60, + alert_blocked_failure_threshold: int = 1, + alert_network_failure_threshold: int = 2, + alert_retry_scheduled_threshold: int = 3, + idempotency_key: str | None = None, + ) -> None: + async with session_factory() as db_session: + try: + # Re-fetch everything to ensure attachment to the new session + run, user_settings, attached_scholars = await self._prepare_execute_run( + db_session, run_id=run_id, user_id=user_id, scholars=scholars + ) + + paging_kwargs = self._paging_kwargs( + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries if rate_limit_retries is not None else settings.ingestion_rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds if rate_limit_backoff_seconds is not None else settings.ingestion_rate_limit_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + ) + threshold_kwargs = self._threshold_kwargs( + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + + progress = await self._run_scholar_iteration( + db_session, + run=run, + scholars=attached_scholars, + user_id=user_id, + start_cstart_map=start_cstart_map, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + **paging_kwargs, + ) + + failure_summary, alert_summary = self._complete_run_for_user( + user_settings=user_settings, + run=run, + scholars=attached_scholars, + user_id=user_id, + progress=progress, + idempotency_key=idempotency_key, + **threshold_kwargs, + ) + + # Capture the final status that _finalize_run_record computed, + # then set RESOLVING so the UI shows enrichment is in progress. + intended_final_status = run.status + if intended_final_status not in (RunStatus.CANCELED,): + run.status = RunStatus.RESOLVING + await db_session.commit() + self._log_run_completed(user_id=user_id, run=run, scholars=attached_scholars, progress=progress, alert_summary=alert_summary, failure_summary=failure_summary) + + # Fire-and-forget enrichment in a separate background task + if intended_final_status not in (RunStatus.CANCELED,): + asyncio.create_task( + self._background_enrich( + session_factory, + run_id=run.id, + intended_final_status=intended_final_status, + openalex_api_key=getattr(user_settings, "openalex_api_key", None), + ) + ) + except Exception as exc: + await db_session.rollback() + logger.exception("ingestion.background_run_failed", extra={"event": "ingestion.background_run_failed", "run_id": run_id, "user_id": user_id}) + await self._fail_run_in_background(session_factory, run_id, exc) + + async def _prepare_execute_run( + self, + db_session: AsyncSession, + *, + run_id: int, + user_id: int, + scholars: list[ScholarProfile], + ) -> tuple[CrawlRun, Any, list[ScholarProfile]]: + run_result = await db_session.execute(select(CrawlRun).where(CrawlRun.id == run_id)) + run = run_result.scalar_one() + user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=user_id) + + scholar_ids = [s.id for s in scholars] + scholars_result = await db_session.execute( + select(ScholarProfile).where(ScholarProfile.id.in_(scholar_ids)) + .order_by(ScholarProfile.created_at.asc(), ScholarProfile.id.asc()) + ) + return run, user_settings, list(scholars_result.scalars().all()) + + async def _fail_run_in_background(self, session_factory: Any, run_id: int, exc: Exception) -> None: + async with session_factory() as cleanup_session: + run_to_fail = await cleanup_session.get(CrawlRun, run_id) + if run_to_fail: + run_to_fail.status = RunStatus.FAILED + run_to_fail.end_dt = datetime.now(timezone.utc) + run_to_fail.error_log["terminal_exception"] = str(exc) + await cleanup_session.commit() + + async def _background_enrich( + self, + session_factory: Any, + *, + run_id: int, + intended_final_status: RunStatus, + openalex_api_key: str | None = None, + ) -> None: + """Run enrichment independently — failures don't affect run status.""" + try: + async with session_factory() as session: + await self._enrich_pending_publications( + session, + run_id=run_id, + openalex_api_key=openalex_api_key, + ) + run = await session.get(CrawlRun, run_id) + if run is not None and run.status == RunStatus.RESOLVING: + run.status = intended_final_status + await session.commit() + logger.info( + "ingestion.background_enrichment_complete", + extra={"run_id": run_id, "final_status": str(intended_final_status)}, + ) + except Exception: + logger.exception( + "ingestion.background_enrichment_failed", + extra={"run_id": run_id}, + ) + # Still transition out of RESOLVING so the run doesn't stay stuck. + try: + async with session_factory() as fallback_session: + run = await fallback_session.get(CrawlRun, run_id) + if run is not None and run.status == RunStatus.RESOLVING: + run.status = intended_final_status + await fallback_session.commit() + except Exception: + logger.exception( + "ingestion.background_enrichment_fallback_failed", + extra={"run_id": run_id}, + ) + + async def run_for_user( + self, + db_session: AsyncSession, + *, + user_id: int, + trigger_type: RunTriggerType, + request_delay_seconds: int, + network_error_retries: int = 1, + retry_backoff_seconds: float = 1.0, + rate_limit_retries: int | None = None, + rate_limit_backoff_seconds: float | None = None, + max_pages_per_scholar: int = 30, + page_size: int = 100, + scholar_profile_ids: set[int] | None = None, + start_cstart_by_scholar_id: dict[int, int] | None = None, + auto_queue_continuations: bool = True, + queue_delay_seconds: int = 60, + idempotency_key: str | None = None, + alert_blocked_failure_threshold: int = 1, + alert_network_failure_threshold: int = 2, + alert_retry_scheduled_threshold: int = 3, + ) -> RunExecutionSummary: + # Legacy/Synchronous trigger (used by tests or older flows) + run, scholars, start_cstart_map = await self.initialize_run( + db_session, + user_id=user_id, + trigger_type=trigger_type, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + scholar_profile_ids=scholar_profile_ids, + start_cstart_by_scholar_id=start_cstart_by_scholar_id, + idempotency_key=idempotency_key, + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + + paging_kwargs = self._paging_kwargs( + request_delay_seconds=self._effective_request_delay_seconds(request_delay_seconds), + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries if rate_limit_retries is not None else settings.ingestion_rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds if rate_limit_backoff_seconds is not None else settings.ingestion_rate_limit_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + ) + + threshold_kwargs = self._threshold_kwargs( + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + progress = await self._run_scholar_iteration( db_session, run=run, @@ -1337,6 +1687,8 @@ class ScholarIngestionService: queue_delay_seconds=queue_delay_seconds, **paging_kwargs, ) + + user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=user_id) failure_summary, alert_summary = self._complete_run_for_user( user_settings=user_settings, run=run, @@ -1346,7 +1698,27 @@ class ScholarIngestionService: idempotency_key=idempotency_key, **threshold_kwargs, ) + + # Set RESOLVING during enrichment (synchronous path). + intended_final_status = run.status + if intended_final_status not in (RunStatus.CANCELED,): + run.status = RunStatus.RESOLVING await db_session.commit() + + try: + await self._enrich_pending_publications( + db_session, + run_id=run.id, + openalex_api_key=getattr(user_settings, "openalex_api_key", None), + ) + except Exception: + logger.exception("ingestion.enrichment_failed", extra={"run_id": run.id}) + + # Finalize to the intended status unless the run was canceled during resolving. + if run.status == RunStatus.RESOLVING: + run.status = intended_final_status + await db_session.commit() + self._log_run_completed(user_id=user_id, run=run, scholars=scholars, progress=progress, alert_summary=alert_summary, failure_summary=failure_summary) return self._run_execution_summary(run=run, scholars=scholars, progress=progress) @@ -1416,32 +1788,45 @@ class ScholarIngestionService: } @staticmethod - def _should_retry_network_page( + def _should_retry_page_fetch( *, parsed_page: ParsedProfilePage, - attempt_index: int, - max_attempts: int, + network_attempt_count: int, + rate_limit_attempt_count: int, + network_error_retries: int, + rate_limit_retries: int, ) -> bool: - return parsed_page.state == ParseState.NETWORK_ERROR and attempt_index < max_attempts - 1 + if parsed_page.state == ParseState.NETWORK_ERROR: + return network_attempt_count <= network_error_retries + if parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA and parsed_page.state_reason == "blocked_http_429_rate_limited": + return rate_limit_attempt_count <= rate_limit_retries + return False @staticmethod async def _sleep_retry_backoff( *, scholar_id: str, cstart: int, - attempt_index: int, - backoff: float, + network_attempt_count: int, + rate_limit_attempt_count: int, + retry_backoff_seconds: float, + rate_limit_backoff_seconds: float, state_reason: str, ) -> None: - sleep_seconds = backoff * (2**attempt_index) + if state_reason == "blocked_http_429_rate_limited": + sleep_seconds = rate_limit_backoff_seconds * rate_limit_attempt_count + attempt_label = rate_limit_attempt_count + else: + sleep_seconds = retry_backoff_seconds * (2 ** (network_attempt_count - 1)) + attempt_label = network_attempt_count + logger.warning( "ingestion.scholar_retry_scheduled", extra={ "event": "ingestion.scholar_retry_scheduled", "scholar_id": scholar_id, "cstart": cstart, - "attempt": attempt_index + 1, - "next_attempt": attempt_index + 2, + "attempt_count": attempt_label, "sleep_seconds": sleep_seconds, "state_reason": state_reason, }, @@ -1457,39 +1842,57 @@ class ScholarIngestionService: page_size: int, network_error_retries: int, retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]: - max_attempts = max(1, int(network_error_retries) + 1) - backoff = max(float(retry_backoff_seconds), 0.0) + network_attempts = 0 + rate_limit_attempts = 0 attempt_log: list[dict[str, Any]] = [] fetch_result: FetchResult | None = None parsed_page: ParsedProfilePage | None = None - for attempt_index in range(max_attempts): + while True: fetch_result = await self._fetch_profile_page( scholar_id=scholar_id, cstart=cstart, page_size=page_size, ) parsed_page = self._parse_profile_page_or_layout_error(fetch_result=fetch_result) + + if parsed_page.state == ParseState.NETWORK_ERROR: + network_attempts += 1 + total_attempts = network_attempts + elif parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA and parsed_page.state_reason == "blocked_http_429_rate_limited": + rate_limit_attempts += 1 + total_attempts = rate_limit_attempts + else: + total_attempts = network_attempts + rate_limit_attempts + 1 + attempt_log.append( self._attempt_log_entry( - attempt=attempt_index + 1, + attempt=total_attempts, cstart=cstart, fetch_result=fetch_result, parsed_page=parsed_page, ) ) - if not self._should_retry_network_page( + + if not self._should_retry_page_fetch( parsed_page=parsed_page, - attempt_index=attempt_index, - max_attempts=max_attempts, + network_attempt_count=network_attempts, + rate_limit_attempt_count=rate_limit_attempts, + network_error_retries=network_error_retries, + rate_limit_retries=rate_limit_retries, ): break + await self._sleep_retry_backoff( scholar_id=scholar_id, cstart=cstart, - attempt_index=attempt_index, - backoff=backoff, + network_attempt_count=network_attempts, + rate_limit_attempt_count=rate_limit_attempts, + retry_backoff_seconds=max(float(retry_backoff_seconds), 0.0), + rate_limit_backoff_seconds=max(float(rate_limit_backoff_seconds), 0.0), state_reason=parsed_page.state_reason, ) @@ -1591,6 +1994,7 @@ class ScholarIngestionService: pagination_truncated_reason=None, continuation_cstart=None, skipped_no_change=True, + discovered_publication_count=0, ) @staticmethod @@ -1619,6 +2023,7 @@ class ScholarIngestionService: pagination_truncated_reason=None, continuation_cstart=continuation_cstart, skipped_no_change=False, + discovered_publication_count=0, ) @staticmethod @@ -1685,6 +2090,8 @@ class ScholarIngestionService: bounded_page_size: int, network_error_retries: int, retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]: if request_delay_seconds > 0: await asyncio.sleep(float(request_delay_seconds)) @@ -1695,6 +2102,8 @@ class ScholarIngestionService: page_size=bounded_page_size, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, ) @staticmethod @@ -1747,6 +2156,8 @@ class ScholarIngestionService: bounded_page_size: int, network_error_retries: int, retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, ) -> tuple[FetchResult, ParsedProfilePage, str | None, list[dict[str, Any]], list[dict[str, Any]]]: fetch_result, parsed_page, first_attempt_log = await self._fetch_and_parse_page_with_retry( scholar_id=scholar_id, @@ -1754,6 +2165,8 @@ class ScholarIngestionService: page_size=bounded_page_size, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, ) first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page) attempt_log = list(first_attempt_log) @@ -1771,24 +2184,56 @@ class ScholarIngestionService: async def _paginate_loop( self, *, - scholar_id: str, + scholar: ScholarProfile, + run: CrawlRun, + db_session: AsyncSession, state: PagedLoopState, bounded_max_pages: int, request_delay_seconds: int, bounded_page_size: int, network_error_retries: int, retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, ) -> None: + # Cross-page canonical dedup state; grows across all pages of this scholar. + seen_canonical: set[str] = set() + + if state.parsed_page.publications: + deduped_first = _dedupe_publication_candidates( + list(state.parsed_page.publications), seen_canonical=seen_canonical + ) + if deduped_first: + discovered_count = await self._upsert_profile_publications( + db_session, run=run, scholar=scholar, publications=deduped_first + ) + state.discovered_publication_count += discovered_count + while state.parsed_page.has_show_more_button: + await db_session.refresh(run) + if run.status == RunStatus.CANCELED: + logger.info( + "ingestion.pagination_canceled", + extra={"event": "ingestion.pagination_canceled", "run_id": run.id}, + ) + self._set_truncated_state( + state=state, + reason="run_canceled", + continuation_cstart=state.current_cstart, + ) + return + if self._should_stop_pagination(state=state, bounded_max_pages=bounded_max_pages): return next_fetch_result, next_parsed_page, next_attempt_log = await self._fetch_next_page( - scholar_id=scholar_id, + scholar_id=scholar.scholar_id, state=state, request_delay_seconds=request_delay_seconds, bounded_page_size=bounded_page_size, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, ) self._record_next_page( state=state, @@ -1796,6 +2241,19 @@ class ScholarIngestionService: parsed_page=next_parsed_page, page_attempt_log=next_attempt_log, ) + + # Deduplicate against all publications already committed this run, + # then immediately commit the surviving candidates to the DB. + if next_parsed_page.publications: + deduped_next = _dedupe_publication_candidates( + list(next_parsed_page.publications), seen_canonical=seen_canonical + ) + if deduped_next: + discovered_count = await self._upsert_profile_publications( + db_session, run=run, scholar=scholar, publications=deduped_next + ) + state.discovered_publication_count += discovered_count + if self._handle_page_state_transition(state=state): return @@ -1822,6 +2280,7 @@ class ScholarIngestionService: pagination_truncated_reason=state.pagination_truncated_reason, continuation_cstart=state.continuation_cstart, skipped_no_change=False, + discovered_publication_count=state.discovered_publication_count, ) def _short_circuit_initial_page( @@ -1859,16 +2318,33 @@ class ScholarIngestionService: ) return None - async def _fetch_and_parse_all_pages_with_retry(self, *, scholar_id: str, start_cstart: int, request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, max_pages: int, page_size: int, previous_initial_page_fingerprint_sha256: str | None = None) -> PagedParseResult: + async def _fetch_and_parse_all_pages_with_retry( + self, + *, + scholar: ScholarProfile, + run: CrawlRun, + db_session: AsyncSession, + start_cstart: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + max_pages: int, + page_size: int, + previous_initial_page_fingerprint_sha256: str | None = None + ) -> PagedParseResult: bounded_max_pages = max(1, int(max_pages)) bounded_page_size = max(1, int(page_size)) fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs = ( await self._fetch_initial_page_context( - scholar_id=scholar_id, + scholar_id=scholar.scholar_id, start_cstart=start_cstart, bounded_page_size=bounded_page_size, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, )) shortcut_result = self._short_circuit_initial_page( start_cstart=start_cstart, @@ -1889,13 +2365,17 @@ class ScholarIngestionService: page_logs=page_logs, ) await self._paginate_loop( - scholar_id=scholar_id, + scholar=scholar, + run=run, + db_session=db_session, state=state, bounded_max_pages=bounded_max_pages, request_delay_seconds=request_delay_seconds, bounded_page_size=bounded_page_size, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, ) return self._result_from_pagination_state( state=state, @@ -1904,6 +2384,222 @@ class ScholarIngestionService: first_page_fingerprint_sha256=first_page_fingerprint_sha256, ) + async def _run_is_canceled( + self, + db_session: AsyncSession, + *, + run_id: int, + ) -> bool: + check_result = await db_session.execute( + select(CrawlRun.status).where(CrawlRun.id == run_id) + ) + status = check_result.scalar_one_or_none() + if status is None: + raise RuntimeError(f"Missing crawl_run for run_id={run_id}.") + return status == RunStatus.CANCELED + + async def _enrich_pending_publications( + self, + db_session: AsyncSession, + *, + run_id: int, + openalex_api_key: str | None = None, + ) -> None: + """Enrich unenriched publications with OpenAlex data. + + Stops immediately on budget exhaustion (429 with $0 remaining). + Sleeps 60s and continues on transient rate limits. + """ + from app.services.domains.openalex.client import ( + OpenAlexBudgetExhaustedError, + OpenAlexClient, + OpenAlexRateLimitError, + ) + from app.services.domains.openalex.matching import find_best_match + + run_result = await db_session.execute( + select(CrawlRun.user_id).where(CrawlRun.id == run_id) + ) + user_id = run_result.scalar_one() + + now = datetime.now(timezone.utc) + cooldown_threshold = now - timedelta(days=7) + + stmt = ( + select(Publication) + .join(ScholarPublication) + .join(ScholarProfile, ScholarPublication.scholar_profile_id == ScholarProfile.id) + .where( + ScholarProfile.user_id == user_id, + Publication.openalex_enriched.is_(False), + or_( + Publication.openalex_last_attempt_at.is_(None), + Publication.openalex_last_attempt_at < cooldown_threshold, + ), + ) + .distinct() + ) + result = await db_session.execute(stmt) + publications = list(result.scalars().all()) + + if not publications: + return + + resolved_key = openalex_api_key or settings.openalex_api_key + client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto) + batch_size = 25 + + for i in range(0, len(publications), batch_size): + if await self._run_is_canceled(db_session, run_id=run_id): + logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id}) + return + batch = publications[i : i + batch_size] + titles = [ + " ".join(re.sub(r"[^\w\s]", " ", p.title_raw).split()) + for p in batch + if p.title_raw and p.title_raw.strip() + ] + + if not titles: + continue + + try: + openalex_works = await client.get_works_by_filter( + {"title.search": "|".join(titles)}, limit=batch_size * 3 + ) + except OpenAlexBudgetExhaustedError: + logger.warning( + "ingestion.openalex_budget_exhausted", + extra={"event": "ingestion.openalex_budget_exhausted", "run_id": run_id}, + ) + break # Stop all enrichment — budget won't reset until midnight UTC + except OpenAlexRateLimitError: + logger.warning( + "ingestion.openalex_rate_limited", + extra={"event": "ingestion.openalex_rate_limited", "run_id": run_id}, + ) + await asyncio.sleep(60) + continue + except Exception as e: + logger.warning( + "ingestion.openalex_enrichment_failed", + extra={"event": "ingestion.openalex_enrichment_failed", "error": str(e), "run_id": run_id}, + ) + continue + + for p in batch: + # Check for cancellation periodically within the batch + if await self._run_is_canceled(db_session, run_id=run_id): + logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id}) + return + + p.openalex_last_attempt_at = now + + # Perform identifier discovery (moved from synchronous ingest loop) + await identifier_service.discover_and_sync_identifiers_for_publication( + db_session, + publication=p, + scholar_label=p.author_text or "", + ) + + match = find_best_match( + target_title=p.title_raw, + target_year=p.year, + target_authors=p.author_text or "", + candidates=openalex_works, + ) + if match: + p.year = match.publication_year or p.year + p.citation_count = match.cited_by_count or p.citation_count + p.pdf_url = match.oa_url or p.pdf_url + p.openalex_enriched = True + + await db_session.flush() + + from app.services.domains.publications.dedup import sweep_identifier_duplicates + + merge_count = await sweep_identifier_duplicates(db_session) + if merge_count: + logger.info( + "ingestion.identifier_dedup_sweep", + extra={ + "event": "ingestion.identifier_dedup_sweep", + "merged_count": merge_count, + "run_id": run_id, + }, + ) + + async def _enrich_publications_with_openalex( + self, + scholar: ScholarProfile, + publications: list[PublicationCandidate], + ) -> list[PublicationCandidate]: + if not publications: + return publications + + from app.services.domains.openalex.client import OpenAlexClient + from app.services.domains.openalex.matching import find_best_match + + client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto) + + # Batch requests by 25 to stay within URL length limits + batch_size = 25 + enriched: list[PublicationCandidate] = [] + + import re + for i in range(0, len(publications), batch_size): + batch = publications[i:i + batch_size] + + titles = [] + for p in batch: + if not p.title: + continue + # Strip all non-alphanumeric words to prevent OpenAlex API injection crashes + # and minimize punctuation-based duplicate misses. + safe_title = re.sub(r"[^\w\s]", " ", p.title) + safe_title = " ".join(safe_title.split()) + if safe_title: + titles.append(safe_title) + + if not titles: + enriched.extend(batch) + continue + + query = "|".join(t for t in titles) + try: + openalex_works = await client.get_works_by_filter( + {"title.search": query}, limit=batch_size * 3 + ) + except Exception as e: + logger.warning( + "ingestion.openalex_enrichment_failed", + extra={"error": str(e), "batch_size": len(batch), "scholar_id": scholar.id} + ) + openalex_works = [] + + for p in batch: + match = find_best_match( + target_title=p.title, + target_year=p.year, + target_authors=p.authors_text or (scholar.display_name or scholar.scholar_id), + candidates=openalex_works + ) + if match: + new_p = PublicationCandidate( + title=p.title, + title_url=p.title_url, + cluster_id=p.cluster_id, + year=match.publication_year or p.year, + citation_count=match.cited_by_count, + authors_text=p.authors_text, + venue_text=p.venue_text, + pdf_url=match.oa_url or p.pdf_url, + ) + enriched.append(new_p) + else: + enriched.append(p) + return enriched + async def _upsert_profile_publications( self, db_session: AsyncSession, @@ -1912,6 +2608,7 @@ class ScholarIngestionService: scholar: ScholarProfile, publications: list[PublicationCandidate], ) -> int: + # We no longer enrich inline here. Enrichment is now deferred to the end of the run. seen_publication_ids: set[int] = set() discovered_count = 0 @@ -1949,12 +2646,43 @@ class ScholarIngestionService: "crawl_run_id": run.id, }, ) + + await self._commit_discovered_publication( + db_session, + run=run, + scholar=scholar, + publication=publication, + ) if not scholar.baseline_completed: scholar.baseline_completed = True return discovered_count + async def _commit_discovered_publication( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + publication: Publication, + ) -> None: + run.new_pub_count = int(run.new_pub_count or 0) + 1 + await db_session.commit() + await run_events.publish( + run_id=run.id, + event_type="publication_discovered", + data={ + "publication_id": publication.id, + "title": publication.title_raw, + "pub_url": publication.pub_url, + "scholar_profile_id": scholar.id, + "scholar_label": scholar.display_name or scholar.scholar_id, + "first_seen_at": datetime.now(timezone.utc).isoformat(), + "new_publication_count": int(run.new_pub_count or 0), + }, + ) + @staticmethod def _validate_publication_candidate(candidate: PublicationCandidate) -> None: if not candidate.title.strip(): @@ -1996,6 +2724,24 @@ class ScholarIngestionService: return cluster_publication return fingerprint_publication + @staticmethod + def _compute_canonical_title_hash(title: str) -> str: + canonical = canonical_title_for_dedup(title) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + async def _find_publication_by_canonical_title_hash( + self, + db_session: AsyncSession, + *, + canonical_title_hash: str, + ) -> Publication | None: + result = await db_session.execute( + select(Publication).where( + Publication.canonical_title_hash == canonical_title_hash + ) + ) + return result.scalar_one_or_none() + async def _create_publication( self, db_session: AsyncSession, @@ -2008,12 +2754,12 @@ class ScholarIngestionService: fingerprint_sha256=fingerprint, title_raw=candidate.title, title_normalized=normalize_title(candidate.title), + canonical_title_hash=self._compute_canonical_title_hash(candidate.title), year=candidate.year, citation_count=int(candidate.citation_count or 0), author_text=candidate.authors_text, venue_text=candidate.venue_text, pub_url=build_publication_url(candidate.title_url), - doi=first_doi_from_texts(candidate.title_url, candidate.venue_text, candidate.title), pdf_url=None, ) db_session.add(publication) @@ -2049,8 +2795,6 @@ class ScholarIngestionService: if candidate.title_url: publication.pub_url = build_publication_url(candidate.title_url) local_doi = first_doi_from_texts(candidate.title_url, candidate.venue_text, candidate.title) - if local_doi and not publication.doi: - publication.doi = local_doi async def _resolve_publication( self, @@ -2071,12 +2815,21 @@ class ScholarIngestionService: cluster_publication=cluster_publication, fingerprint_publication=fingerprint_publication, ) + if publication is None: + # Fallback: canonical title hash — catches cross-scholar noise variants + # (e.g. "Adam preprint (2014)" vs "Adam arXiv 2017" → same normalized form) + canonical_hash = self._compute_canonical_title_hash(candidate.title) + publication = await self._find_publication_by_canonical_title_hash( + db_session, + canonical_title_hash=canonical_hash, + ) if publication is None: created = await self._create_publication( db_session, candidate=candidate, fingerprint=fingerprint, ) + # Sync identifiers from local fields only for fast UI response await identifier_service.sync_identifiers_for_publication_fields( db_session, publication=created, @@ -2086,6 +2839,7 @@ class ScholarIngestionService: publication=publication, candidate=candidate, ) + # Sync identifiers from local fields only for fast UI response await identifier_service.sync_identifiers_for_publication_fields( db_session, publication=publication, diff --git a/app/services/domains/ingestion/fingerprints.py b/app/services/domains/ingestion/fingerprints.py index cf3e796..30745b0 100644 --- a/app/services/domains/ingestion/fingerprints.py +++ b/app/services/domains/ingestion/fingerprints.py @@ -15,12 +15,59 @@ from app.services.domains.ingestion.constants import ( ) from app.services.domains.scholar.parser import ParseState, ParsedProfilePage, PublicationCandidate +# Scholar-specific noise patterns stripped before canonical comparison. +# Applied in order; each targets a different Scholar metadata injection style. +_NOISE_DOI_RE = re.compile(r"[,.\s]+doi\s*:\s*\S+.*$", re.IGNORECASE) +_NOISE_ARXIV_RE = re.compile(r"[,.\s]+arxiv\b.*$", re.IGNORECASE) +_NOISE_PREPRINT_RE = re.compile( + r"[,\s]+(?:preprint|extended\s+version|technical\s+report|working\s+paper)\b.*$", + re.IGNORECASE, +) +_NOISE_TRAILING_YEAR_RE = re.compile(r"\s*[,(]\s*\d{4}\s*[),]?\s*$") +# Strips ". Capitalised sentence" appended as venue: ". Comput. Sci…", ". Journal of…" +_NOISE_VENUE_SENTENCE_RE = re.compile(r"(?<=\w{3})\.\s+[A-Z][a-z].*$") + +_CANONICAL_DEDUP_THRESHOLD = 0.82 + def normalize_title(value: str) -> str: lowered = value.lower() return TITLE_ALNUM_RE.sub("", lowered) +def canonical_title_for_dedup(title: str) -> str: + """Strip Scholar-specific noise suffixes then normalize for dedup comparison.""" + t = title.strip() + t = _NOISE_DOI_RE.sub("", t) + t = _NOISE_ARXIV_RE.sub("", t) + t = _NOISE_PREPRINT_RE.sub("", t) + t = _NOISE_TRAILING_YEAR_RE.sub("", t) + t = _NOISE_VENUE_SENTENCE_RE.sub("", t) + return normalize_title(t.strip()) + + +def _stripped_title_for_canonical(title: str) -> str: + """Apply noise-stripping and lowercase but PRESERVE spaces (for later tokenization).""" + t = title.strip() + t = _NOISE_DOI_RE.sub("", t) + t = _NOISE_ARXIV_RE.sub("", t) + t = _NOISE_PREPRINT_RE.sub("", t) + t = _NOISE_TRAILING_YEAR_RE.sub("", t) + t = _NOISE_VENUE_SENTENCE_RE.sub("", t) + return t.lower().strip() + + +def _canonical_title_tokens(title: str) -> set[str]: + """Word tokens of the noise-stripped title (preserves token boundaries).""" + return set(WORD_RE.findall(_stripped_title_for_canonical(title))) + + +def _jaccard(a: set[str], b: set[str]) -> float: + if not a or not b: + return 0.0 + return len(a & b) / len(a | b) + + def _first_author_last_name(authors_text: str | None) -> str: if not authors_text: return "" @@ -100,31 +147,91 @@ def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int: return int(fallback) +def _title_tokens(value: str) -> set[str]: + """Extract normalized word tokens for fuzzy title comparison.""" + return set(WORD_RE.findall(value.lower())) + + +def fuzzy_titles_match( + title_a: str, + title_b: str, + *, + threshold: float = 0.85, +) -> bool: + """Return True if two titles are near-duplicates by token-level Jaccard similarity. + + A threshold of 0.85 catches common academic duplicate patterns: + differences in punctuation, minor word variations, subtitle changes. + """ + tokens_a = _title_tokens(title_a) + tokens_b = _title_tokens(title_b) + return _jaccard(tokens_a, tokens_b) >= threshold + + def _dedupe_publication_candidates( publications: list[PublicationCandidate], + *, + seen_canonical: set[str] | None = None, ) -> list[PublicationCandidate]: + """Deduplicate candidates using canonical title matching. + + Args: + publications: candidates to filter + seen_canonical: optional mutable set shared across pages. Stores the + noise-stripped *lowercased* (but space-preserved) canonical string + so it can be tokenized on the next page for cross-page fuzzy dedup. + Accepted canonicals are added; existing entries are consulted. + """ deduped: list[PublicationCandidate] = [] - seen: set[str] = set() - for publication in publications: - if publication.cluster_id: - identity = f"cluster:{publication.cluster_id}" - else: - identity = "|".join( - [ - "fallback", - normalize_title(publication.title), - str(publication.year) if publication.year is not None else "", - publication.authors_text or "", - publication.venue_text or "", - ] - ) - if identity in seen: + seen_exact: set[str] = set() + # Token sets for fuzzy comparison; seeded from cross-page state. + seen_tokens: list[set[str]] = [] + + if seen_canonical: + for stripped in seen_canonical: + seen_tokens.append(set(WORD_RE.findall(stripped))) + + for pub in publications: + identity = _publication_identity(pub) + if identity in seen_exact: continue - seen.add(identity) - deduped.append(publication) + + # Use space-preserving stripped form for token-level fuzzy match. + tokens = _canonical_title_tokens(pub.title) + if _is_fuzzy_dup(tokens, seen_tokens): + continue + + seen_exact.add(identity) + seen_tokens.append(tokens) + if seen_canonical is not None: + # Store the noise-stripped lowercased (space-preserved) form. + seen_canonical.add(_stripped_title_for_canonical(pub.title)) + deduped.append(pub) + return deduped +def _publication_identity(pub: PublicationCandidate) -> str: + if pub.cluster_id: + return f"cluster:{pub.cluster_id}" + canonical = canonical_title_for_dedup(pub.title) + return "|".join( + [ + "fallback", + canonical, + str(pub.year) if pub.year is not None else "", + _first_author_last_name(pub.authors_text), + ] + ) + + +def _is_fuzzy_dup(tokens: set[str], seen: list[set[str]]) -> bool: + for existing in seen: + if _jaccard(tokens, existing) >= _CANONICAL_DEDUP_THRESHOLD: + return True + return False + + def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None: if not body: return None diff --git a/app/services/domains/ingestion/scheduler.py b/app/services/domains/ingestion/scheduler.py index 9db0204..b05e84d 100644 --- a/app/services/domains/ingestion/scheduler.py +++ b/app/services/domains/ingestion/scheduler.py @@ -146,6 +146,9 @@ class SchedulerService: async def _tick_once(self) -> None: if self._continuation_queue_enabled: await self._drain_continuation_queue() + + await self._drain_pdf_queue() + candidates = await self._load_candidates() if not candidates: return @@ -488,6 +491,8 @@ class SchedulerService: request_delay_seconds=request_delay_seconds, network_error_retries=self._network_error_retries, retry_backoff_seconds=self._retry_backoff_seconds, + rate_limit_retries=settings.ingestion_rate_limit_retries, + rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds, max_pages_per_scholar=self._max_pages_per_scholar, page_size=self._page_size, scholar_profile_ids={job.scholar_profile_id}, @@ -583,6 +588,27 @@ class SchedulerService: return await self._finalize_queue_job_after_run(job, run_summary) + async def _drain_pdf_queue(self) -> None: + from app.services.domains.publications.pdf_queue import drain_ready_jobs + + session_factory = get_session_factory() + async with session_factory() as session: + try: + processed = await drain_ready_jobs( + session, + limit=settings.scheduler_pdf_queue_batch_size, + max_attempts=settings.pdf_auto_retry_max_attempts, + ) + if processed > 0: + logger.info("scheduler.pdf_queue_drain_completed", extra={ + "event": "scheduler.pdf_queue_drain_completed", + "processed_count": processed, + }) + except Exception: + logger.exception("scheduler.pdf_queue_drain_failed", extra={ + "event": "scheduler.pdf_queue_drain_failed", + }) + async def _load_request_delay_for_user( self, db_session: AsyncSession, diff --git a/app/services/domains/ingestion/types.py b/app/services/domains/ingestion/types.py index f6de578..09d87f1 100644 --- a/app/services/domains/ingestion/types.py +++ b/app/services/domains/ingestion/types.py @@ -35,6 +35,7 @@ class PagedParseResult: pagination_truncated_reason: str | None continuation_cstart: int | None skipped_no_change: bool + discovered_publication_count: int @dataclass @@ -88,6 +89,7 @@ class PagedLoopState: has_more_remaining: bool = False pagination_truncated_reason: str | None = None continuation_cstart: int | None = None + discovered_publication_count: int = 0 class RunAlreadyInProgressError(RuntimeError): diff --git a/app/services/domains/openalex/__init__.py b/app/services/domains/openalex/__init__.py new file mode 100644 index 0000000..a51555a --- /dev/null +++ b/app/services/domains/openalex/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +import logging + +logger = logging.getLogger(__name__) diff --git a/app/services/domains/openalex/client.py b/app/services/domains/openalex/client.py new file mode 100644 index 0000000..a85b630 --- /dev/null +++ b/app/services/domains/openalex/client.py @@ -0,0 +1,151 @@ +import asyncio +import logging +from typing import Any, Mapping + +import httpx +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_exponential, +) + +from app.services.domains.openalex.types import OpenAlexWork + +logger = logging.getLogger(__name__) + +OPENALEX_BASE_URL = "https://api.openalex.org" + + +class OpenAlexClientError(Exception): + pass + + +class OpenAlexRateLimitError(OpenAlexClientError): + """Transient rate limit (too many requests per second).""" + pass + + +class OpenAlexBudgetExhaustedError(OpenAlexClientError): + """Daily API budget exhausted — retrying is futile until midnight UTC.""" + pass + + +class OpenAlexClient: + def __init__( + self, + api_key: str | None = None, + mailto: str | None = None, + timeout: float = 10.0, + ) -> None: + self.api_key = api_key + self.mailto = mailto + self.timeout = timeout + + @property + def _base_params(self) -> dict[str, str]: + params = {} + if self.mailto: + params["mailto"] = self.mailto + if self.api_key: + params["api_key"] = self.api_key + return params + + @retry( + retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException)), + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=10), + reraise=True, + ) + async def get_work_by_doi(self, doi: str) -> OpenAlexWork | None: + """Fetch a single work by DOI directly.""" + clean_doi = doi.replace("https://doi.org/", "") + if not clean_doi: + return None + + url = f"{OPENALEX_BASE_URL}/works/{clean_doi}" + + headers = {} + if self.mailto: + headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})" + else: + headers["User-Agent"] = "scholar-scraper/1.0" + + async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client: + response = await client.get(url, params=self._base_params) + + if response.status_code == 404: + return None + if response.status_code == 429: + remaining = response.headers.get("X-RateLimit-Remaining-USD", "") + if remaining == "0" or remaining.startswith("-"): + raise OpenAlexBudgetExhaustedError( + "Daily API budget exhausted; retrying won't help until midnight UTC" + ) + raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI") + if response.status_code >= 400: + logger.warning("OpenAlex API error: %s %s", response.status_code, response.text) + raise OpenAlexClientError(f"API Error {response.status_code}") + + data = response.json() + return OpenAlexWork.from_api_dict(data) + + @retry( + retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException)), + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=10), + reraise=True, + ) + async def get_works_by_filter( + self, + filters: dict[str, str], + limit: int = 50, + ) -> list[OpenAlexWork]: + """ + Fetch works using the ?filter= query parameter. + Supports fetching multiple records by joining filters with | (OR logic). + """ + if not filters: + return [] + + # Example: {"doi": "10.foo|10.bar", "title.search": "query"} + filter_str = ",".join(f"{k}:{v}" for k, v in filters.items()) + + params = self._base_params.copy() + params["filter"] = filter_str + params["per-page"] = str(limit) + + url = f"{OPENALEX_BASE_URL}/works" + + headers = {} + if self.mailto: + headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})" + else: + headers["User-Agent"] = "scholar-scraper/1.0" + + async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client: + response = await client.get(url, params=params) + + if response.status_code == 429: + remaining = response.headers.get("X-RateLimit-Remaining-USD", "") + if remaining == "0" or remaining.startswith("-"): + raise OpenAlexBudgetExhaustedError( + "Daily API budget exhausted; retrying won't help until midnight UTC" + ) + raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list") + if response.status_code >= 400: + logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text) + raise OpenAlexClientError(f"API Error {response.status_code}") + + data = response.json() + results = data.get("results") or [] + + parsed_works = [] + for raw_work in results: + try: + parsed_works.append(OpenAlexWork.from_api_dict(raw_work)) + except Exception as e: + logger.warning("Failed to parse OpenAlex raw dict: %s", e) + continue + + return parsed_works diff --git a/app/services/domains/openalex/matching.py b/app/services/domains/openalex/matching.py new file mode 100644 index 0000000..cfa511c --- /dev/null +++ b/app/services/domains/openalex/matching.py @@ -0,0 +1,108 @@ +import logging +import re + +from rapidfuzz import fuzz + +from app.services.domains.openalex.types import OpenAlexWork + +logger = logging.getLogger(__name__) + +# A minimum similarity score out of 100 for a title to be considered a match candidate. +TITLE_MATCH_THRESHOLD = 90.0 +# The margin within the top score where a secondary tiebreaker (author/year) is necessary. +TIEBREAKER_MARGIN = 5.0 + + +def _clean_string(s: str | None) -> str: + if not s: + return "" + # Strip non-alphanumeric (keep spaces), lowercase, and collapse whitespace + cleaned = re.sub(r"[^a-z0-9\s]", " ", s.lower()) + return " ".join(cleaned.split()) + + +def _author_overlap_score(target_authors: str | None, candidate_authors: list[str]) -> bool: + if not target_authors or not candidate_authors: + return False + + target_clean = _clean_string(target_authors) + if not target_clean: + return False + + for candidate in candidate_authors: + cand_clean = _clean_string(candidate) + if cand_clean and (cand_clean in target_clean or target_clean in cand_clean): + return True + # Alternatively check rapidfuzz token_set_ratio + if cand_clean and fuzz.token_set_ratio(target_clean, cand_clean) > 80: + return True + + return False + + +def find_best_match( + target_title: str, + target_year: int | None, + target_authors: str | None, + candidates: list[OpenAlexWork], +) -> OpenAlexWork | None: + """ + Finds the best matching OpenAlexWork from a list of candidates, prioritizing title similarity (>90%) + with year and author overlap as tiebreakers for close candidates. + """ + if not target_title or not candidates: + return None + + clean_target = _clean_string(target_title) + if not clean_target: + return None + + scored_candidates: list[tuple[float, OpenAlexWork]] = [] + + for cand in candidates: + if not cand.title: + continue + + clean_cand = _clean_string(cand.title) + + # Primary sort: string similarity ratio + score = fuzz.ratio(clean_target, clean_cand) + + if score >= TITLE_MATCH_THRESHOLD: + scored_candidates.append((score, cand)) + + if not scored_candidates: + return None + + # Sort descending by score + scored_candidates.sort(key=lambda x: x[0], reverse=True) + + best_score = scored_candidates[0][0] + + # Extract all candidates within the tiebreaker margin + top_scored_candidates = [ + (score, cand) for score, cand in scored_candidates + if best_score - score <= TIEBREAKER_MARGIN + ] + + if len(top_scored_candidates) == 1: + return top_scored_candidates[0][1] + + # We have a tie or near-tie. Use year and author overlap to break the tie. + # Score candidates: +1 for year match (within 1 year), +1 for author overlap + tiebreaker_scores: list[tuple[int, float, OpenAlexWork]] = [] + + for original_score, cand in top_scored_candidates: + tb_score = 0 + if target_year is not None and cand.publication_year is not None: + if abs(target_year - cand.publication_year) <= 1: + tb_score += 1 + + candidate_author_names = [a.display_name for a in cand.authors if a.display_name] + if _author_overlap_score(target_authors, candidate_author_names): + tb_score += 1 + + tiebreaker_scores.append((tb_score, original_score, cand)) + + tiebreaker_scores.sort(key=lambda x: (x[0], x[1]), reverse=True) + return tiebreaker_scores[0][2] diff --git a/app/services/domains/openalex/types.py b/app/services/domains/openalex/types.py new file mode 100644 index 0000000..0d86754 --- /dev/null +++ b/app/services/domains/openalex/types.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Mapping + + +@dataclass(frozen=True) +class OpenAlexAuthor: + openalex_id: str | None + display_name: str | None + + +@dataclass(frozen=True) +class OpenAlexWork: + openalex_id: str + doi: str | None + pmid: str | None + pmcid: str | None + title: str | None + publication_year: int | None + cited_by_count: int + is_oa: bool + oa_url: str | None + authors: list[OpenAlexAuthor] = field(default_factory=list) + raw_data: Mapping[str, Any] = field(default_factory=dict, repr=False) + + @classmethod + def from_api_dict(cls, data: Mapping[str, Any]) -> OpenAlexWork: + ids = data.get("ids") or {} + + # Extract DOI without the https://doi.org/ prefix + doi = ids.get("doi") + if doi and doi.startswith("https://doi.org/"): + doi = doi[16:] + + # Extract PMID without the url prefix + pmid = ids.get("pmid") + if pmid and pmid.startswith("https://pubmed.ncbi.nlm.nih.gov/"): + pmid = pmid[32:] + + # Extract PMCID without the url prefix + pmcid = ids.get("pmcid") + if pmcid and pmcid.startswith("https://www.ncbi.nlm.nih.gov/pmc/articles/"): + pmcid = pmcid[42:] + + open_access = data.get("open_access") or {} + + authors = [] + for authorship in data.get("authorships") or []: + author_data = authorship.get("author") or {} + authors.append( + OpenAlexAuthor( + openalex_id=author_data.get("id"), + display_name=author_data.get("display_name"), + ) + ) + + return cls( + openalex_id=data.get("id", ""), + doi=doi, + pmid=pmid, + pmcid=pmcid, + title=data.get("title"), + publication_year=data.get("publication_year"), + cited_by_count=data.get("cited_by_count", 0), + is_oa=bool(open_access.get("is_oa")), + oa_url=open_access.get("oa_url"), + authors=authors, + raw_data=dict(data), + ) diff --git a/app/services/domains/portability/exporting.py b/app/services/domains/portability/exporting.py index ef43a89..5c89006 100644 --- a/app/services/domains/portability/exporting.py +++ b/app/services/domains/portability/exporting.py @@ -34,7 +34,6 @@ def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]: author_text, venue_text, pub_url, - doi, pdf_url, is_read, ) = row @@ -48,7 +47,6 @@ def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]: "author_text": author_text, "venue_text": venue_text, "pub_url": pub_url, - "doi": doi, "pdf_url": pdf_url, "is_read": bool(is_read), } @@ -75,7 +73,6 @@ async def export_user_data( Publication.author_text, Publication.venue_text, Publication.pub_url, - Publication.doi, Publication.pdf_url, ScholarPublication.is_read, ) diff --git a/app/services/domains/portability/publication_import.py b/app/services/domains/portability/publication_import.py index daa1d5d..7654586 100644 --- a/app/services/domains/portability/publication_import.py +++ b/app/services/domains/portability/publication_import.py @@ -71,7 +71,6 @@ def _apply_imported_publication_values( author_text: str | None, venue_text: str | None, pub_url: str | None, - doi: str | None, pdf_url: str | None, cluster_id: str | None, ) -> bool: @@ -98,9 +97,6 @@ def _apply_imported_publication_values( if pub_url and publication.pub_url != pub_url: publication.pub_url = pub_url updated = True - if doi and publication.doi != doi: - publication.doi = doi - updated = True if pdf_url and publication.pdf_url != pdf_url: publication.pdf_url = pdf_url updated = True @@ -117,7 +113,6 @@ def _new_publication( author_text: str | None, venue_text: str | None, pub_url: str | None, - doi: str | None, pdf_url: str | None, ) -> Publication: return Publication( @@ -130,7 +125,6 @@ def _new_publication( author_text=author_text, venue_text=venue_text, pub_url=pub_url, - doi=doi, pdf_url=pdf_url, ) @@ -285,7 +279,6 @@ async def _create_import_publication( author_text=payload.author_text, venue_text=payload.venue_text, pub_url=payload.pub_url, - doi=payload.doi, pdf_url=payload.pdf_url, ) db_session.add(publication) @@ -306,7 +299,6 @@ def _update_import_publication( author_text=payload.author_text, venue_text=payload.venue_text, pub_url=payload.pub_url, - doi=payload.doi, pdf_url=payload.pdf_url, cluster_id=payload.cluster_id, ) diff --git a/app/services/domains/publication_identifiers/application.py b/app/services/domains/publication_identifiers/application.py index d34e757..ca57d57 100644 --- a/app/services/domains/publication_identifiers/application.py +++ b/app/services/domains/publication_identifiers/application.py @@ -21,7 +21,7 @@ from app.services.domains.publication_identifiers.types import ( if TYPE_CHECKING: from app.services.domains.publications.pdf_queue import PdfQueueListItem - from app.services.domains.publications.types import PublicationListItem + from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem CONFIDENCE_HIGH = 0.98 CONFIDENCE_MEDIUM = 0.9 @@ -117,9 +117,76 @@ async def sync_identifiers_for_publication_fields( await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=candidates) +async def discover_and_sync_identifiers_for_publication( + db_session: AsyncSession, + *, + publication: Publication, + scholar_label: str, +) -> None: + await sync_identifiers_for_publication_fields(db_session, publication=publication) + + existing_doi = await _existing_identifier_by_kind( + db_session, + publication_id=int(publication.id), + kind=IdentifierKind.DOI.value, + ) + if existing_doi is not None: + return + + from app.services.domains.crossref import application as crossref_service + from app.services.domains.publications.types import UnreadPublicationItem + + item = UnreadPublicationItem( + publication_id=int(publication.id), + scholar_profile_id=0, + scholar_label=scholar_label, + title=str(publication.title_raw or ""), + year=publication.year, + citation_count=publication.citation_count, + venue_text=publication.venue_text, + pub_url=publication.pub_url, + pdf_url=publication.pdf_url, + ) + + discovered_doi = await crossref_service.discover_doi_for_publication(item=item) + if discovered_doi: + normalized_doi = normalize_doi(discovered_doi) + if normalized_doi: + candidate = _candidate( + IdentifierKind.DOI, + discovered_doi, + normalized_doi, + "crossref_api", + CONFIDENCE_MEDIUM, + None, + ) + await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate) + + existing_arxiv = await _existing_identifier_by_kind( + db_session, + publication_id=int(publication.id), + kind=IdentifierKind.ARXIV.value, + ) + if existing_arxiv is None: + from app.services.domains.arxiv import application as arxiv_service + discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item) + if discovered_arxiv: + normalized_arxiv = normalize_arxiv_id(discovered_arxiv) + if normalized_arxiv: + candidate = _candidate( + IdentifierKind.ARXIV, + discovered_arxiv, + normalized_arxiv, + "arxiv_api", + CONFIDENCE_MEDIUM, + None, + ) + await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate) + + def _publication_field_candidates(publication: Publication) -> list[IdentifierCandidate]: return _fallback_candidates_from_values( - doi=publication.doi, + doi=None, pub_url=publication.pub_url, pdf_url=publication.pdf_url, ) @@ -194,6 +261,21 @@ async def _existing_identifier( return result.scalar_one_or_none() +async def _existing_identifier_by_kind( + db_session: AsyncSession, + *, + publication_id: int, + kind: str, +) -> PublicationIdentifier | None: + result = await db_session.execute( + select(PublicationIdentifier).where( + PublicationIdentifier.publication_id == publication_id, + PublicationIdentifier.kind == kind, + ).order_by(PublicationIdentifier.confidence_score.desc()).limit(1) + ) + return result.scalar_one_or_none() + + def _new_identifier_row( *, publication_id: int, @@ -234,7 +316,7 @@ def _overlay_publication_item( item: PublicationListItem, display_identifier: DisplayIdentifier | None, ) -> PublicationListItem: - fallback = display_identifier or derive_display_identifier_from_values(doi=item.doi, pub_url=item.pub_url, pdf_url=item.pdf_url) + fallback = display_identifier or derive_display_identifier_from_values(doi=None, pub_url=item.pub_url, pdf_url=item.pdf_url) return replace(item, display_identifier=fallback) @@ -253,7 +335,7 @@ def _overlay_queue_item( item: PdfQueueListItem, display_identifier: DisplayIdentifier | None, ) -> PdfQueueListItem: - fallback = display_identifier or derive_display_identifier_from_values(doi=item.doi, pdf_url=item.pdf_url) + fallback = display_identifier or derive_display_identifier_from_values(doi=None, pdf_url=item.pdf_url) return replace(item, display_identifier=fallback) diff --git a/app/services/domains/publication_identifiers/normalize.py b/app/services/domains/publication_identifiers/normalize.py index 50036be..9f7484a 100644 --- a/app/services/domains/publication_identifiers/normalize.py +++ b/app/services/domains/publication_identifiers/normalize.py @@ -7,7 +7,7 @@ from app.services.domains.doi.normalize import normalize_doi from app.services.domains.publication_identifiers.types import IdentifierKind ARXIV_ABS_RE = re.compile(r"\barxiv:\s*([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?\b", re.I) -ARXIV_PATH_RE = re.compile(r"^/(?:abs|pdf)/([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?(?:\.pdf)?/?$", re.I) +ARXIV_PATH_RE = re.compile(r"^/(?:abs|pdf|html|ps|format)/([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?(?:\.pdf)?/?$", re.I) PMCID_RE = re.compile(r"\b(PMC\d+)\b", re.I) PUBMED_PATH_RE = re.compile(r"^/(\d+)/?$") diff --git a/app/services/domains/publications/application.py b/app/services/domains/publications/application.py index b1161fb..f51c79a 100644 --- a/app/services/domains/publications/application.py +++ b/app/services/domains/publications/application.py @@ -24,7 +24,7 @@ from app.services.domains.publications.modes import ( resolve_publication_view_mode, ) from app.services.domains.publications.queries import ( - get_latest_completed_run_id_for_user, + get_latest_run_id_for_user, get_publication_item_for_user, publications_query, ) @@ -50,7 +50,7 @@ __all__ = [ "PublicationListItem", "UnreadPublicationItem", "resolve_publication_view_mode", - "get_latest_completed_run_id_for_user", + "get_latest_run_id_for_user", "publications_query", "get_publication_item_for_user", "list_for_user", diff --git a/app/services/domains/publications/counts.py b/app/services/domains/publications/counts.py index 8f65172..51a524d 100644 --- a/app/services/domains/publications/counts.py +++ b/app/services/domains/publications/counts.py @@ -1,6 +1,8 @@ from __future__ import annotations -from sqlalchemy import func, select +from datetime import datetime + +from sqlalchemy import distinct, func, select from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import ScholarProfile, ScholarPublication @@ -10,7 +12,7 @@ from app.services.domains.publications.modes import ( MODE_UNREAD, resolve_publication_view_mode, ) -from app.services.domains.publications.queries import get_latest_completed_run_id_for_user +from app.services.domains.publications.queries import get_latest_run_id_for_user async def count_for_user( @@ -20,11 +22,12 @@ async def count_for_user( mode: str = MODE_ALL, scholar_profile_id: int | None = None, favorite_only: bool = False, + snapshot_before: datetime | None = None, ) -> int: resolved_mode = resolve_publication_view_mode(mode) - latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) + latest_run_id = await get_latest_run_id_for_user(db_session, user_id=user_id) stmt = ( - select(func.count()) + select(func.count(distinct(ScholarPublication.publication_id))) .select_from(ScholarPublication) .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) .where(ScholarProfile.user_id == user_id) @@ -33,6 +36,8 @@ async def count_for_user( stmt = stmt.where(ScholarProfile.id == scholar_profile_id) if favorite_only: stmt = stmt.where(ScholarPublication.is_favorite.is_(True)) + if snapshot_before is not None: + stmt = stmt.where(ScholarPublication.created_at <= snapshot_before) if resolved_mode == MODE_UNREAD: stmt = stmt.where(ScholarPublication.is_read.is_(False)) if resolved_mode == MODE_LATEST: @@ -49,6 +54,7 @@ async def count_unread_for_user( user_id: int, scholar_profile_id: int | None = None, favorite_only: bool = False, + snapshot_before: datetime | None = None, ) -> int: return await count_for_user( db_session, @@ -56,6 +62,7 @@ async def count_unread_for_user( mode=MODE_UNREAD, scholar_profile_id=scholar_profile_id, favorite_only=favorite_only, + snapshot_before=snapshot_before, ) @@ -65,6 +72,7 @@ async def count_latest_for_user( user_id: int, scholar_profile_id: int | None = None, favorite_only: bool = False, + snapshot_before: datetime | None = None, ) -> int: return await count_for_user( db_session, @@ -72,6 +80,7 @@ async def count_latest_for_user( mode=MODE_LATEST, scholar_profile_id=scholar_profile_id, favorite_only=favorite_only, + snapshot_before=snapshot_before, ) @@ -80,6 +89,7 @@ async def count_favorite_for_user( *, user_id: int, scholar_profile_id: int | None = None, + snapshot_before: datetime | None = None, ) -> int: return await count_for_user( db_session, @@ -87,4 +97,5 @@ async def count_favorite_for_user( mode=MODE_ALL, scholar_profile_id=scholar_profile_id, favorite_only=True, + snapshot_before=snapshot_before, ) diff --git a/app/services/domains/publications/dedup.py b/app/services/domains/publications/dedup.py new file mode 100644 index 0000000..8f3c7b7 --- /dev/null +++ b/app/services/domains/publications/dedup.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import logging + +from sqlalchemy import delete, select +from sqlalchemy.orm import aliased +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Publication, PublicationIdentifier, ScholarPublication + +logger = logging.getLogger(__name__) + + +async def find_identifier_duplicate_pairs( + db_session: AsyncSession, +) -> list[tuple[int, int]]: + """Return (winner_id, dup_id) pairs where two publications share the same identifier. + + Winner is always the lower publication_id (earlier-created). Uses the existing + ix_publication_identifiers_kind_value index for the self-join. + """ + pi1 = aliased(PublicationIdentifier, name="pi1") + pi2 = aliased(PublicationIdentifier, name="pi2") + rows = await db_session.execute( + select(pi1.publication_id, pi2.publication_id) + .join( + pi2, + (pi1.kind == pi2.kind) + & (pi1.value_normalized == pi2.value_normalized) + & (pi1.publication_id < pi2.publication_id), + ) + .distinct() + ) + return [(winner_id, dup_id) for winner_id, dup_id in rows] + + +async def merge_duplicate_publication( + db_session: AsyncSession, + *, + winner_id: int, + dup_id: int, +) -> None: + """Merge dup_id into winner_id: migrate scholar links, then delete the dup.""" + await _migrate_scholar_links(db_session, winner_id=winner_id, dup_id=dup_id) + await db_session.execute( + delete(Publication).where(Publication.id == dup_id) + ) + logger.info( + "publications.identifier_merge", + extra={ + "event": "publications.identifier_merge", + "winner_id": winner_id, + "dup_id": dup_id, + }, + ) + + +async def _migrate_scholar_links( + db_session: AsyncSession, + *, + winner_id: int, + dup_id: int, +) -> None: + """Move ScholarPublication links from dup to winner, dropping conflicts.""" + dup_links_result = await db_session.execute( + select(ScholarPublication).where(ScholarPublication.publication_id == dup_id) + ) + dup_links = dup_links_result.scalars().all() + + winner_profiles_result = await db_session.execute( + select(ScholarPublication.scholar_profile_id).where( + ScholarPublication.publication_id == winner_id + ) + ) + winner_profiles: set[int] = {row for (row,) in winner_profiles_result} + + for link in dup_links: + if link.scholar_profile_id in winner_profiles: + await db_session.delete(link) + else: + link.publication_id = winner_id + + +async def sweep_identifier_duplicates(db_session: AsyncSession) -> int: + """Find publications sharing an identifier and merge duplicates into the winner. + + Returns the number of duplicate publications removed. + """ + pairs = await find_identifier_duplicate_pairs(db_session) + if not pairs: + return 0 + + # Deduplicate the pairs — a dup may appear multiple times if it shares + # several identifiers with the winner; process each dup only once. + processed_dups: set[int] = set() + for winner_id, dup_id in pairs: + if dup_id in processed_dups: + continue + processed_dups.add(dup_id) + await merge_duplicate_publication(db_session, winner_id=winner_id, dup_id=dup_id) + + await db_session.flush() + return len(processed_dups) diff --git a/app/services/domains/publications/listing.py b/app/services/domains/publications/listing.py index 0d09d4c..5290f6f 100644 --- a/app/services/domains/publications/listing.py +++ b/app/services/domains/publications/listing.py @@ -1,5 +1,7 @@ from __future__ import annotations +from datetime import datetime + from sqlalchemy.ext.asyncio import AsyncSession from app.services.domains.publication_identifiers import application as identifier_service @@ -9,7 +11,7 @@ from app.services.domains.publications.modes import ( resolve_publication_view_mode, ) from app.services.domains.publications.queries import ( - get_latest_completed_run_id_for_user, + get_latest_run_id_for_user, get_publication_item_for_user, publication_list_item_from_row, publications_query, @@ -25,11 +27,15 @@ async def list_for_user( mode: str = MODE_ALL, scholar_profile_id: int | None = None, favorite_only: bool = False, - limit: int = 300, + search: str | None = None, + sort_by: str = "first_seen", + sort_dir: str = "desc", + limit: int = 100, offset: int = 0, + snapshot_before: datetime | None = None, ) -> list[PublicationListItem]: resolved_mode = resolve_publication_view_mode(mode) - latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) + latest_run_id = await get_latest_run_id_for_user(db_session, user_id=user_id) result = await db_session.execute( publications_query( user_id=user_id, @@ -37,8 +43,12 @@ async def list_for_user( latest_run_id=latest_run_id, scholar_profile_id=scholar_profile_id, favorite_only=favorite_only, + search=search, + sort_by=sort_by, + sort_dir=sort_dir, limit=limit, offset=offset, + snapshot_before=snapshot_before, ) ) rows = [ diff --git a/app/services/domains/publications/pdf_queue.py b/app/services/domains/publications/pdf_queue.py index 3972f7d..7503138 100644 --- a/app/services/domains/publications/pdf_queue.py +++ b/app/services/domains/publications/pdf_queue.py @@ -2,10 +2,10 @@ from __future__ import annotations import asyncio from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone import logging -from sqlalchemy import Select, func, literal, or_, select, union_all +from sqlalchemy import Select, and_, func, literal, or_, select, union_all from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import ( @@ -20,7 +20,6 @@ from app.db.session import get_session_factory from app.services.domains.publication_identifiers import application as identifier_service from app.services.domains.publication_identifiers.types import DisplayIdentifier from app.services.domains.publications.pdf_resolution_pipeline import ( - PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE, resolve_publication_pdf_outcome_for_row, ) from app.services.domains.publications.types import PublicationListItem @@ -49,7 +48,6 @@ _scheduled_tasks: set[asyncio.Task[None]] = set() class PdfQueueListItem: publication_id: int title: str - doi: str | None pdf_url: str | None status: str attempt_count: int @@ -114,7 +112,6 @@ def _item_from_row_and_job( citation_count=row.citation_count, venue_text=row.venue_text, pub_url=row.pub_url, - doi=row.doi, pdf_url=row.pdf_url, is_read=row.is_read, is_favorite=row.is_favorite, @@ -360,7 +357,7 @@ def _failed_outcome( ) -> OaResolutionOutcome: return OaResolutionOutcome( publication_id=row.publication_id, - doi=row.doi, + doi=None, pdf_url=None, failure_reason=FAILURE_RESOLUTION_EXCEPTION, source=None, @@ -372,10 +369,12 @@ async def _fetch_outcome_for_row( *, row: PublicationListItem, request_email: str | None, + openalex_api_key: str | None = None, ) -> OaResolutionOutcome: pipeline_result = await resolve_publication_pdf_outcome_for_row( row=row, request_email=request_email, + openalex_api_key=openalex_api_key, ) outcome = pipeline_result.outcome if outcome is not None: @@ -386,11 +385,8 @@ async def _fetch_outcome_for_row( def _apply_publication_update( publication: Publication, *, - doi: str | None, pdf_url: str | None, ) -> None: - if doi and publication.doi != doi: - publication.doi = doi if pdf_url and publication.pdf_url != pdf_url: publication.pdf_url = pdf_url @@ -426,7 +422,7 @@ async def _persist_outcome( job = await db_session.get(PublicationPdfJob, publication_id) if publication is None or job is None: return - _apply_publication_update(publication, doi=outcome.doi, pdf_url=outcome.pdf_url) + _apply_publication_update(publication, pdf_url=outcome.pdf_url) await identifier_service.sync_identifiers_for_publication_resolution( db_session, publication=publication, @@ -453,10 +449,26 @@ async def _resolve_publication_row( user_id: int, request_email: str | None, row: PublicationListItem, + openalex_api_key: str | None = None, ) -> None: + from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError + from app.services.domains.arxiv.application import ArxivRateLimitError await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id) try: - outcome = await _fetch_outcome_for_row(row=row, request_email=request_email) + outcome = await _fetch_outcome_for_row( + row=row, + request_email=request_email, + openalex_api_key=openalex_api_key, + ) + except (OpenAlexBudgetExhaustedError, ArxivRateLimitError): + # Persist a terminal outcome so jobs do not remain stuck in "running". + await _persist_outcome( + publication_id=row.publication_id, + user_id=user_id, + outcome=_failed_outcome(row=row), + ) + # Propagate upward so the batch loop can stop immediately. + raise except Exception as exc: # pragma: no cover - defensive network boundary logger.warning( "publications.pdf_queue.resolve_failed", @@ -480,12 +492,44 @@ async def _run_resolution_task( request_email: str | None, rows: list[PublicationListItem], ) -> None: + from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError + from app.services.domains.arxiv.application import ArxivRateLimitError + from app.services.domains.settings import application as user_settings_service + + # Resolve the best available API key: per-user setting → env var fallback. + openalex_api_key: str | None = None + try: + session_factory = get_session_factory() + async with session_factory() as key_session: + user_settings = await user_settings_service.get_or_create_settings(key_session, user_id=user_id) + openalex_api_key = getattr(user_settings, "openalex_api_key", None) or settings.openalex_api_key + except Exception: + openalex_api_key = settings.openalex_api_key + for row in rows: - await _resolve_publication_row( - user_id=user_id, - request_email=request_email, - row=row, - ) + try: + await _resolve_publication_row( + user_id=user_id, + request_email=request_email, + row=row, + openalex_api_key=openalex_api_key, + ) + except OpenAlexBudgetExhaustedError: + logger.warning( + "publications.pdf_queue.budget_exhausted", + extra={"event": "publications.pdf_queue.budget_exhausted", + "detail": "Stopping PDF resolution batch — OpenAlex daily budget exhausted"}, + ) + break + except ArxivRateLimitError: + logger.warning( + "publications.pdf_queue.arxiv_rate_limited", + extra={ + "event": "publications.pdf_queue.arxiv_rate_limited", + "detail": "Stopping PDF resolution batch — arXiv rate limit hit (429)", + }, + ) + break def _schedule_rows( @@ -571,7 +615,6 @@ def _retry_item_from_publication( citation_count=int(publication.citation_count or 0), venue_text=publication.venue_text, pub_url=publication.pub_url, - doi=publication.doi, pdf_url=publication.pdf_url, is_read=is_read, first_seen_at=first_seen_at, @@ -629,13 +672,12 @@ def _queue_candidate_from_publication(publication: Publication) -> PublicationLi return PublicationListItem( publication_id=int(publication.id), scholar_profile_id=0, - scholar_label="admin", + scholar_label="", title=publication.title_raw, year=publication.year, citation_count=int(publication.citation_count or 0), venue_text=publication.venue_text, pub_url=publication.pub_url, - doi=publication.doi, pdf_url=publication.pdf_url, is_read=True, first_seen_at=publication.created_at or _utcnow(), @@ -649,6 +691,9 @@ async def _missing_pdf_candidates( limit: int, ) -> list[PublicationListItem]: bounded_limit = max(1, min(int(limit), 5000)) + now = datetime.now(timezone.utc) + cooldown_threshold = now - timedelta(days=7) + result = await db_session.execute( select(Publication) .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id) @@ -656,7 +701,13 @@ async def _missing_pdf_candidates( .where( or_( PublicationPdfJob.publication_id.is_(None), - PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]), + and_( + PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]), + or_( + PublicationPdfJob.last_attempt_at.is_(None), + PublicationPdfJob.last_attempt_at < cooldown_threshold, + ), + ), ) ) .order_by(Publication.updated_at.desc(), Publication.id.desc()) @@ -694,7 +745,6 @@ def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]: select( PublicationPdfJob.publication_id, Publication.title_raw, - Publication.doi, Publication.pdf_url, PublicationPdfJob.status, PublicationPdfJob.attempt_count, @@ -730,7 +780,6 @@ def _untracked_queue_select_base() -> Select[tuple]: select( Publication.id, Publication.title_raw, - Publication.doi, Publication.pdf_url, literal(PDF_STATUS_UNTRACKED), literal(0), @@ -793,19 +842,18 @@ def _queue_item_from_row(row: tuple) -> PdfQueueListItem: return PdfQueueListItem( publication_id=int(row[0]), title=str(row[1] or ""), - doi=row[2], - pdf_url=row[3], - status=str(row[4] or PDF_STATUS_UNTRACKED), - attempt_count=int(row[5] or 0), - last_failure_reason=row[6], - last_failure_detail=row[7], - last_source=row[8], - requested_by_user_id=int(row[9]) if row[9] is not None else None, - requested_by_email=row[10], - queued_at=row[11], - last_attempt_at=row[12], - resolved_at=row[13], - updated_at=row[14], + pdf_url=row[2], + status=str(row[3] or PDF_STATUS_UNTRACKED), + attempt_count=int(row[4] or 0), + last_failure_reason=row[5], + last_failure_detail=row[6], + last_source=row[7], + requested_by_user_id=int(row[8]) if row[8] is not None else None, + requested_by_email=row[9], + queued_at=row[10], + last_attempt_at=row[11], + resolved_at=row[12], + updated_at=row[13], ) @@ -902,3 +950,25 @@ async def list_pdf_queue_page( limit=bounded_limit, offset=bounded_offset, ) + + +async def drain_ready_jobs( + db_session: AsyncSession, + *, + limit: int, + max_attempts: int, +) -> int: + result = await db_session.execute( + select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1) + ) + system_user_id = result.scalar_one_or_none() + if system_user_id is None: + return 0 + + bulk_result = await enqueue_all_missing_pdf_jobs( + db_session, + user_id=system_user_id, + request_email=settings.unpaywall_email, + limit=limit, + ) + return bulk_result.queued_count diff --git a/app/services/domains/publications/pdf_resolution_pipeline.py b/app/services/domains/publications/pdf_resolution_pipeline.py index bc41257..0dc133c 100644 --- a/app/services/domains/publications/pdf_resolution_pipeline.py +++ b/app/services/domains/publications/pdf_resolution_pipeline.py @@ -2,100 +2,114 @@ from __future__ import annotations from dataclasses import dataclass import logging -from urllib.parse import urlparse - -import httpx +from typing import Any +from app.services.domains.arxiv.application import ArxivRateLimitError +from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError from app.services.domains.publications.types import PublicationListItem -from app.services.domains.scholar.publication_pdf import ( - ScholarPublicationLinkCandidate, - ScholarPublicationLinkCandidates, - fetch_link_candidates_from_scholar_publication_page, -) -from app.services.domains.unpaywall import pdf_discovery as pdf_discovery_service from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes from app.settings import settings logger = logging.getLogger(__name__) -PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE = "scholar_publication_page" -PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED = "scholar_publication_page_unlabeled_fallback" -PDF_PATH_TOKEN = "/pdf/" -HTTP_TIMEOUT_FLOOR_SECONDS = 0.5 - @dataclass(frozen=True) class PipelineOutcome: outcome: OaResolutionOutcome | None - scholar_candidates: ScholarPublicationLinkCandidates | None + scholar_candidates: Any | None # Kept for backward compatibility with calling signatures async def resolve_publication_pdf_outcome_for_row( *, row: PublicationListItem, request_email: str | None, + openalex_api_key: str | None = None, ) -> PipelineOutcome: - candidates = await _safe_scholar_candidates(row.pub_url) - labeled = _labeled_candidate(candidates) - if labeled is not None: - return PipelineOutcome(_scholar_outcome(row=row, candidate=labeled), candidates) + # 1. OpenAlex OA — raises OpenAlexBudgetExhaustedError if budget is gone + openalex_outcome = await _openalex_outcome(row, request_email=request_email, openalex_api_key=openalex_api_key) + if openalex_outcome and openalex_outcome.pdf_url: + return PipelineOutcome(openalex_outcome, None) + + # 2. arXiv + arxiv_outcome = await _arxiv_outcome(row, request_email=request_email) + if arxiv_outcome and arxiv_outcome.pdf_url: + return PipelineOutcome(arxiv_outcome, None) + + # 3. Unpaywall (which falls back to Crossref) oa_outcome = await _oa_outcome(row=row, request_email=request_email) - if _oa_has_pdf(oa_outcome): - return PipelineOutcome(oa_outcome, candidates) - unlabeled = _unlabeled_candidate(candidates) - if unlabeled is None: - return PipelineOutcome(oa_outcome, candidates) - fallback_outcome = await _unlabeled_fallback_outcome(row=row, candidate=unlabeled) - if fallback_outcome is not None: - return PipelineOutcome(fallback_outcome, candidates) - return PipelineOutcome(oa_outcome, candidates) + return PipelineOutcome(oa_outcome, None) -async def _safe_scholar_candidates(pub_url: str | None) -> ScholarPublicationLinkCandidates | None: - try: - return await fetch_link_candidates_from_scholar_publication_page(pub_url) - except Exception as exc: # pragma: no cover - defensive boundary - logger.warning( - "publications.pdf_resolution.scholar_candidates_failed", - extra={"event": "publications.pdf_resolution.scholar_candidates_failed", "error": str(exc)}, - ) - return None - - -def _labeled_candidate( - candidates: ScholarPublicationLinkCandidates | None, -) -> ScholarPublicationLinkCandidate | None: - if candidates is None: - return None - return candidates.labeled_candidate - - -def _unlabeled_candidate( - candidates: ScholarPublicationLinkCandidates | None, -) -> ScholarPublicationLinkCandidate | None: - if candidates is None: - return None - return candidates.fallback_candidate - - -def _scholar_outcome( - *, +async def _openalex_outcome( row: PublicationListItem, - candidate: ScholarPublicationLinkCandidate, -) -> OaResolutionOutcome: - source = ( - PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE - if candidate.label_present - else PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED - ) - return OaResolutionOutcome( - publication_id=row.publication_id, - doi=row.doi, - pdf_url=candidate.url, - failure_reason=None, - source=source, - used_crossref=False, - ) + request_email: str | None, + openalex_api_key: str | None = None, +) -> OaResolutionOutcome | None: + from app.services.domains.openalex.client import OpenAlexClient + from app.services.domains.openalex.matching import find_best_match + + if not row.title: + return None + + import re + safe_title = re.sub(r"[^\w\s]", " ", row.title) + safe_title = " ".join(safe_title.split()) + if not safe_title: + return None + + api_key = openalex_api_key or settings.openalex_api_key + client = OpenAlexClient(api_key=api_key, mailto=request_email or settings.crossref_api_mailto) + try: + openalex_works = await client.get_works_by_filter({"title.search": safe_title}, limit=5) + match = find_best_match( + target_title=row.title, + target_year=row.year, + target_authors=row.scholar_label, + candidates=openalex_works, + ) + if match and match.oa_url: + return OaResolutionOutcome( + publication_id=row.publication_id, + doi=match.doi, + pdf_url=match.oa_url, + failure_reason=None, + source="openalex", + used_crossref=False, + ) + except OpenAlexBudgetExhaustedError: + # Re-raise so the caller's batch loop can stop hitting the API. + raise + except Exception as exc: + logger.warning( + "publications.pdf_resolution.openalex_failed", + extra={"event": "publications.pdf_resolution.openalex_failed", "error": str(exc)}, + ) + return None + + +async def _arxiv_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None: + from app.services.domains.arxiv.application import discover_arxiv_id_for_publication + + try: + arxiv_id = await discover_arxiv_id_for_publication(item=row, request_email=request_email) + if arxiv_id: + pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf" + return OaResolutionOutcome( + publication_id=row.publication_id, + doi=None, + pdf_url=pdf_url, + failure_reason=None, + source="arxiv", + used_crossref=False, + ) + except ArxivRateLimitError: + raise # propagate so the batch loop can stop + except Exception as exc: + logger.warning( + "publications.pdf_resolution.arxiv_failed", + extra={"event": "publications.pdf_resolution.arxiv_failed", "error": str(exc)}, + ) + return None async def _oa_outcome( @@ -105,46 +119,3 @@ async def _oa_outcome( ) -> OaResolutionOutcome | None: outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email) return outcomes.get(row.publication_id) - - -def _oa_has_pdf(outcome: OaResolutionOutcome | None) -> bool: - return bool(outcome and outcome.pdf_url) - - -async def _unlabeled_fallback_outcome( - *, - row: PublicationListItem, - candidate: ScholarPublicationLinkCandidate, -) -> OaResolutionOutcome | None: - pdf_url = await _validated_pdf_url(candidate.url) - if pdf_url is None: - return None - return _scholar_outcome(row=row, candidate=ScholarPublicationLinkCandidate( - url=pdf_url, - confidence_score=candidate.confidence_score, - label_present=False, - reason=candidate.reason, - )) - - -async def _validated_pdf_url(candidate_url: str) -> str | None: - if _looks_direct_pdf(candidate_url): - return candidate_url - timeout_seconds = _discovery_timeout_seconds() - async with httpx.AsyncClient(timeout=timeout_seconds) as client: - if await pdf_discovery_service._candidate_is_pdf(client, candidate_url=candidate_url): - return candidate_url - return await pdf_discovery_service.resolve_pdf_from_landing_page(client, page_url=candidate_url) - - -def _looks_direct_pdf(url: str | None) -> bool: - if pdf_discovery_service.looks_like_pdf_url(url): - return True - if not isinstance(url, str): - return False - path = (urlparse(url).path or "").lower() - return PDF_PATH_TOKEN in path - - -def _discovery_timeout_seconds() -> float: - return max(float(settings.unpaywall_timeout_seconds), HTTP_TIMEOUT_FLOOR_SECONDS) diff --git a/app/services/domains/publications/queries.py b/app/services/domains/publications/queries.py index 292ec8e..6f57cab 100644 --- a/app/services/domains/publications/queries.py +++ b/app/services/domains/publications/queries.py @@ -1,5 +1,7 @@ from __future__ import annotations +from datetime import datetime + from sqlalchemy import Select, func, select from sqlalchemy.ext.asyncio import AsyncSession @@ -15,15 +17,24 @@ def _normalized_citation_count(value: object) -> int: return 0 -async def get_latest_completed_run_id_for_user( +async def get_latest_run_id_for_user( db_session: AsyncSession, *, user_id: int, ) -> int | None: + # We include RUNNING and RESOLVING statuses so that the "New" tab shows + # results in real-time as they are discovered. result = await db_session.execute( select(func.max(CrawlRun.id)).where( CrawlRun.user_id == user_id, - CrawlRun.status != RunStatus.RUNNING, + CrawlRun.status.in_( + [ + RunStatus.RUNNING, + RunStatus.RESOLVING, + RunStatus.SUCCESS, + RunStatus.PARTIAL_FAILURE, + ] + ), ) ) latest_run_id = result.scalar_one_or_none() @@ -39,7 +50,19 @@ def publications_query( favorite_only: bool, limit: int, offset: int = 0, + search: str | None = None, + sort_by: str = "first_seen", + sort_dir: str = "desc", + snapshot_before: datetime | None = None, ) -> Select[tuple]: + _SORT_COLUMNS = { + "first_seen": ScholarPublication.created_at, + "title": Publication.title_raw, + "year": Publication.year, + "citations": Publication.citation_count, + "scholar": ScholarProfile.display_name, + } + scholar_label = ScholarProfile.display_name stmt = ( select( @@ -52,7 +75,6 @@ def publications_query( Publication.citation_count, Publication.venue_text, Publication.pub_url, - Publication.doi, Publication.pdf_url, ScholarPublication.is_read, ScholarPublication.is_favorite, @@ -62,10 +84,15 @@ def publications_query( .join(ScholarPublication, ScholarPublication.publication_id == Publication.id) .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) .where(ScholarProfile.user_id == user_id) - .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) - .offset(max(int(offset), 0)) - .limit(limit) ) + if search: + safe_search = search.replace("%", r"\%").replace("_", r"\_") + pat = f"%{safe_search}%" + stmt = stmt.where( + Publication.title_raw.ilike(pat) + | ScholarProfile.display_name.ilike(pat) + | Publication.venue_text.ilike(pat) + ) if scholar_profile_id is not None: stmt = stmt.where(ScholarProfile.id == scholar_profile_id) if favorite_only: @@ -76,6 +103,16 @@ def publications_query( if latest_run_id is None: return stmt.where(False) stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id) + if snapshot_before is not None: + stmt = stmt.where(ScholarPublication.created_at <= snapshot_before) + + sort_col = _SORT_COLUMNS.get(sort_by, ScholarPublication.created_at) + order = sort_col.desc() if sort_dir == "desc" else sort_col.asc() + stmt = stmt.order_by(order, Publication.id.desc()) + + if limit is not None: + stmt = stmt.offset(max(int(offset), 0)).limit(limit) + return stmt @@ -96,7 +133,6 @@ def publication_query_for_user( Publication.citation_count, Publication.venue_text, Publication.pub_url, - Publication.doi, Publication.pdf_url, ScholarPublication.is_read, ScholarPublication.is_favorite, @@ -121,7 +157,7 @@ async def get_publication_item_for_user( scholar_profile_id: int, publication_id: int, ) -> PublicationListItem | None: - latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) + latest_run_id = await get_latest_run_id_for_user(db_session, user_id=user_id) result = await db_session.execute( publication_query_for_user( user_id=user_id, @@ -150,7 +186,6 @@ def publication_list_item_from_row( citation_count, venue_text, pub_url, - doi, pdf_url, is_read, is_favorite, @@ -166,7 +201,6 @@ def publication_list_item_from_row( citation_count=_normalized_citation_count(citation_count), venue_text=venue_text, pub_url=pub_url, - doi=doi, pdf_url=pdf_url, is_read=bool(is_read), is_favorite=bool(is_favorite), @@ -188,7 +222,6 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem: citation_count, venue_text, pub_url, - doi, pdf_url, _is_read, _is_favorite, @@ -204,6 +237,5 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem: citation_count=_normalized_citation_count(citation_count), venue_text=venue_text, pub_url=pub_url, - doi=doi, pdf_url=pdf_url, ) diff --git a/app/services/domains/publications/types.py b/app/services/domains/publications/types.py index 3bde82d..62d038b 100644 --- a/app/services/domains/publications/types.py +++ b/app/services/domains/publications/types.py @@ -16,7 +16,6 @@ class PublicationListItem: citation_count: int venue_text: str | None pub_url: str | None - doi: str | None pdf_url: str | None is_read: bool first_seen_at: datetime @@ -39,5 +38,4 @@ class UnreadPublicationItem: citation_count: int venue_text: str | None pub_url: str | None - doi: str | None pdf_url: str | None diff --git a/app/services/domains/runs/events.py b/app/services/domains/runs/events.py new file mode 100644 index 0000000..ef28069 --- /dev/null +++ b/app/services/domains/runs/events.py @@ -0,0 +1,59 @@ +import asyncio +import json +import logging +from typing import Any, AsyncGenerator, Dict, Set + +logger = logging.getLogger(__name__) + +class RunEventPublisher: + def __init__(self) -> None: + # Maps run_id to a set of subscriber queues + self._subscribers: Dict[int, Set[asyncio.Queue]] = {} + + def subscribe(self, run_id: int) -> asyncio.Queue: + if run_id not in self._subscribers: + self._subscribers[run_id] = set() + queue: asyncio.Queue[Any] = asyncio.Queue() + self._subscribers[run_id].add(queue) + logger.debug(f"New subscriber for run {run_id}. Total: {len(self._subscribers[run_id])}") + return queue + + def unsubscribe(self, run_id: int, queue: asyncio.Queue) -> None: + if run_id in self._subscribers: + self._subscribers[run_id].discard(queue) + if not self._subscribers[run_id]: + self._subscribers.pop(run_id, None) + + async def publish(self, run_id: int, event_type: str, data: dict[str, Any]) -> None: + if run_id not in self._subscribers: + return + + message = { + "type": event_type, + "data": data + } + + # Fan-out to all active subscribers for this run + for queue in list(self._subscribers[run_id]): + try: + queue.put_nowait(message) + except asyncio.QueueFull: + logger.warning(f"Subscriber queue full for run {run_id}, dropping message") + +run_events = RunEventPublisher() + +async def event_generator(run_id: int) -> AsyncGenerator[str, None]: + queue = run_events.subscribe(run_id) + try: + while True: + # Wait for a new event + message = await queue.get() + # Server-Sent Events format: "event: \ndata: \n\n" + event_type = message["type"] + data_str = json.dumps(message["data"]) + yield f"event: {event_type}\ndata: {data_str}\n\n" + except asyncio.CancelledError: + logger.debug(f"Client disconnected from SSE stream for run {run_id}") + raise + finally: + run_events.unsubscribe(run_id, queue) diff --git a/app/services/domains/scholar/publication_pdf.py b/app/services/domains/scholar/publication_pdf.py deleted file mode 100644 index 6f8b6c1..0000000 --- a/app/services/domains/scholar/publication_pdf.py +++ /dev/null @@ -1,232 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from html.parser import HTMLParser -from urllib.parse import parse_qs, urlparse - -from app.services.domains.scholar.parser_types import ScholarDomInvariantError -from app.services.domains.scholar.parser_utils import attr_href, normalize_space -from app.services.domains.scholar.rate_limit import wait_for_scholar_slot -from app.services.domains.scholar.source import LiveScholarSource -from app.services.domains.settings.application import resolve_request_delay_minimum -from app.settings import settings - -CONTAINER_ID = "gsc_oci_title_gg" -PDF_LABEL_TOKEN = "[pdf]" -SCHOLAR_PDF_LABELED_CONFIDENCE = 0.98 -SCHOLAR_PDF_UNLABELED_CONFIDENCE = 0.2 -ALLOWED_URL_SCHEMES = frozenset({"http", "https"}) - - -@dataclass(frozen=True) -class ScholarPublicationLinkCandidate: - url: str - confidence_score: float - label_present: bool - reason: str - - -@dataclass(frozen=True) -class ScholarPublicationLinkCandidates: - container_seen: bool - labeled_candidate: ScholarPublicationLinkCandidate | None - fallback_candidate: ScholarPublicationLinkCandidate | None - warnings: tuple[str, ...] = () - - -@dataclass(frozen=True) -class _ParsedAnchor: - href: str | None - text: str - - -class _ScholarPublicationPdfParser(HTMLParser): - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.container_seen = False - self.anchors: list[_ParsedAnchor] = [] - self._container_depth = 0 - self._anchor_depth = 0 - self._anchor_href: str | None = None - self._anchor_parts: list[str] = [] - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - self._increment_depths() - if self._starts_container(tag, attrs): - self.container_seen = True - self._container_depth = 1 - return - if self._container_depth <= 0 or tag != "a": - return - if self._anchor_depth > 0: - return - self._anchor_href = attr_href(attrs) - self._anchor_parts = [] - self._anchor_depth = 1 - - def handle_data(self, data: str) -> None: - if self._anchor_depth > 0: - self._anchor_parts.append(data) - - def handle_endtag(self, tag: str) -> None: - if self._anchor_depth > 0: - self._anchor_depth -= 1 - if self._anchor_depth == 0: - self._finish_anchor() - if self._container_depth > 0: - self._container_depth -= 1 - - def _increment_depths(self) -> None: - if self._container_depth > 0: - self._container_depth += 1 - if self._anchor_depth > 0: - self._anchor_depth += 1 - - def _starts_container(self, tag: str, attrs: list[tuple[str, str | None]]) -> bool: - if tag != "div": - return False - attrs_map = {name.lower(): (value or "") for name, value in attrs} - return attrs_map.get("id") == CONTAINER_ID - - def _finish_anchor(self) -> None: - self.anchors.append( - _ParsedAnchor( - href=self._anchor_href, - text=normalize_space("".join(self._anchor_parts)), - ) - ) - self._anchor_href = None - self._anchor_parts = [] - - -def is_scholar_publication_detail_url(url: str | None) -> bool: - if not isinstance(url, str) or not url.strip(): - return False - parsed = urlparse(url) - if parsed.scheme not in ALLOWED_URL_SCHEMES: - return False - if parsed.netloc.lower() != "scholar.google.com": - return False - query = parse_qs(parsed.query) - return _has_view_citation_params(query) - - -def _has_view_citation_params(query: dict[str, list[str]]) -> bool: - view_op = (query.get("view_op") or [""])[0] - citation = (query.get("citation_for_view") or [""])[0].strip() - return view_op == "view_citation" and bool(citation) - - -def extract_link_candidates_from_publication_detail_html(html: str) -> ScholarPublicationLinkCandidates: - parser = _parsed_publication_detail(html) - if not parser.container_seen: - return ScholarPublicationLinkCandidates(False, None, None) - anchors = _validated_container_anchors(parser.anchors) - labeled = _select_labeled_candidate(anchors) - fallback = _select_fallback_candidate(anchors, labeled=labeled) - warnings = _candidate_warnings(labeled=labeled, fallback=fallback) - return ScholarPublicationLinkCandidates(True, labeled, fallback, warnings) - - -def _parsed_publication_detail(html: str) -> _ScholarPublicationPdfParser: - parser = _ScholarPublicationPdfParser() - parser.feed(html) - parser.close() - return parser - - -def _validated_container_anchors(anchors: list[_ParsedAnchor]) -> list[_ParsedAnchor]: - if not anchors: - raise ScholarDomInvariantError( - code="layout_publication_link_container_missing_anchor", - message="Scholar publication link container was present without an anchor.", - ) - validated: list[_ParsedAnchor] = [] - for anchor in anchors: - validated.append(_validated_anchor(anchor)) - return validated - - -def _validated_anchor(anchor: _ParsedAnchor) -> _ParsedAnchor: - href = (anchor.href or "").strip() - if not href: - raise ScholarDomInvariantError( - code="layout_publication_link_missing_href", - message="Scholar publication link container anchor was missing href.", - ) - parsed = urlparse(href) - if parsed.scheme not in ALLOWED_URL_SCHEMES: - raise ScholarDomInvariantError( - code="layout_publication_link_invalid_scheme", - message="Scholar publication link used a non-http URL.", - ) - return _ParsedAnchor(href=href, text=anchor.text) - - -def _select_labeled_candidate(anchors: list[_ParsedAnchor]) -> ScholarPublicationLinkCandidate | None: - for anchor in anchors: - if PDF_LABEL_TOKEN in anchor.text.lower(): - return ScholarPublicationLinkCandidate( - url=str(anchor.href), - confidence_score=SCHOLAR_PDF_LABELED_CONFIDENCE, - label_present=True, - reason="scholar_link_labeled_pdf", - ) - return None - - -def _select_fallback_candidate( - anchors: list[_ParsedAnchor], - *, - labeled: ScholarPublicationLinkCandidate | None, -) -> ScholarPublicationLinkCandidate | None: - for anchor in anchors: - if labeled and anchor.href == labeled.url: - continue - return ScholarPublicationLinkCandidate( - url=str(anchor.href), - confidence_score=SCHOLAR_PDF_UNLABELED_CONFIDENCE, - label_present=False, - reason="scholar_link_unlabeled_fallback", - ) - if labeled is None and anchors: - anchor = anchors[0] - return ScholarPublicationLinkCandidate( - url=str(anchor.href), - confidence_score=SCHOLAR_PDF_UNLABELED_CONFIDENCE, - label_present=False, - reason="scholar_link_unlabeled_fallback", - ) - return None - - -def _candidate_warnings( - *, - labeled: ScholarPublicationLinkCandidate | None, - fallback: ScholarPublicationLinkCandidate | None, -) -> tuple[str, ...]: - warnings: list[str] = [] - if labeled is None and fallback is not None: - warnings.append("scholar_publication_link_unlabeled_only") - return tuple(warnings) - - -def _scholar_request_delay_seconds() -> int: - return resolve_request_delay_minimum(settings.ingestion_min_request_delay_seconds) - - -def _fetch_succeeded(fetch_result) -> bool: - return int(fetch_result.status_code or 0) == 200 and not fetch_result.error - - -async def fetch_link_candidates_from_scholar_publication_page( - publication_url: str | None, -) -> ScholarPublicationLinkCandidates | None: - if not is_scholar_publication_detail_url(publication_url): - return None - await wait_for_scholar_slot(min_interval_seconds=float(_scholar_request_delay_seconds())) - source = LiveScholarSource() - fetch_result = await source.fetch_publication_html(str(publication_url)) - if not _fetch_succeeded(fetch_result): - return None - return extract_link_candidates_from_publication_detail_html(fetch_result.body) diff --git a/app/services/domains/settings/application.py b/app/services/domains/settings/application.py index 3548e44..7f435e8 100644 --- a/app/services/domains/settings/application.py +++ b/app/services/domains/settings/application.py @@ -109,6 +109,8 @@ def parse_nav_visible_pages(value: object) -> list[str]: return deduped +from app.settings import settings as app_settings + async def get_or_create_settings( db_session: AsyncSession, *, @@ -121,7 +123,12 @@ async def get_or_create_settings( if settings is not None: return settings - settings = UserSetting(user_id=user_id) + settings = UserSetting( + user_id=user_id, + openalex_api_key=app_settings.openalex_api_key, + crossref_api_token=app_settings.crossref_api_token, + crossref_api_mailto=app_settings.crossref_api_mailto, + ) db_session.add(settings) await db_session.commit() await db_session.refresh(settings) @@ -136,11 +143,17 @@ async def update_settings( run_interval_minutes: int, request_delay_seconds: int, nav_visible_pages: list[str], + openalex_api_key: str | None, + crossref_api_token: str | None, + crossref_api_mailto: str | None, ) -> UserSetting: settings.auto_run_enabled = auto_run_enabled settings.run_interval_minutes = run_interval_minutes settings.request_delay_seconds = request_delay_seconds settings.nav_visible_pages = nav_visible_pages + settings.openalex_api_key = openalex_api_key + settings.crossref_api_token = crossref_api_token + settings.crossref_api_mailto = crossref_api_mailto await db_session.commit() await db_session.refresh(settings) return settings diff --git a/app/services/domains/unpaywall/application.py b/app/services/domains/unpaywall/application.py index a658818..13104e9 100644 --- a/app/services/domains/unpaywall/application.py +++ b/app/services/domains/unpaywall/application.py @@ -63,7 +63,10 @@ def _extract_explicit_doi(text: str | None) -> str | None: def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str | None: - stored = normalize_doi(item.doi) + stored = None + if getattr(item, "display_identifier", None) and item.display_identifier.kind == "doi": + stored = normalize_doi(item.display_identifier.value) + explicit_doi = ( _extract_explicit_doi(item.pub_url) or _extract_explicit_doi(item.venue_text) @@ -169,9 +172,11 @@ async def _fetch_unpaywall_payload_by_doi( email: str, ) -> dict | None: await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds) + headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{email})"} response = await client.get( UNPAYWALL_URL_TEMPLATE.format(doi=doi), params={"email": email}, + headers=headers, ) if response.status_code != 200: return None @@ -427,7 +432,8 @@ async def resolve_publication_oa_outcomes( timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5) targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)] - async with httpx.AsyncClient(timeout=timeout_seconds) as client: + headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{email})"} + async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True, headers=headers) as client: outcomes = await _resolve_outcomes_with_client( client=client, targets=targets, diff --git a/app/settings.py b/app/settings.py index 43871cf..15b0c19 100644 --- a/app/settings.py +++ b/app/settings.py @@ -140,6 +140,11 @@ class Settings: "INGESTION_RETRY_BACKOFF_SECONDS", 1.0, ) + ingestion_rate_limit_retries: int = _env_int("INGESTION_RATE_LIMIT_RETRIES", 3) + ingestion_rate_limit_backoff_seconds: float = _env_float( + "INGESTION_RATE_LIMIT_BACKOFF_SECONDS", + 30.0, + ) ingestion_max_pages_per_scholar: int = _env_int( "INGESTION_MAX_PAGES_PER_SCHOLAR", 30, @@ -182,6 +187,7 @@ class Settings: 6, ) scheduler_queue_batch_size: int = _env_int("SCHEDULER_QUEUE_BATCH_SIZE", 10) + scheduler_pdf_queue_batch_size: int = _env_int("SCHEDULER_PDF_QUEUE_BATCH_SIZE", 15) frontend_enabled: bool = _env_bool("FRONTEND_ENABLED", True) frontend_dist_dir: str = _env_str("FRONTEND_DIST_DIR", "/app/frontend/dist") scholar_image_upload_dir: str = _env_str( @@ -241,9 +247,9 @@ class Settings: ) pdf_auto_retry_first_interval_seconds: int = _env_int( "PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS", - 3_600, + 86_400, ) - pdf_auto_retry_max_attempts: int = _env_int("PDF_AUTO_RETRY_MAX_ATTEMPTS", 3) + pdf_auto_retry_max_attempts: int = _env_int("PDF_AUTO_RETRY_MAX_ATTEMPTS", 2) unpaywall_pdf_discovery_enabled: bool = _env_bool("UNPAYWALL_PDF_DISCOVERY_ENABLED", True) unpaywall_pdf_discovery_max_candidates: int = _env_int("UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES", 5) unpaywall_pdf_discovery_max_html_bytes: int = _env_int("UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES", 500_000) @@ -252,6 +258,10 @@ class Settings: crossref_timeout_seconds: float = _env_float("CROSSREF_TIMEOUT_SECONDS", 8.0) crossref_min_interval_seconds: float = _env_float("CROSSREF_MIN_INTERVAL_SECONDS", 0.6) crossref_max_lookups_per_request: int = _env_int("CROSSREF_MAX_LOOKUPS_PER_REQUEST", 8) + + openalex_api_key: str | None = os.getenv("OPENALEX_API_KEY") + crossref_api_token: str | None = os.getenv("CROSSREF_API_TOKEN") + crossref_api_mailto: str | None = os.getenv("CROSSREF_API_MAILTO") settings = Settings() diff --git a/build/lib/app/__init__.py b/build/lib/app/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/build/lib/app/__init__.py @@ -0,0 +1 @@ + diff --git a/build/lib/app/api/__init__.py b/build/lib/app/api/__init__.py new file mode 100644 index 0000000..3ce55a6 --- /dev/null +++ b/build/lib/app/api/__init__.py @@ -0,0 +1,2 @@ +from __future__ import annotations + diff --git a/build/lib/app/api/deps.py b/build/lib/app/api/deps.py new file mode 100644 index 0000000..f9c5377 --- /dev/null +++ b/build/lib/app/api/deps.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from fastapi import Depends, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.errors import ApiException +from app.auth import runtime as auth_runtime +from app.db.models import User +from app.db.session import get_db_session + + +async def get_api_current_user( + request: Request, + db_session: AsyncSession = Depends(get_db_session), +) -> User: + current_user = await auth_runtime.get_authenticated_user(request, db_session) + if current_user is None: + raise ApiException( + status_code=401, + code="auth_required", + message="Authentication required.", + ) + return current_user + + +async def get_api_admin_user( + current_user: User = Depends(get_api_current_user), +) -> User: + if not current_user.is_admin: + raise ApiException( + status_code=403, + code="forbidden", + message="Admin access required.", + ) + return current_user diff --git a/build/lib/app/api/errors.py b/build/lib/app/api/errors.py new file mode 100644 index 0000000..976ccb8 --- /dev/null +++ b/build/lib/app/api/errors.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exception_handlers import ( + http_exception_handler as fastapi_http_exception_handler, +) +from fastapi.exception_handlers import ( + request_validation_exception_handler as fastapi_validation_exception_handler, +) +from fastapi.exceptions import RequestValidationError + +from app.api.responses import error_response + + +def _is_api_path(path: str) -> bool: + return path.startswith("/api/") + + +def _status_code_to_error_code(status_code: int) -> str: + mapping = { + 400: "bad_request", + 401: "auth_required", + 403: "forbidden", + 404: "not_found", + 409: "conflict", + 422: "validation_error", + 429: "rate_limited", + 500: "internal_error", + } + return mapping.get(status_code, "error") + + +class ApiException(Exception): + def __init__( + self, + *, + status_code: int, + code: str, + message: str, + details: Any | None = None, + ) -> None: + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.details = details + + +def register_api_exception_handlers(app: FastAPI) -> None: + @app.exception_handler(ApiException) + async def _handle_api_exception(request: Request, exc: ApiException): + return error_response( + request, + status_code=exc.status_code, + code=exc.code, + message=exc.message, + details=exc.details, + ) + + @app.exception_handler(HTTPException) + async def _handle_http_exception(request: Request, exc: HTTPException): + if not _is_api_path(request.url.path): + return await fastapi_http_exception_handler(request, exc) + return error_response( + request, + status_code=exc.status_code, + code=_status_code_to_error_code(exc.status_code), + message=str(exc.detail) if exc.detail is not None else "Request failed.", + details=exc.detail if isinstance(exc.detail, (dict, list)) else None, + ) + + @app.exception_handler(RequestValidationError) + async def _handle_validation_exception(request: Request, exc: RequestValidationError): + if not _is_api_path(request.url.path): + return await fastapi_validation_exception_handler(request, exc) + return error_response( + request, + status_code=422, + code="validation_error", + message="Request validation failed.", + details=exc.errors(), + ) + diff --git a/build/lib/app/api/media.py b/build/lib/app/api/media.py new file mode 100644 index 0000000..00943bd --- /dev/null +++ b/build/lib/app/api/media.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import mimetypes + +from fastapi import APIRouter, Depends +from fastapi.responses import FileResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_current_user +from app.api.errors import ApiException +from app.db.models import User +from app.db.session import get_db_session +from app.services.domains.scholars import application as scholar_service +from app.settings import settings + +router = APIRouter(tags=["media"]) + + +@router.get("/scholar-images/{scholar_profile_id}/upload") +async def get_uploaded_scholar_image( + scholar_profile_id: int, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + profile = await scholar_service.get_user_scholar_by_id( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + ) + if profile is None: + raise ApiException( + status_code=404, + code="scholar_not_found", + message="Scholar not found.", + ) + if not profile.profile_image_upload_path: + raise ApiException( + status_code=404, + code="scholar_image_not_found", + message="Scholar image not found.", + ) + + try: + image_path = scholar_service.resolve_upload_file_path( + upload_dir=settings.scholar_image_upload_dir, + relative_path=profile.profile_image_upload_path, + ) + except scholar_service.ScholarServiceError as exc: + raise ApiException( + status_code=404, + code="scholar_image_not_found", + message="Scholar image not found.", + ) from exc + + if not image_path.exists() or not image_path.is_file(): + raise ApiException( + status_code=404, + code="scholar_image_not_found", + message="Scholar image not found.", + ) + + media_type = mimetypes.guess_type(str(image_path))[0] or "application/octet-stream" + return FileResponse(path=image_path, media_type=media_type) diff --git a/build/lib/app/api/responses.py b/build/lib/app/api/responses.py new file mode 100644 index 0000000..9976aba --- /dev/null +++ b/build/lib/app/api/responses.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from typing import Any + +from fastapi.encoders import jsonable_encoder +from starlette.requests import Request +from starlette.responses import JSONResponse + + +def _request_id(request: Request) -> str | None: + request_state = getattr(request, "state", None) + if request_state is None: + return None + return getattr(request_state, "request_id", None) + + +def success_payload( + request: Request, + *, + data: Any, +) -> dict[str, Any]: + return { + "data": data, + "meta": { + "request_id": _request_id(request), + }, + } + + +def success_response( + request: Request, + *, + data: Any, + status_code: int = 200, +) -> JSONResponse: + payload = success_payload(request, data=data) + return JSONResponse( + status_code=status_code, + content=jsonable_encoder(payload), + ) + + +def error_response( + request: Request, + *, + status_code: int, + code: str, + message: str, + details: Any | None = None, +) -> JSONResponse: + payload = { + "error": { + "code": code, + "message": message, + "details": details, + }, + "meta": { + "request_id": _request_id(request), + }, + } + return JSONResponse( + status_code=status_code, + content=jsonable_encoder(payload), + ) diff --git a/build/lib/app/api/router.py b/build/lib/app/api/router.py new file mode 100644 index 0000000..a11d98a --- /dev/null +++ b/build/lib/app/api/router.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from fastapi import APIRouter + +from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings + +router = APIRouter(prefix="/api/v1") +router.include_router(auth.router) +router.include_router(admin.router) +router.include_router(admin_dbops.router) +router.include_router(scholars.router) +router.include_router(settings.router) +router.include_router(runs.router) +router.include_router(publications.router) diff --git a/build/lib/app/api/routers/__init__.py b/build/lib/app/api/routers/__init__.py new file mode 100644 index 0000000..3ce55a6 --- /dev/null +++ b/build/lib/app/api/routers/__init__.py @@ -0,0 +1,2 @@ +from __future__ import annotations + diff --git a/build/lib/app/api/routers/admin.py b/build/lib/app/api/routers/admin.py new file mode 100644 index 0000000..9844c3e --- /dev/null +++ b/build/lib/app/api/routers/admin.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_admin_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from app.api.schemas import ( + AdminResetPasswordRequest, + AdminUserActiveUpdateRequest, + AdminUserCreateRequest, + AdminUserEnvelope, + AdminUsersListEnvelope, + MessageEnvelope, +) +from app.auth.deps import get_auth_service +from app.auth.service import AuthService +from app.db.models import User +from app.db.session import get_db_session +from app.services.domains.users import application as user_service + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/admin/users", tags=["api-admin-users"]) + + +def _serialize_user(user: User) -> dict[str, object]: + return { + "id": int(user.id), + "email": user.email, + "is_active": bool(user.is_active), + "is_admin": bool(user.is_admin), + "created_at": user.created_at, + "updated_at": user.updated_at, + } + + +@router.get( + "", + response_model=AdminUsersListEnvelope, +) +async def list_users( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + users = await user_service.list_users(db_session) + logger.info( + "api.admin.users_listed", + extra={ + "event": "api.admin.users_listed", + "admin_user_id": int(admin_user.id), + "user_count": len(users), + }, + ) + return success_payload( + request, + data={ + "users": [_serialize_user(user) for user in users], + }, + ) + + +@router.post( + "", + response_model=AdminUserEnvelope, + status_code=201, +) +async def create_user( + payload: AdminUserCreateRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + auth_service: AuthService = Depends(get_auth_service), + admin_user: User = Depends(get_api_admin_user), +): + try: + validated_email = user_service.validate_email(payload.email) + validated_password = user_service.validate_password(payload.password) + created_user = await user_service.create_user( + db_session, + email=validated_email, + password_hash=auth_service.hash_password(validated_password), + is_admin=bool(payload.is_admin), + ) + except user_service.UserServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_user_input", + message=str(exc), + ) from exc + + logger.info( + "api.admin.user_created", + extra={ + "event": "api.admin.user_created", + "admin_user_id": int(admin_user.id), + "target_user_id": int(created_user.id), + "target_is_admin": bool(created_user.is_admin), + }, + ) + return success_payload( + request, + data=_serialize_user(created_user), + ) + + +@router.patch( + "/{user_id}/active", + response_model=AdminUserEnvelope, +) +async def set_user_active( + user_id: int, + payload: AdminUserActiveUpdateRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + target_user = await user_service.get_user_by_id(db_session, user_id) + if target_user is None: + raise ApiException( + status_code=404, + code="user_not_found", + message="User not found.", + ) + if ( + int(target_user.id) == int(admin_user.id) + and bool(target_user.is_active) + and not bool(payload.is_active) + ): + raise ApiException( + status_code=400, + code="cannot_deactivate_self", + message="You cannot deactivate your own account.", + ) + updated_user = await user_service.set_user_active( + db_session, + user=target_user, + is_active=bool(payload.is_active), + ) + logger.info( + "api.admin.user_active_updated", + extra={ + "event": "api.admin.user_active_updated", + "admin_user_id": int(admin_user.id), + "target_user_id": int(updated_user.id), + "is_active": bool(updated_user.is_active), + }, + ) + return success_payload( + request, + data=_serialize_user(updated_user), + ) + + +@router.post( + "/{user_id}/reset-password", + response_model=MessageEnvelope, +) +async def reset_user_password( + user_id: int, + payload: AdminResetPasswordRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + auth_service: AuthService = Depends(get_auth_service), + admin_user: User = Depends(get_api_admin_user), +): + target_user = await user_service.get_user_by_id(db_session, user_id) + if target_user is None: + raise ApiException( + status_code=404, + code="user_not_found", + message="User not found.", + ) + try: + validated_password = user_service.validate_password(payload.new_password) + except user_service.UserServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_password", + message=str(exc), + ) from exc + + await user_service.set_user_password_hash( + db_session, + user=target_user, + password_hash=auth_service.hash_password(validated_password), + ) + logger.info( + "api.admin.user_password_reset", + extra={ + "event": "api.admin.user_password_reset", + "admin_user_id": int(admin_user.id), + "target_user_id": int(target_user.id), + }, + ) + return success_payload( + request, + data={"message": f"Password reset: {target_user.email}"}, + ) diff --git a/build/lib/app/api/routers/admin_dbops.py b/build/lib/app/api/routers/admin_dbops.py new file mode 100644 index 0000000..87fc3b8 --- /dev/null +++ b/build/lib/app/api/routers/admin_dbops.py @@ -0,0 +1,349 @@ +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, Path, Query, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_admin_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from app.api.schemas import ( + AdminDbIntegrityEnvelope, + AdminPdfQueueBulkEnqueueEnvelope, + AdminPdfQueueRequeueEnvelope, + AdminPdfQueueEnvelope, + AdminDbRepairJobsEnvelope, + AdminRepairPublicationLinksEnvelope, + AdminRepairPublicationLinksRequest, +) +from app.db.models import DataRepairJob, User +from app.db.session import get_db_session +from app.services.domains.dbops import ( + collect_integrity_report, + list_repair_jobs, + run_publication_link_repair, +) +from app.services.domains.publications import application as publication_service + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/admin/db", tags=["api-admin-dbops"]) + + +def _serialize_repair_job(job: DataRepairJob) -> dict[str, object]: + return { + "id": int(job.id), + "job_name": job.job_name, + "requested_by": job.requested_by, + "dry_run": bool(job.dry_run), + "status": job.status, + "scope": dict(job.scope or {}), + "summary": dict(job.summary or {}), + "error_text": job.error_text, + "started_at": job.started_at, + "finished_at": job.finished_at, + "created_at": job.created_at, + "updated_at": job.updated_at, + } + + +def _requested_by_value(*, payload: AdminRepairPublicationLinksRequest, admin_user: User) -> str: + from_payload = (payload.requested_by or "").strip() + return from_payload or admin_user.email + + +def _serialize_pdf_queue_item(item) -> dict[str, object]: + return { + "publication_id": item.publication_id, + "title": item.title, + "display_identifier": _serialize_display_identifier(item.display_identifier), + "pdf_url": item.pdf_url, + "status": item.status, + "attempt_count": item.attempt_count, + "last_failure_reason": item.last_failure_reason, + "last_failure_detail": item.last_failure_detail, + "last_source": item.last_source, + "requested_by_user_id": item.requested_by_user_id, + "requested_by_email": item.requested_by_email, + "queued_at": item.queued_at, + "last_attempt_at": item.last_attempt_at, + "resolved_at": item.resolved_at, + "updated_at": item.updated_at, + } + + +def _serialize_display_identifier(value) -> dict[str, object] | None: + if value is None: + return None + return { + "kind": value.kind, + "value": value.value, + "label": value.label, + "url": value.url, + "confidence_score": float(value.confidence_score), + } + + +def _requeue_response_state(*, queued: bool) -> tuple[str, str]: + if queued: + return "queued", "PDF lookup queued." + return "blocked", "PDF lookup is already queued or currently running." + + +def _bulk_enqueue_message(*, queued_count: int, requested_count: int) -> str: + if requested_count == 0: + return "No missing-PDF publications were found." + if queued_count == 0: + return "No publications were queued; all candidates were already in-flight." + return f"Queued {queued_count} publication(s) for PDF lookup." + + +def _resolve_pdf_queue_paging( + *, + page: int, + page_size: int, + limit: int | None, + offset: int | None, +) -> tuple[int, int, int]: + resolved_limit = int(limit) if limit is not None else int(page_size) + resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0) + resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1) + return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0) + + +def _pdf_queue_page_data(*, total_count: int, page: int, page_size: int, offset: int) -> dict[str, object]: + return { + "total_count": int(total_count), + "page": int(page), + "page_size": int(page_size), + "has_prev": int(offset) > 0, + "has_next": int(offset) + int(page_size) < int(total_count), + } + + +@router.get( + "/integrity", + response_model=AdminDbIntegrityEnvelope, +) +async def get_integrity_report( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + report = await collect_integrity_report(db_session) + logger.info( + "api.admin.db.integrity_checked", + extra={ + "event": "api.admin.db.integrity_checked", + "admin_user_id": int(admin_user.id), + "status": report.get("status"), + "failure_count": len(report.get("failures", [])), + "warning_count": len(report.get("warnings", [])), + }, + ) + return success_payload(request, data=report) + + +@router.get( + "/repair-jobs", + response_model=AdminDbRepairJobsEnvelope, +) +async def get_repair_jobs( + request: Request, + limit: int = Query(default=50, ge=1, le=200), + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + jobs = await list_repair_jobs(db_session, limit=limit) + logger.info( + "api.admin.db.repair_jobs_listed", + extra={ + "event": "api.admin.db.repair_jobs_listed", + "admin_user_id": int(admin_user.id), + "limit": int(limit), + "job_count": len(jobs), + }, + ) + return success_payload( + request, + data={"jobs": [_serialize_repair_job(job) for job in jobs]}, + ) + + +@router.get( + "/pdf-queue", + response_model=AdminPdfQueueEnvelope, +) +async def get_pdf_queue( + request: Request, + page: int = Query(default=1, ge=1), + page_size: int = Query(default=100, ge=1, le=500), + limit: int | None = Query(default=None, ge=1, le=500), + offset: int | None = Query(default=None, ge=0), + status: str | None = Query(default=None), + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + normalized_status = (status or "").strip().lower() or None + resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging( + page=page, + page_size=page_size, + limit=limit, + offset=offset, + ) + queue_page = await publication_service.list_pdf_queue_page( + db_session, + limit=resolved_limit, + offset=resolved_offset, + status=normalized_status, + ) + logger.info( + "api.admin.db.pdf_queue_listed", + extra={ + "event": "api.admin.db.pdf_queue_listed", + "admin_user_id": int(admin_user.id), + "page": int(resolved_page), + "page_size": int(resolved_limit), + "offset": int(resolved_offset), + "status": normalized_status, + "item_count": len(queue_page.items), + "total_count": int(queue_page.total_count), + }, + ) + return success_payload( + request, + data={ + "items": [_serialize_pdf_queue_item(item) for item in queue_page.items], + **_pdf_queue_page_data( + total_count=queue_page.total_count, + page=resolved_page, + page_size=resolved_limit, + offset=resolved_offset, + ), + }, + ) + + +@router.post( + "/pdf-queue/{publication_id}/requeue", + response_model=AdminPdfQueueRequeueEnvelope, +) +async def requeue_pdf_lookup( + request: Request, + publication_id: int = Path(ge=1), + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + result = await publication_service.enqueue_retry_pdf_job_for_publication_id( + db_session, + user_id=int(admin_user.id), + request_email=admin_user.email, + publication_id=publication_id, + ) + if not result.publication_exists: + raise ApiException( + status_code=404, + code="publication_not_found", + message="Publication not found.", + ) + status, message = _requeue_response_state(queued=result.queued) + logger.info( + "api.admin.db.pdf_queue_requeue_requested", + extra={ + "event": "api.admin.db.pdf_queue_requeue_requested", + "admin_user_id": int(admin_user.id), + "publication_id": int(publication_id), + "queued": bool(result.queued), + }, + ) + return success_payload( + request, + data={ + "publication_id": int(publication_id), + "queued": bool(result.queued), + "status": status, + "message": message, + }, + ) + + +@router.post( + "/pdf-queue/requeue-all", + response_model=AdminPdfQueueBulkEnqueueEnvelope, +) +async def requeue_all_missing_pdfs( + request: Request, + limit: int = Query(default=1000, ge=1, le=5000), + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + result = await publication_service.enqueue_all_missing_pdf_jobs( + db_session, + user_id=int(admin_user.id), + request_email=admin_user.email, + limit=limit, + ) + logger.info( + "api.admin.db.pdf_queue_requeue_all", + extra={ + "event": "api.admin.db.pdf_queue_requeue_all", + "admin_user_id": int(admin_user.id), + "limit": int(limit), + "requested_count": int(result.requested_count), + "queued_count": int(result.queued_count), + }, + ) + return success_payload( + request, + data={ + "requested_count": int(result.requested_count), + "queued_count": int(result.queued_count), + "message": _bulk_enqueue_message( + queued_count=int(result.queued_count), + requested_count=int(result.requested_count), + ), + }, + ) + + +@router.post( + "/repairs/publication-links", + response_model=AdminRepairPublicationLinksEnvelope, +) +async def trigger_publication_link_repair( + payload: AdminRepairPublicationLinksRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + try: + result = await run_publication_link_repair( + db_session, + scope_mode=payload.scope_mode, + user_id=payload.user_id, + scholar_profile_ids=payload.scholar_profile_ids, + dry_run=bool(payload.dry_run), + gc_orphan_publications=bool(payload.gc_orphan_publications), + requested_by=_requested_by_value(payload=payload, admin_user=admin_user), + ) + except ValueError as exc: + raise ApiException( + status_code=400, + code="invalid_repair_scope", + message=str(exc), + ) from exc + logger.info( + "api.admin.db.publication_link_repair_triggered", + extra={ + "event": "api.admin.db.publication_link_repair_triggered", + "admin_user_id": int(admin_user.id), + "scope_mode": payload.scope_mode, + "target_user_id": int(payload.user_id) if payload.user_id is not None else None, + "dry_run": bool(payload.dry_run), + "gc_orphan_publications": bool(payload.gc_orphan_publications), + "job_id": int(result["job_id"]), + "status": result["status"], + }, + ) + return success_payload(request, data=result) diff --git a/build/lib/app/api/routers/auth.py b/build/lib/app/api/routers/auth.py new file mode 100644 index 0000000..8f8da17 --- /dev/null +++ b/build/lib/app/api/routers/auth.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.errors import ApiException +from app.api.deps import get_api_current_user +from app.api.responses import success_payload +from app.api.schemas import ( + AuthMeEnvelope, + ChangePasswordRequest, + CsrfBootstrapEnvelope, + LoginEnvelope, + LoginRequest, + MessageEnvelope, +) +from app.auth.deps import get_auth_service, get_login_rate_limiter +from app.auth.rate_limit import SlidingWindowRateLimiter +from app.auth import runtime as auth_runtime +from app.auth.service import AuthService +from app.auth.session import set_session_user +from app.db.models import User +from app.db.session import get_db_session +from app.security.csrf import ensure_csrf_token +from app.services.domains.users import application as user_service + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/auth", tags=["api-auth"]) + + +def _login_limiter_key_and_email(request: Request, payload: LoginRequest) -> tuple[str, str]: + return auth_runtime.login_rate_limit_key(request, payload.email), payload.email.strip().lower() + + +def _raise_rate_limited(normalized_email: str, retry_after_seconds: int) -> None: + logger.warning( + "api.auth.login_rate_limited", + extra={ + "event": "api.auth.login_rate_limited", + "email": normalized_email, + "retry_after_seconds": retry_after_seconds, + }, + ) + raise ApiException( + status_code=429, + code="rate_limited", + message="Too many login attempts. Please try again later.", + details={"retry_after_seconds": retry_after_seconds}, + ) + + +def _serialize_user_payload(user: User) -> dict[str, object]: + return { + "id": int(user.id), + "email": user.email, + "is_admin": bool(user.is_admin), + "is_active": bool(user.is_active), + } + + +def _raise_invalid_credentials(*, normalized_email: str) -> None: + logger.info( + "api.auth.login_failed", + extra={"event": "api.auth.login_failed", "email": normalized_email}, + ) + raise ApiException( + status_code=401, + code="invalid_credentials", + message="Invalid email or password.", + ) + + +@router.post( + "/login", + response_model=LoginEnvelope, +) +async def login( + payload: LoginRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + auth_service: AuthService = Depends(get_auth_service), + rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter), +): + limiter_key, normalized_email = _login_limiter_key_and_email(request, payload) + decision = rate_limiter.check(limiter_key) + if not decision.allowed: + _raise_rate_limited(normalized_email, int(decision.retry_after_seconds)) + + user = await auth_service.authenticate_user( + db_session, + email=payload.email, + password=payload.password, + ) + if user is None: + rate_limiter.record_failure(limiter_key) + _raise_invalid_credentials(normalized_email=normalized_email) + + rate_limiter.reset(limiter_key) + set_session_user( + request, + user_id=user.id, + email=user.email, + is_admin=user.is_admin, + ) + logger.info( + "api.auth.login_succeeded", + extra={ + "event": "api.auth.login_succeeded", + "user_id": user.id, + "is_admin": user.is_admin, + }, + ) + return success_payload( + request, + data={ + "authenticated": True, + "csrf_token": ensure_csrf_token(request), + "user": _serialize_user_payload(user), + }, + ) + + +@router.get( + "/me", + response_model=AuthMeEnvelope, +) +async def get_current_session( + request: Request, + current_user: User = Depends(get_api_current_user), +): + return success_payload( + request, + data={ + "authenticated": True, + "csrf_token": ensure_csrf_token(request), + "user": _serialize_user_payload(current_user), + }, + ) + + +@router.get( + "/csrf", + response_model=CsrfBootstrapEnvelope, +) +async def get_csrf_bootstrap( + request: Request, + db_session: AsyncSession = Depends(get_db_session), +): + current_user = await auth_runtime.get_authenticated_user(request, db_session) + return success_payload( + request, + data={ + "csrf_token": ensure_csrf_token(request), + "authenticated": current_user is not None, + }, + ) + + +@router.post( + "/change-password", + response_model=MessageEnvelope, +) +async def change_password( + payload: ChangePasswordRequest, + request: Request, + current_user: User = Depends(get_api_current_user), + db_session: AsyncSession = Depends(get_db_session), + auth_service: AuthService = Depends(get_auth_service), +): + if not auth_service.verify_password( + password_hash=current_user.password_hash, + password=payload.current_password, + ): + raise ApiException( + status_code=400, + code="invalid_current_password", + message="Current password is incorrect.", + ) + if payload.new_password != payload.confirm_password: + raise ApiException( + status_code=400, + code="password_confirmation_mismatch", + message="New password and confirmation do not match.", + ) + try: + validated_password = user_service.validate_password(payload.new_password) + except user_service.UserServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_password", + message=str(exc), + ) from exc + + await user_service.set_user_password_hash( + db_session, + user=current_user, + password_hash=auth_service.hash_password(validated_password), + ) + logger.info( + "api.auth.password_changed", + extra={ + "event": "api.auth.password_changed", + "user_id": int(current_user.id), + }, + ) + return success_payload( + request, + data={"message": "Password updated successfully."}, + ) + + +@router.post( + "/logout", + response_model=MessageEnvelope, +) +async def logout( + request: Request, + db_session: AsyncSession = Depends(get_db_session), +): + current_user = await auth_runtime.get_authenticated_user(request, db_session) + auth_runtime.invalidate_session(request) + logger.info( + "api.auth.logout", + extra={ + "event": "api.auth.logout", + "user_id": int(current_user.id) if current_user is not None else None, + }, + ) + return success_payload( + request, + data={ + "message": "Logged out.", + }, + ) diff --git a/build/lib/app/api/routers/publications.py b/build/lib/app/api/routers/publications.py new file mode 100644 index 0000000..2a96b8b --- /dev/null +++ b/build/lib/app/api/routers/publications.py @@ -0,0 +1,528 @@ +from __future__ import annotations + +import logging +from typing import Literal + +from fastapi import APIRouter, Depends, Path, Query, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_current_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from app.api.schemas import ( + MarkAllReadEnvelope, + MarkSelectedReadEnvelope, + MarkSelectedReadRequest, + PublicationsListEnvelope, + RetryPublicationPdfEnvelope, + RetryPublicationPdfRequest, + TogglePublicationFavoriteEnvelope, + TogglePublicationFavoriteRequest, +) +from app.db.models import User +from app.db.session import get_db_session +from app.services.domains.publication_identifiers import application as identifier_service +from app.services.domains.publications import application as publication_service +from app.services.domains.scholars import application as scholar_service +from app.settings import settings + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/publications", tags=["api-publications"]) + + +async def _require_selected_profile( + db_session: AsyncSession, + *, + user_id: int, + selected_scholar_id: int | None, +) -> None: + if selected_scholar_id is None: + return + selected_profile = await scholar_service.get_user_scholar_by_id( + db_session, + user_id=user_id, + scholar_profile_id=selected_scholar_id, + ) + if selected_profile is None: + raise ApiException( + status_code=404, + code="scholar_not_found", + message="Scholar filter not found.", + ) + + +def _serialize_publication_item(item) -> dict[str, object]: + return { + "publication_id": item.publication_id, + "scholar_profile_id": item.scholar_profile_id, + "scholar_label": item.scholar_label, + "title": item.title, + "year": item.year, + "citation_count": item.citation_count, + "venue_text": item.venue_text, + "pub_url": item.pub_url, + "display_identifier": _serialize_display_identifier(item.display_identifier), + "pdf_url": item.pdf_url, + "pdf_status": item.pdf_status, + "pdf_attempt_count": item.pdf_attempt_count, + "pdf_failure_reason": item.pdf_failure_reason, + "pdf_failure_detail": item.pdf_failure_detail, + "is_read": item.is_read, + "is_favorite": item.is_favorite, + "first_seen_at": item.first_seen_at, + "is_new_in_latest_run": item.is_new_in_latest_run, + } + + +def _serialize_display_identifier(value) -> dict[str, object] | None: + if value is None: + return None + return { + "kind": value.kind, + "value": value.value, + "label": value.label, + "url": value.url, + "confidence_score": float(value.confidence_score), + } + + +async def _publication_counts( + db_session: AsyncSession, + *, + user_id: int, + selected_scholar_id: int | None, + favorite_only: bool, +) -> tuple[int, int, int, int]: + unread_count = await publication_service.count_unread_for_user( + db_session, + user_id=user_id, + scholar_profile_id=selected_scholar_id, + favorite_only=favorite_only, + ) + favorites_count = await publication_service.count_favorite_for_user( + db_session, + user_id=user_id, + scholar_profile_id=selected_scholar_id, + ) + latest_count = await publication_service.count_latest_for_user( + db_session, + user_id=user_id, + scholar_profile_id=selected_scholar_id, + favorite_only=favorite_only, + ) + total_count = await publication_service.count_for_user( + db_session, + user_id=user_id, + mode=publication_service.MODE_ALL, + scholar_profile_id=selected_scholar_id, + favorite_only=favorite_only, + ) + return unread_count, favorites_count, latest_count, total_count + + +async def _list_publications_for_request( + db_session: AsyncSession, + *, + current_user: User, + mode: Literal["all", "unread", "latest", "new"] | None, + favorite_only: bool, + scholar_profile_id: int | None, + limit: int, + offset: int, +) -> tuple[str, int | None, list]: + resolved_mode = publication_service.resolve_publication_view_mode(mode) + selected_scholar_id = scholar_profile_id + await _require_selected_profile( + db_session, + user_id=current_user.id, + selected_scholar_id=selected_scholar_id, + ) + publications = await publication_service.list_for_user( + db_session, + user_id=current_user.id, + mode=resolved_mode, + scholar_profile_id=selected_scholar_id, + favorite_only=favorite_only, + limit=limit, + offset=offset, + ) + await publication_service.schedule_missing_pdf_enrichment_for_user( + db_session, + user_id=current_user.id, + request_email=current_user.email, + items=publications, + max_items=settings.unpaywall_max_items_per_request, + ) + hydrated = await publication_service.hydrate_pdf_enrichment_state( + db_session, + items=publications, + ) + return resolved_mode, selected_scholar_id, hydrated + + +def _resolve_publications_paging( + *, + page: int, + page_size: int, + limit: int | None, + offset: int | None, +) -> tuple[int, int, int]: + resolved_limit = int(limit) if limit is not None else int(page_size) + resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0) + resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1) + return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0) + + +def _publications_list_data( + *, + mode: str, + favorite_only: bool, + selected_scholar_id: int | None, + unread_count: int, + favorites_count: int, + latest_count: int, + total_count: int, + publications: list, + page: int, + page_size: int, + offset: int, +) -> dict[str, object]: + return { + "mode": mode, + "favorite_only": favorite_only, + "selected_scholar_profile_id": selected_scholar_id, + "unread_count": unread_count, + "favorites_count": favorites_count, + "latest_count": latest_count, + "new_count": latest_count, + "total_count": total_count, + "page": int(page), + "page_size": int(page_size), + "has_prev": int(offset) > 0, + "has_next": int(offset) + int(page_size) < int(total_count), + "publications": [_serialize_publication_item(item) for item in publications], + } + + +def _retry_pdf_message(*, queued: bool, resolved_pdf: bool, pdf_status: str) -> str: + if resolved_pdf: + return "Open-access PDF link already resolved." + if queued: + return "PDF lookup queued." + if pdf_status in {"queued", "running"}: + return "PDF lookup is already queued." + return "No open-access PDF link found." + + +def _favorite_message(*, is_favorite: bool) -> str: + if is_favorite: + return "Publication marked as favorite." + return "Publication removed from favorites." + + +async def _retry_publication_state( + db_session: AsyncSession, + *, + current_user: User, + scholar_profile_id: int, + publication_id: int, +): + publication = await publication_service.retry_pdf_for_user( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + publication_id=publication_id, + ) + if publication is None: + raise ApiException( + status_code=404, + code="publication_not_found", + message="Publication not found.", + ) + queued = False + if not publication.pdf_url: + queued = await publication_service.schedule_retry_pdf_enrichment_for_row( + db_session, + user_id=current_user.id, + request_email=current_user.email, + item=publication, + ) + hydrated = await publication_service.hydrate_pdf_enrichment_state( + db_session, + items=[publication], + ) + current = hydrated[0] if hydrated else publication + return current, queued + + +async def _favorite_publication_state( + db_session: AsyncSession, + *, + current_user: User, + scholar_profile_id: int, + publication_id: int, + is_favorite: bool, +): + updated_count = await publication_service.set_publication_favorite_for_user( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + publication_id=publication_id, + is_favorite=is_favorite, + ) + if updated_count <= 0: + raise ApiException( + status_code=404, + code="publication_not_found", + message="Publication not found.", + ) + publication = await publication_service.get_publication_item_for_user( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + publication_id=publication_id, + ) + if publication is None: + raise ApiException( + status_code=404, + code="publication_not_found", + message="Publication not found.", + ) + hydrated = await publication_service.hydrate_pdf_enrichment_state( + db_session, + items=[publication], + ) + current = hydrated[0] if hydrated else publication + identifiers = await identifier_service.overlay_publication_items_with_display_identifiers( + db_session, + items=[current], + ) + return identifiers[0] if identifiers else current + + +def _log_retry_pdf_result( + *, + current_user: User, + scholar_profile_id: int, + publication_id: int, + queued: bool, + resolved_pdf: bool, + pdf_status: str, + doi: str | None, +) -> None: + logger.info( + "api.publications.retry_pdf", + extra={ + "event": "api.publications.retry_pdf", + "user_id": current_user.id, + "scholar_profile_id": scholar_profile_id, + "publication_id": publication_id, + "queued": queued, + "resolved_pdf": resolved_pdf, + "pdf_status": pdf_status, + }, + ) + + +@router.get( + "", + response_model=PublicationsListEnvelope, +) +async def list_publications( + request: Request, + mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None), + favorite_only: bool = Query(default=False), + scholar_profile_id: int | None = Query(default=None, ge=1), + page: int = Query(default=1, ge=1), + page_size: int = Query(default=100, ge=1, le=500), + limit: int | None = Query(default=None, ge=1, le=1000), + offset: int | None = Query(default=None, ge=0), + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + resolved_page, resolved_limit, resolved_offset = _resolve_publications_paging( + page=page, + page_size=page_size, + limit=limit, + offset=offset, + ) + resolved_mode, selected_scholar_id, publications = await _list_publications_for_request( + db_session, + current_user=current_user, + mode=mode, + favorite_only=favorite_only, + scholar_profile_id=scholar_profile_id, + limit=resolved_limit, + offset=resolved_offset, + ) + unread_count, favorites_count, latest_count, total_count = await _publication_counts( + db_session, + user_id=current_user.id, + selected_scholar_id=selected_scholar_id, + favorite_only=favorite_only, + ) + data = _publications_list_data( + mode=resolved_mode, + favorite_only=favorite_only, + selected_scholar_id=selected_scholar_id, + unread_count=unread_count, + favorites_count=favorites_count, + latest_count=latest_count, + total_count=total_count, + publications=publications, + page=resolved_page, + page_size=resolved_limit, + offset=resolved_offset, + ) + return success_payload(request, data=data) + + +@router.post( + "/mark-all-read", + response_model=MarkAllReadEnvelope, +) +async def mark_all_publications_read( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + updated_count = await publication_service.mark_all_unread_as_read_for_user( + db_session, + user_id=current_user.id, + ) + logger.info( + "api.publications.mark_all_read", + extra={ + "event": "api.publications.mark_all_read", + "user_id": current_user.id, + "updated_count": updated_count, + }, + ) + return success_payload( + request, + data={ + "message": "Marked all unread publications as read.", + "updated_count": updated_count, + }, + ) + + +@router.post( + "/mark-read", + response_model=MarkSelectedReadEnvelope, +) +async def mark_selected_publications_read( + payload: MarkSelectedReadRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + selection_pairs = sorted( + { + (int(item.scholar_profile_id), int(item.publication_id)) + for item in payload.selections + } + ) + updated_count = await publication_service.mark_selected_as_read_for_user( + db_session, + user_id=current_user.id, + selections=selection_pairs, + ) + logger.info( + "api.publications.mark_selected_read", + extra={ + "event": "api.publications.mark_selected_read", + "user_id": current_user.id, + "requested_count": len(selection_pairs), + "updated_count": updated_count, + }, + ) + return success_payload( + request, + data={ + "message": "Marked selected publications as read.", + "requested_count": len(selection_pairs), + "updated_count": updated_count, + }, + ) + + +@router.post( + "/{publication_id}/retry-pdf", + response_model=RetryPublicationPdfEnvelope, +) +async def retry_publication_pdf( + payload: RetryPublicationPdfRequest, + request: Request, + publication_id: int = Path(ge=1), + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + current, queued = await _retry_publication_state( + db_session, + current_user=current_user, + scholar_profile_id=payload.scholar_profile_id, + publication_id=publication_id, + ) + resolved_pdf = bool(current.pdf_url) + message = _retry_pdf_message( + queued=queued, + resolved_pdf=resolved_pdf, + pdf_status=current.pdf_status, + ) + _log_retry_pdf_result( + current_user=current_user, + scholar_profile_id=payload.scholar_profile_id, + publication_id=publication_id, + queued=queued, + resolved_pdf=resolved_pdf, + pdf_status=current.pdf_status, + doi=None, + ) + return success_payload( + request, + data={ + "message": message, + "queued": queued, + "resolved_pdf": resolved_pdf, + "publication": _serialize_publication_item(current), + }, + ) + + +@router.post( + "/{publication_id}/favorite", + response_model=TogglePublicationFavoriteEnvelope, +) +async def toggle_publication_favorite( + payload: TogglePublicationFavoriteRequest, + request: Request, + publication_id: int = Path(ge=1), + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + current = await _favorite_publication_state( + db_session, + current_user=current_user, + scholar_profile_id=payload.scholar_profile_id, + publication_id=publication_id, + is_favorite=payload.is_favorite, + ) + logger.info( + "api.publications.favorite", + extra={ + "event": "api.publications.favorite", + "user_id": current_user.id, + "scholar_profile_id": payload.scholar_profile_id, + "publication_id": publication_id, + "is_favorite": bool(payload.is_favorite), + }, + ) + return success_payload( + request, + data={ + "message": _favorite_message(is_favorite=bool(payload.is_favorite)), + "publication": _serialize_publication_item(current), + }, + ) diff --git a/build/lib/app/api/routers/runs.py b/build/lib/app/api/routers/runs.py new file mode 100644 index 0000000..4e85af4 --- /dev/null +++ b/build/lib/app/api/routers/runs.py @@ -0,0 +1,845 @@ +from __future__ import annotations + +import logging +import re +from typing import Any + +from fastapi import APIRouter, Depends, Query, Request +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_current_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from fastapi.responses import StreamingResponse +from app.api.schemas import ( + ManualRunEnvelope, + QueueClearEnvelope, + QueueItemEnvelope, + QueueListEnvelope, + RunDetailEnvelope, + RunsListEnvelope, +) +from app.db.models import RunStatus, RunTriggerType, User +from app.db.session import get_db_session +from app.services.domains.ingestion import application as ingestion_service +from app.services.domains.ingestion import safety as run_safety_service +from app.services.domains.runs import application as run_service +from app.services.domains.runs.events import event_generator +from app.services.domains.settings import application as user_settings_service +from app.settings import settings +from app.api.runtime_deps import get_ingestion_service + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/runs", tags=["api-runs"]) + +IDEMPOTENCY_HEADER = "Idempotency-Key" +IDEMPOTENCY_MAX_LENGTH = 128 +IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$") + + +def _int_value(value: Any, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _str_value(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text if text else None + + +def _bool_value(value: Any, default: bool = False) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"1", "true", "yes", "on"}: + return True + if lowered in {"0", "false", "no", "off"}: + return False + return default + + +def _normalize_idempotency_key(raw_value: str | None) -> str | None: + if raw_value is None: + return None + candidate = raw_value.strip() + if not candidate: + return None + if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate): + raise ApiException( + status_code=400, + code="invalid_idempotency_key", + message=( + "Invalid Idempotency-Key. Use 8-128 characters from: " + "A-Z a-z 0-9 . _ : -" + ), + ) + return candidate + + +def _serialize_run(run) -> dict[str, Any]: + summary = run_service.extract_run_summary(run.error_log) + return { + "id": int(run.id), + "trigger_type": run.trigger_type.value, + "status": run.status.value, + "start_dt": run.start_dt, + "end_dt": run.end_dt, + "scholar_count": int(run.scholar_count or 0), + "new_publication_count": int(run.new_pub_count or 0), + "failed_count": int(summary["failed_count"]), + "partial_count": int(summary["partial_count"]), + } + + +def _serialize_queue_item(item) -> dict[str, Any]: + return { + "id": int(item.id), + "scholar_profile_id": int(item.scholar_profile_id), + "scholar_label": item.scholar_label, + "status": item.status, + "reason": item.reason, + "dropped_reason": item.dropped_reason, + "attempt_count": int(item.attempt_count), + "resume_cstart": int(item.resume_cstart), + "next_attempt_dt": item.next_attempt_dt, + "updated_at": item.updated_at, + "last_error": item.last_error, + "last_run_id": item.last_run_id, + } + + +def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + return [] + normalized: list[dict[str, Any]] = [] + for item in value: + if not isinstance(item, dict): + continue + normalized.append( + { + "attempt": _int_value(item.get("attempt"), 0), + "cstart": _int_value(item.get("cstart"), 0), + "state": _str_value(item.get("state")), + "state_reason": _str_value(item.get("state_reason")), + "status_code": ( + _int_value(item.get("status_code")) + if item.get("status_code") is not None + else None + ), + "fetch_error": _str_value(item.get("fetch_error")), + } + ) + return normalized + + +def _normalize_page_logs(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + return [] + normalized: list[dict[str, Any]] = [] + for item in value: + if not isinstance(item, dict): + continue + warning_codes = item.get("warning_codes") + normalized.append( + { + "page": _int_value(item.get("page"), 0), + "cstart": _int_value(item.get("cstart"), 0), + "state": _str_value(item.get("state")) or "unknown", + "state_reason": _str_value(item.get("state_reason")), + "status_code": ( + _int_value(item.get("status_code")) + if item.get("status_code") is not None + else None + ), + "publication_count": _int_value(item.get("publication_count"), 0), + "attempt_count": _int_value(item.get("attempt_count"), 0), + "has_show_more_button": _bool_value(item.get("has_show_more_button"), False), + "articles_range": _str_value(item.get("articles_range")), + "warning_codes": [ + str(code) + for code in (warning_codes if isinstance(warning_codes, list) else []) + ], + } + ) + return normalized + + +def _normalize_debug(value: Any) -> dict[str, Any] | None: + if not isinstance(value, dict): + return None + marker_counts = value.get("marker_counts_nonzero") + warning_codes = value.get("warning_codes") + return { + "status_code": ( + _int_value(value.get("status_code")) + if value.get("status_code") is not None + else None + ), + "final_url": _str_value(value.get("final_url")), + "fetch_error": _str_value(value.get("fetch_error")), + "body_sha256": _str_value(value.get("body_sha256")), + "body_length": ( + _int_value(value.get("body_length")) + if value.get("body_length") is not None + else None + ), + "has_show_more_button": ( + _bool_value(value.get("has_show_more_button"), False) + if value.get("has_show_more_button") is not None + else None + ), + "articles_range": _str_value(value.get("articles_range")), + "state_reason": _str_value(value.get("state_reason")), + "warning_codes": [ + str(code) + for code in (warning_codes if isinstance(warning_codes, list) else []) + ], + "marker_counts_nonzero": { + str(key): _int_value(count, 0) + for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else []) + }, + "page_logs": _normalize_page_logs(value.get("page_logs")), + "attempt_log": _normalize_attempt_log(value.get("attempt_log")), + } + + +def _normalize_scholar_result(value: Any) -> dict[str, Any]: + if not isinstance(value, dict): + return { + "scholar_profile_id": 0, + "scholar_id": "unknown", + "state": "unknown", + "state_reason": None, + "outcome": "failed", + "attempt_count": 0, + "publication_count": 0, + "start_cstart": 0, + "continuation_cstart": None, + "continuation_enqueued": False, + "continuation_cleared": False, + "warnings": [], + "error": None, + "debug": None, + } + warnings = value.get("warnings") + return { + "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0), + "scholar_id": _str_value(value.get("scholar_id")) or "unknown", + "state": _str_value(value.get("state")) or "unknown", + "state_reason": _str_value(value.get("state_reason")), + "outcome": _str_value(value.get("outcome")) or "failed", + "attempt_count": _int_value(value.get("attempt_count"), 0), + "publication_count": _int_value(value.get("publication_count"), 0), + "start_cstart": _int_value(value.get("start_cstart"), 0), + "continuation_cstart": ( + _int_value(value.get("continuation_cstart")) + if value.get("continuation_cstart") is not None + else None + ), + "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False), + "continuation_cleared": _bool_value(value.get("continuation_cleared"), False), + "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])], + "error": _str_value(value.get("error")), + "debug": _normalize_debug(value.get("debug")), + } + + +def _manual_run_payload_from_run( + run, + *, + idempotency_key: str | None, + reused_existing_run: bool, + safety_state: dict[str, Any], +) -> dict[str, Any]: + summary = run_service.extract_run_summary(run.error_log) + return { + "run_id": int(run.id), + "status": run.status.value, + "scholar_count": int(run.scholar_count or 0), + "succeeded_count": int(summary["succeeded_count"]), + "failed_count": int(summary["failed_count"]), + "partial_count": int(summary["partial_count"]), + "new_publication_count": int(run.new_pub_count or 0), + "reused_existing_run": reused_existing_run, + "idempotency_key": idempotency_key, + "safety_state": safety_state, + } + + +async def _load_safety_state( + db_session: AsyncSession, + *, + user_id: int, +) -> dict[str, Any]: + user_settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=user_id, + ) + previous_safety_state = run_safety_service.get_safety_event_context(user_settings) + if run_safety_service.clear_expired_cooldown(user_settings): + await db_session.commit() + await db_session.refresh(user_settings) + logger.info( + "api.runs.safety_cooldown_cleared", + extra={ + "event": "api.runs.safety_cooldown_cleared", + "user_id": user_id, + "reason": previous_safety_state.get("cooldown_reason"), + "cooldown_until": previous_safety_state.get("cooldown_until"), + "metric_name": "api_runs_safety_cooldown_cleared_total", + "metric_value": 1, + }, + ) + return run_safety_service.get_safety_state_payload(user_settings) + + +def _raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None: + logger.warning( + "api.runs.manual_blocked_policy", + extra={ + "event": "api.runs.manual_blocked_policy", + "user_id": user_id, + "policy": {"manual_run_allowed": False}, + "safety_state": safety_state, + "metric_name": "api_runs_manual_blocked_policy_total", + "metric_value": 1, + }, + ) + raise ApiException( + status_code=403, + code="manual_runs_disabled", + message="Manual checks are disabled by server policy.", + details={ + "policy": {"manual_run_allowed": False}, + "safety_state": safety_state, + }, + ) + + +async def _reused_manual_run_payload( + db_session: AsyncSession, + *, + request: Request, + user_id: int, + idempotency_key: str | None, + safety_state: dict[str, Any], +) -> dict[str, Any] | None: + if idempotency_key is None: + return None + previous_run = await run_service.get_manual_run_by_idempotency_key( + db_session, + user_id=user_id, + idempotency_key=idempotency_key, + ) + if previous_run is None: + return None + if previous_run.status == RunStatus.RUNNING: + raise ApiException( + status_code=409, + code="run_in_progress", + message="A run with this idempotency key is still in progress.", + details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key}, + ) + return success_payload( + request, + data=_manual_run_payload_from_run( + previous_run, + idempotency_key=idempotency_key, + reused_existing_run=True, + safety_state=safety_state, + ), + ) + + +async def _run_ingestion_for_manual( + db_session: AsyncSession, + *, + ingest_service: ingestion_service.ScholarIngestionService, + user_id: int, + idempotency_key: str | None, +): + user_settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=user_id, + ) + return await ingest_service.run_for_user( + db_session, + user_id=user_id, + trigger_type=RunTriggerType.MANUAL, + request_delay_seconds=user_settings.request_delay_seconds, + network_error_retries=settings.ingestion_network_error_retries, + retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, + max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, + page_size=settings.ingestion_page_size, + auto_queue_continuations=settings.ingestion_continuation_queue_enabled, + queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds, + idempotency_key=idempotency_key, + alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, + alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, + alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, + ) + + +async def _recover_integrity_error( + db_session: AsyncSession, + *, + request: Request, + user_id: int, + idempotency_key: str | None, + original_exc: IntegrityError, +) -> dict[str, Any]: + if idempotency_key is None: + logger.exception( + "api.runs.manual_integrity_error", + extra={"event": "api.runs.manual_integrity_error", "user_id": user_id}, + ) + raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc + existing_run = await run_service.get_manual_run_by_idempotency_key( + db_session, + user_id=user_id, + idempotency_key=idempotency_key, + ) + if existing_run is None: + logger.exception( + "api.runs.manual_integrity_error", + extra={"event": "api.runs.manual_integrity_error", "user_id": user_id}, + ) + raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc + if existing_run.status == RunStatus.RUNNING: + raise ApiException( + status_code=409, + code="run_in_progress", + message="A run with this idempotency key is still in progress.", + details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key}, + ) from original_exc + return success_payload( + request, + data=_manual_run_payload_from_run( + existing_run, + idempotency_key=idempotency_key, + reused_existing_run=True, + safety_state=await _load_safety_state(db_session, user_id=user_id), + ), + ) + + +async def _execute_manual_run( + db_session: AsyncSession, + *, + request: Request, + ingest_service: ingestion_service.ScholarIngestionService, + user_id: int, + idempotency_key: str | None, +): + try: + return await _run_ingestion_for_manual( + db_session, + ingest_service=ingest_service, + user_id=user_id, + idempotency_key=idempotency_key, + ) + except ingestion_service.RunAlreadyInProgressError as exc: + await db_session.rollback() + raise ApiException( + status_code=409, + code="run_in_progress", + message="A run is already in progress for this account.", + ) from exc + except ingestion_service.RunBlockedBySafetyPolicyError as exc: + await db_session.rollback() + _raise_manual_blocked_safety(exc=exc, user_id=user_id) + except IntegrityError as exc: + await db_session.rollback() + return await _recover_integrity_error( + db_session, + request=request, + user_id=user_id, + idempotency_key=idempotency_key, + original_exc=exc, + ) + except Exception as exc: + await db_session.rollback() + _raise_manual_failed(exc=exc, user_id=user_id) + + +def _raise_manual_blocked_safety(*, exc, user_id: int) -> None: + logger.info( + "api.runs.manual_blocked_safety", + extra={ + "event": "api.runs.manual_blocked_safety", + "user_id": user_id, + "reason": exc.safety_state.get("cooldown_reason"), + "cooldown_until": exc.safety_state.get("cooldown_until"), + "cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"), + "metric_name": "api_runs_manual_blocked_safety_total", + "metric_value": 1, + }, + ) + raise ApiException( + status_code=429, + code=exc.code, + message=exc.message, + details={"safety_state": exc.safety_state}, + ) from exc + + +def _raise_manual_failed(*, exc: Exception, user_id: int) -> None: + logger.exception( + "api.runs.manual_failed", + extra={"event": "api.runs.manual_failed", "user_id": user_id}, + ) + raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc + + +def _manual_run_success_payload( + *, + run_summary, + idempotency_key: str | None, + safety_state: dict[str, Any], +) -> dict[str, Any]: + return { + "run_id": run_summary.crawl_run_id, + "status": run_summary.status.value, + "scholar_count": run_summary.scholar_count, + "succeeded_count": run_summary.succeeded_count, + "failed_count": run_summary.failed_count, + "partial_count": run_summary.partial_count, + "new_publication_count": run_summary.new_publication_count, + "reused_existing_run": False, + "idempotency_key": idempotency_key, + "safety_state": safety_state, + } + + +@router.get( + "", + response_model=RunsListEnvelope, +) +async def list_runs( + request: Request, + failed_only: bool = Query(default=False), + limit: int = Query(default=100, ge=1, le=500), + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + runs = await run_service.list_runs_for_user( + db_session, + user_id=current_user.id, + limit=limit, + failed_only=failed_only, + ) + safety_state = await _load_safety_state( + db_session, + user_id=current_user.id, + ) + return success_payload( + request, + data={ + "runs": [_serialize_run(run) for run in runs], + "safety_state": safety_state, + }, + ) + + +@router.get( + "/{run_id}", + response_model=RunDetailEnvelope, +) +async def get_run( + run_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + run = await run_service.get_run_for_user( + db_session, + user_id=current_user.id, + run_id=run_id, + ) + if run is None: + raise ApiException( + status_code=404, + code="run_not_found", + message="Run not found.", + ) + error_log = run.error_log if isinstance(run.error_log, dict) else {} + scholar_results = error_log.get("scholar_results") + if not isinstance(scholar_results, list): + scholar_results = [] + safety_state = await _load_safety_state( + db_session, + user_id=current_user.id, + ) + return success_payload( + request, + data={ + "run": _serialize_run(run), + "summary": run_service.extract_run_summary(error_log), + "scholar_results": [_normalize_scholar_result(item) for item in scholar_results], + "safety_state": safety_state, + }, + ) + + +@router.post( + "/{run_id}/cancel", + response_model=RunDetailEnvelope, +) +async def cancel_run( + run_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + run = await run_service.get_run_for_user( + db_session, + user_id=current_user.id, + run_id=run_id, + ) + if run is None: + raise ApiException( + status_code=404, + code="run_not_found", + message="Run not found.", + ) + + if run.status == RunStatus.RUNNING: + run.status = RunStatus.CANCELED + await db_session.commit() + await db_session.refresh(run) + + error_log = run.error_log if isinstance(run.error_log, dict) else {} + scholar_results = error_log.get("scholar_results") + if not isinstance(scholar_results, list): + scholar_results = [] + + safety_state = await _load_safety_state( + db_session, + user_id=current_user.id, + ) + + return success_payload( + request, + data={ + "run": _serialize_run(run), + "summary": run_service.extract_run_summary(error_log), + "scholar_results": [_normalize_scholar_result(item) for item in scholar_results], + "safety_state": safety_state, + }, + ) + + +@router.post( + "/manual", + response_model=ManualRunEnvelope, +) +async def run_manual( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), + ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service), +): + safety_state = await _load_safety_state( + db_session, + user_id=current_user.id, + ) + if not settings.ingestion_manual_run_allowed: + _raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state) + + idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER)) + reused_payload = await _reused_manual_run_payload( + db_session, + request=request, + user_id=current_user.id, + idempotency_key=idempotency_key, + safety_state=safety_state, + ) + if reused_payload is not None: + return reused_payload + + run_summary = await _execute_manual_run( + db_session, + request=request, + ingest_service=ingest_service, + user_id=current_user.id, + idempotency_key=idempotency_key, + ) + if isinstance(run_summary, dict): + return run_summary + + current_safety_state = await _load_safety_state( + db_session, + user_id=current_user.id, + ) + return success_payload( + request, + data=_manual_run_success_payload( + run_summary=run_summary, + idempotency_key=idempotency_key, + safety_state=current_safety_state, + ), + ) + + + +@router.get( + "/queue/items", + response_model=QueueListEnvelope, +) +async def list_queue_items( + request: Request, + limit: int = Query(default=200, ge=1, le=500), + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + items = await run_service.list_queue_items_for_user( + db_session, + user_id=current_user.id, + limit=limit, + ) + return success_payload( + request, + data={ + "queue_items": [_serialize_queue_item(item) for item in items], + }, + ) + + +@router.post( + "/queue/{queue_item_id}/retry", + response_model=QueueItemEnvelope, +) +async def retry_queue_item( + queue_item_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + try: + queue_item = await run_service.retry_queue_item_for_user( + db_session, + user_id=current_user.id, + queue_item_id=queue_item_id, + ) + except run_service.QueueTransitionError as exc: + raise ApiException( + status_code=409, + code=exc.code, + message=exc.message, + details={"current_status": exc.current_status}, + ) from exc + if queue_item is None: + raise ApiException( + status_code=404, + code="queue_item_not_found", + message="Queue item not found.", + ) + return success_payload( + request, + data=_serialize_queue_item(queue_item), + ) + + +@router.post( + "/queue/{queue_item_id}/drop", + response_model=QueueItemEnvelope, +) +async def drop_queue_item( + queue_item_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + try: + dropped = await run_service.drop_queue_item_for_user( + db_session, + user_id=current_user.id, + queue_item_id=queue_item_id, + ) + except run_service.QueueTransitionError as exc: + raise ApiException( + status_code=409, + code=exc.code, + message=exc.message, + details={"current_status": exc.current_status}, + ) from exc + if dropped is None: + raise ApiException( + status_code=404, + code="queue_item_not_found", + message="Queue item not found.", + ) + return success_payload( + request, + data=_serialize_queue_item(dropped), + ) + + +@router.delete( + "/queue/{queue_item_id}", + response_model=QueueClearEnvelope, +) +async def clear_queue_item( + queue_item_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + try: + deleted = await run_service.clear_queue_item_for_user( + db_session, + user_id=current_user.id, + queue_item_id=queue_item_id, + ) + except run_service.QueueTransitionError as exc: + raise ApiException( + status_code=409, + code=exc.code, + message=exc.message, + details={"current_status": exc.current_status}, + ) from exc + if deleted is None: + raise ApiException( + status_code=404, + code="queue_item_not_found", + message="Queue item not found.", + ) + return success_payload( + request, + data={ + "queue_item_id": deleted.queue_item_id, + "previous_status": deleted.previous_status, + "status": "cleared", + "message": "Queue item cleared.", + }, + ) + + +@router.get("/{run_id}/stream") +async def stream_run_events( + run_id: int, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + run = await run_service.get_run_for_user( + db_session, + user_id=current_user.id, + run_id=run_id, + ) + if run is None: + raise ApiException( + status_code=404, + code="run_not_found", + message="Run not found.", + ) + return StreamingResponse( + event_generator(run_id), + media_type="text/event-stream" + ) diff --git a/build/lib/app/api/routers/scholars.py b/build/lib/app/api/routers/scholars.py new file mode 100644 index 0000000..f4be187 --- /dev/null +++ b/build/lib/app/api/routers/scholars.py @@ -0,0 +1,557 @@ +from __future__ import annotations + +import asyncio +import logging + +from fastapi import APIRouter, Depends, File, Query, Request, UploadFile +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_current_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from app.api.runtime_deps import get_scholar_source +from app.api.schemas import ( + DataExportEnvelope, + DataImportEnvelope, + DataImportRequest, + MessageEnvelope, + ScholarCreateRequest, + ScholarEnvelope, + ScholarImageUrlUpdateRequest, + ScholarSearchEnvelope, + ScholarsListEnvelope, +) +from app.db.models import User +from app.db.session import get_db_session +from app.services.domains.portability import application as import_export_service +from app.services.domains.scholars import application as scholar_service +from app.services.domains.scholar.source import ScholarSource +from app.settings import settings + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/scholars", tags=["api-scholars"]) + + +def _uploaded_image_media_path(scholar_profile_id: int) -> str: + return f"/scholar-images/{scholar_profile_id}/upload" + + +def _serialize_scholar(profile) -> dict[str, object]: + uploaded_image_url = None + if profile.profile_image_upload_path: + uploaded_image_url = _uploaded_image_media_path(int(profile.id)) + + profile_image_url, profile_image_source = scholar_service.resolve_profile_image( + profile, + uploaded_image_url=uploaded_image_url, + ) + + return { + "id": int(profile.id), + "scholar_id": profile.scholar_id, + "display_name": profile.display_name, + "profile_image_url": profile_image_url, + "profile_image_source": profile_image_source, + "is_enabled": bool(profile.is_enabled), + "baseline_completed": bool(profile.baseline_completed), + "last_run_dt": profile.last_run_dt, + "last_run_status": ( + profile.last_run_status.value if profile.last_run_status is not None else None + ), + } + + +async def _hydrate_scholar_metadata_if_needed( + db_session: AsyncSession, + *, + profile, + source: ScholarSource, + user_id: int, +): + try: + if not profile.profile_image_url or not (profile.display_name or "").strip(): + return await asyncio.wait_for( + scholar_service.hydrate_profile_metadata( + db_session, + profile=profile, + source=source, + ), + timeout=5.0, + ) + except Exception: + logger.warning( + "api.scholars.create_metadata_hydration_failed", + extra={ + "event": "api.scholars.create_metadata_hydration_failed", + "user_id": user_id, + "scholar_profile_id": profile.id, + }, + ) + return profile + + +def _search_kwargs() -> dict[str, object]: + return { + "network_error_retries": settings.ingestion_network_error_retries, + "retry_backoff_seconds": settings.ingestion_retry_backoff_seconds, + "search_enabled": settings.scholar_name_search_enabled, + "cache_ttl_seconds": settings.scholar_name_search_cache_ttl_seconds, + "blocked_cache_ttl_seconds": settings.scholar_name_search_blocked_cache_ttl_seconds, + "cache_max_entries": settings.scholar_name_search_cache_max_entries, + "min_interval_seconds": settings.scholar_name_search_min_interval_seconds, + "interval_jitter_seconds": settings.scholar_name_search_interval_jitter_seconds, + "cooldown_block_threshold": settings.scholar_name_search_cooldown_block_threshold, + "cooldown_seconds": settings.scholar_name_search_cooldown_seconds, + "retry_alert_threshold": settings.scholar_name_search_alert_retry_count_threshold, + "cooldown_rejection_alert_threshold": ( + settings.scholar_name_search_alert_cooldown_rejections_threshold + ), + } + + +def _search_response_data(query: str, parsed) -> dict[str, object]: + return { + "query": query.strip(), + "state": parsed.state.value, + "state_reason": parsed.state_reason, + "action_hint": scholar_service.scrape_state_hint( + state=parsed.state, + state_reason=parsed.state_reason, + ), + "candidates": [ + { + "scholar_id": item.scholar_id, + "display_name": item.display_name, + "affiliation": item.affiliation, + "email_domain": item.email_domain, + "cited_by_count": item.cited_by_count, + "interests": item.interests, + "profile_url": item.profile_url, + "profile_image_url": item.profile_image_url, + } + for item in parsed.candidates + ], + "warnings": parsed.warnings, + } + + +async def _read_uploaded_image(image: UploadFile) -> bytes: + try: + return await image.read() + finally: + await image.close() + + +async def _require_user_profile( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, +): + profile = await scholar_service.get_user_scholar_by_id( + db_session, + user_id=user_id, + scholar_profile_id=scholar_profile_id, + ) + if profile is None: + raise ApiException( + status_code=404, + code="scholar_not_found", + message="Scholar not found.", + ) + return profile + + +@router.get( + "", + response_model=ScholarsListEnvelope, +) +async def list_scholars( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + scholars = await scholar_service.list_scholars_for_user( + db_session, + user_id=current_user.id, + ) + return success_payload( + request, + data={ + "scholars": [_serialize_scholar(profile) for profile in scholars], + }, + ) + + +@router.get( + "/export", + response_model=DataExportEnvelope, +) +async def export_scholars_and_publications( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + data = await import_export_service.export_user_data( + db_session, + user_id=current_user.id, + ) + return success_payload(request, data=data) + + +@router.post( + "/import", + response_model=DataImportEnvelope, +) +async def import_scholars_and_publications( + payload: DataImportRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + if ( + payload.schema_version is not None + and int(payload.schema_version) != import_export_service.EXPORT_SCHEMA_VERSION + ): + raise ApiException( + status_code=400, + code="invalid_import_schema_version", + message=( + "Import schema version is not supported. " + f"Expected {import_export_service.EXPORT_SCHEMA_VERSION}." + ), + ) + try: + result = await import_export_service.import_user_data( + db_session, + user_id=current_user.id, + scholars=[item.model_dump() for item in payload.scholars], + publications=[item.model_dump() for item in payload.publications], + ) + except import_export_service.ImportExportError as exc: + raise ApiException( + status_code=400, + code="invalid_import_payload", + message=str(exc), + ) from exc + logger.info( + "api.scholars.imported", + extra={ + "event": "api.scholars.imported", + "user_id": current_user.id, + **result, + }, + ) + return success_payload(request, data=result) + + +@router.post( + "", + response_model=ScholarEnvelope, + status_code=201, +) +async def create_scholar( + payload: ScholarCreateRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + source: ScholarSource = Depends(get_scholar_source), + current_user: User = Depends(get_api_current_user), +): + try: + created = await scholar_service.create_scholar_for_user( + db_session, + user_id=current_user.id, + scholar_id=payload.scholar_id, + display_name="", + profile_image_url=payload.profile_image_url, + ) + except scholar_service.ScholarServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_scholar", + message=str(exc), + ) from exc + logger.info( + "api.scholars.created", + extra={ + "event": "api.scholars.created", + "user_id": current_user.id, + "scholar_profile_id": created.id, + }, + ) + created = await _hydrate_scholar_metadata_if_needed( + db_session, + profile=created, + source=source, + user_id=current_user.id, + ) + + return success_payload( + request, + data=_serialize_scholar(created), + ) + + +@router.get( + "/search", + response_model=ScholarSearchEnvelope, +) +async def search_scholars( + request: Request, + query: str = Query(..., min_length=2, max_length=120), + limit: int = Query(10, ge=1, le=25), + db_session: AsyncSession = Depends(get_db_session), + source: ScholarSource = Depends(get_scholar_source), + current_user: User = Depends(get_api_current_user), +): + try: + parsed = await scholar_service.search_author_candidates( + source=source, + db_session=db_session, + query=query, + limit=limit, + **_search_kwargs(), + ) + except scholar_service.ScholarServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_scholar_search", + message=str(exc), + ) from exc + + logger.info( + "api.scholars.search.completed", + extra={ + "event": "api.scholars.search.completed", + "user_id": current_user.id, + "query": query.strip(), + "candidate_count": len(parsed.candidates), + "state": parsed.state.value, + }, + ) + return success_payload( + request, + data=_search_response_data(query, parsed), + ) + + +@router.patch( + "/{scholar_profile_id}/toggle", + response_model=ScholarEnvelope, +) +async def toggle_scholar( + scholar_profile_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + profile = await scholar_service.get_user_scholar_by_id( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + ) + if profile is None: + raise ApiException( + status_code=404, + code="scholar_not_found", + message="Scholar not found.", + ) + updated = await scholar_service.toggle_scholar_enabled(db_session, profile=profile) + logger.info( + "api.scholars.toggled", + extra={ + "event": "api.scholars.toggled", + "user_id": current_user.id, + "scholar_profile_id": updated.id, + "is_enabled": updated.is_enabled, + }, + ) + return success_payload( + request, + data=_serialize_scholar(updated), + ) + + +@router.delete( + "/{scholar_profile_id}", + response_model=MessageEnvelope, +) +async def delete_scholar( + scholar_profile_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + profile = await scholar_service.get_user_scholar_by_id( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + ) + if profile is None: + raise ApiException( + status_code=404, + code="scholar_not_found", + message="Scholar not found.", + ) + await scholar_service.delete_scholar( + db_session, + profile=profile, + upload_dir=settings.scholar_image_upload_dir, + ) + logger.info( + "api.scholars.deleted", + extra={ + "event": "api.scholars.deleted", + "user_id": current_user.id, + "scholar_profile_id": scholar_profile_id, + }, + ) + return success_payload( + request, + data={"message": "Scholar deleted."}, + ) + + +@router.put( + "/{scholar_profile_id}/image/url", + response_model=ScholarEnvelope, +) +async def update_scholar_image_url( + scholar_profile_id: int, + payload: ScholarImageUrlUpdateRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + profile = await scholar_service.get_user_scholar_by_id( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + ) + if profile is None: + raise ApiException( + status_code=404, + code="scholar_not_found", + message="Scholar not found.", + ) + + try: + updated = await scholar_service.set_profile_image_override_url( + db_session, + profile=profile, + image_url=payload.image_url, + upload_dir=settings.scholar_image_upload_dir, + ) + except scholar_service.ScholarServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_scholar_image", + message=str(exc), + ) from exc + + logger.info( + "api.scholars.image_url_updated", + extra={ + "event": "api.scholars.image_url_updated", + "user_id": current_user.id, + "scholar_profile_id": updated.id, + }, + ) + return success_payload( + request, + data=_serialize_scholar(updated), + ) + + +@router.post( + "/{scholar_profile_id}/image/upload", + response_model=ScholarEnvelope, +) +async def upload_scholar_image( + scholar_profile_id: int, + request: Request, + image: UploadFile = File(...), + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + profile = await _require_user_profile( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + ) + + image_bytes = await _read_uploaded_image(image) + try: + updated = await scholar_service.set_profile_image_upload( + db_session, + profile=profile, + content_type=image.content_type, + image_bytes=image_bytes, + upload_dir=settings.scholar_image_upload_dir, + max_upload_bytes=settings.scholar_image_upload_max_bytes, + ) + except scholar_service.ScholarServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_scholar_image", + message=str(exc), + ) from exc + + image_size = len(image_bytes) + logger.info( + "api.scholars.image_uploaded", + extra={ + "event": "api.scholars.image_uploaded", + "user_id": current_user.id, + "scholar_profile_id": updated.id, + "content_type": image.content_type, + "size_bytes": image_size, + }, + ) + return success_payload( + request, + data=_serialize_scholar(updated), + ) + + +@router.delete( + "/{scholar_profile_id}/image", + response_model=ScholarEnvelope, +) +async def clear_scholar_image_customization( + scholar_profile_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + profile = await scholar_service.get_user_scholar_by_id( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + ) + if profile is None: + raise ApiException( + status_code=404, + code="scholar_not_found", + message="Scholar not found.", + ) + + updated = await scholar_service.clear_profile_image_customization( + db_session, + profile=profile, + upload_dir=settings.scholar_image_upload_dir, + ) + logger.info( + "api.scholars.image_customization_cleared", + extra={ + "event": "api.scholars.image_customization_cleared", + "user_id": current_user.id, + "scholar_profile_id": updated.id, + }, + ) + return success_payload( + request, + data=_serialize_scholar(updated), + ) + diff --git a/build/lib/app/api/routers/settings.py b/build/lib/app/api/routers/settings.py new file mode 100644 index 0000000..5a2ef91 --- /dev/null +++ b/build/lib/app/api/routers/settings.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_current_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest +from app.db.models import User +from app.db.session import get_db_session +from app.services.domains.ingestion import safety as run_safety_service +from app.services.domains.settings import application as user_settings_service +from app.settings import settings as settings_module + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/settings", tags=["api-settings"]) + + +def _serialize_settings(user_settings) -> dict[str, object]: + min_run_interval_minutes, min_request_delay_seconds = _minimum_policy() + return { + "auto_run_enabled": bool(user_settings.auto_run_enabled) and settings_module.ingestion_automation_allowed, + "run_interval_minutes": int(user_settings.run_interval_minutes), + "request_delay_seconds": int(user_settings.request_delay_seconds), + "nav_visible_pages": list(user_settings.nav_visible_pages or []), + "policy": { + "min_run_interval_minutes": min_run_interval_minutes, + "min_request_delay_seconds": min_request_delay_seconds, + "automation_allowed": bool(settings_module.ingestion_automation_allowed), + "manual_run_allowed": bool(settings_module.ingestion_manual_run_allowed), + "blocked_failure_threshold": max(1, int(settings_module.ingestion_alert_blocked_failure_threshold)), + "network_failure_threshold": max(1, int(settings_module.ingestion_alert_network_failure_threshold)), + "cooldown_blocked_seconds": max(60, int(settings_module.ingestion_safety_cooldown_blocked_seconds)), + "cooldown_network_seconds": max(60, int(settings_module.ingestion_safety_cooldown_network_seconds)), + }, + "safety_state": run_safety_service.get_safety_state_payload(user_settings), + "openalex_api_key": user_settings.openalex_api_key, + "crossref_api_token": user_settings.crossref_api_token, + "crossref_api_mailto": user_settings.crossref_api_mailto, + } + + +def _minimum_policy() -> tuple[int, int]: + min_run_interval_minutes = user_settings_service.resolve_run_interval_minimum( + settings_module.ingestion_min_run_interval_minutes + ) + min_request_delay_seconds = user_settings_service.resolve_request_delay_minimum( + settings_module.ingestion_min_request_delay_seconds + ) + return min_run_interval_minutes, min_request_delay_seconds + + +def _parse_settings_payload(payload: SettingsUpdateRequest, user_settings) -> tuple[int, int, list[str]]: + min_run_interval_minutes, min_request_delay_seconds = _minimum_policy() + parsed_interval = user_settings_service.parse_run_interval_minutes( + str(payload.run_interval_minutes), + minimum=min_run_interval_minutes, + ) + parsed_delay = user_settings_service.parse_request_delay_seconds( + str(payload.request_delay_seconds), + minimum=min_request_delay_seconds, + ) + parsed_nav_visible_pages = user_settings_service.parse_nav_visible_pages( + payload.nav_visible_pages + if payload.nav_visible_pages is not None + else list(user_settings.nav_visible_pages or user_settings_service.DEFAULT_NAV_VISIBLE_PAGES) + ) + return parsed_interval, parsed_delay, parsed_nav_visible_pages + + +async def _clear_expired_cooldown_with_log( + db_session: AsyncSession, + *, + user_id: int, + user_settings, +) -> None: + previous_safety_state = run_safety_service.get_safety_event_context(user_settings) + if not run_safety_service.clear_expired_cooldown(user_settings): + return + await db_session.commit() + await db_session.refresh(user_settings) + logger.info( + "api.settings.safety_cooldown_cleared", + extra={ + "event": "api.settings.safety_cooldown_cleared", + "user_id": user_id, + "reason": previous_safety_state.get("cooldown_reason"), + "cooldown_until": previous_safety_state.get("cooldown_until"), + "metric_name": "api_settings_safety_cooldown_cleared_total", + "metric_value": 1, + }, + ) + + +def _effective_auto_run_enabled(payload: SettingsUpdateRequest) -> bool: + return bool(payload.auto_run_enabled) and settings_module.ingestion_automation_allowed + + +def _log_settings_update(*, user_id: int, updated) -> None: + logger.info( + "api.settings.updated", + extra={ + "event": "api.settings.updated", + "user_id": user_id, + "auto_run_enabled": updated.auto_run_enabled, + "run_interval_minutes": updated.run_interval_minutes, + "request_delay_seconds": updated.request_delay_seconds, + "nav_visible_pages": updated.nav_visible_pages, + "openalex_api_key": "SET" if updated.openalex_api_key else "UNSET", + }, + ) + + +@router.get( + "", + response_model=SettingsEnvelope, +) +async def get_settings( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + user_settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=current_user.id, + ) + await _clear_expired_cooldown_with_log( + db_session, + user_id=current_user.id, + user_settings=user_settings, + ) + return success_payload( + request, + data=_serialize_settings(user_settings), + ) + + +@router.put( + "", + response_model=SettingsEnvelope, +) +async def update_settings( + payload: SettingsUpdateRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + user_settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=current_user.id, + ) + + try: + parsed_interval, parsed_delay, parsed_nav_visible_pages = _parse_settings_payload( + payload, + user_settings, + ) + except user_settings_service.UserSettingsServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_settings", + message=str(exc), + ) from exc + + updated = await user_settings_service.update_settings( + db_session, + settings=user_settings, + auto_run_enabled=_effective_auto_run_enabled(payload), + run_interval_minutes=parsed_interval, + request_delay_seconds=parsed_delay, + nav_visible_pages=parsed_nav_visible_pages, + openalex_api_key=payload.openalex_api_key, + crossref_api_token=payload.crossref_api_token, + crossref_api_mailto=payload.crossref_api_mailto, + ) + await _clear_expired_cooldown_with_log( + db_session, + user_id=current_user.id, + user_settings=updated, + ) + _log_settings_update(user_id=current_user.id, updated=updated) + return success_payload( + request, + data=_serialize_settings(updated), + ) diff --git a/build/lib/app/api/runtime_deps.py b/build/lib/app/api/runtime_deps.py new file mode 100644 index 0000000..3a692a3 --- /dev/null +++ b/build/lib/app/api/runtime_deps.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from fastapi import Depends + +from app.services.domains.ingestion import application as ingestion_service +from app.services.domains.scholar import source as scholar_source_service + + +def get_scholar_source() -> scholar_source_service.ScholarSource: + return scholar_source_service.LiveScholarSource() + + +def get_ingestion_service( + source: scholar_source_service.ScholarSource = Depends(get_scholar_source), +) -> ingestion_service.ScholarIngestionService: + return ingestion_service.ScholarIngestionService(source=source) diff --git a/build/lib/app/api/schemas.py b/build/lib/app/api/schemas.py new file mode 100644 index 0000000..bac3bb9 --- /dev/null +++ b/build/lib/app/api/schemas.py @@ -0,0 +1,884 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class ApiMeta(BaseModel): + request_id: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class ApiErrorData(BaseModel): + code: str + message: str + details: Any | None = None + + model_config = ConfigDict(extra="forbid") + + +class ApiErrorEnvelope(BaseModel): + error: ApiErrorData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class MessageData(BaseModel): + message: str + + model_config = ConfigDict(extra="forbid") + + +class MessageEnvelope(BaseModel): + data: MessageData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class SessionUserData(BaseModel): + id: int + email: str + is_admin: bool + is_active: bool + + model_config = ConfigDict(extra="forbid") + + +class AuthMeData(BaseModel): + authenticated: bool + csrf_token: str + user: SessionUserData + + model_config = ConfigDict(extra="forbid") + + +class AuthMeEnvelope(BaseModel): + data: AuthMeData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class CsrfBootstrapData(BaseModel): + csrf_token: str + authenticated: bool + + model_config = ConfigDict(extra="forbid") + + +class CsrfBootstrapEnvelope(BaseModel): + data: CsrfBootstrapData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class LoginRequest(BaseModel): + email: str + password: str + + model_config = ConfigDict(extra="forbid") + + +class LoginData(BaseModel): + authenticated: bool + csrf_token: str + user: SessionUserData + + model_config = ConfigDict(extra="forbid") + + +class LoginEnvelope(BaseModel): + data: LoginData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ChangePasswordRequest(BaseModel): + current_password: str + new_password: str + confirm_password: str + + model_config = ConfigDict(extra="forbid") + + +class ScholarItemData(BaseModel): + id: int + scholar_id: str + display_name: str | None + profile_image_url: str | None + profile_image_source: str + is_enabled: bool + baseline_completed: bool + last_run_dt: datetime | None + last_run_status: str | None + + model_config = ConfigDict(extra="forbid") + + +class ScholarsListData(BaseModel): + scholars: list[ScholarItemData] + + model_config = ConfigDict(extra="forbid") + + +class ScholarsListEnvelope(BaseModel): + data: ScholarsListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ScholarCreateRequest(BaseModel): + scholar_id: str + profile_image_url: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class ScholarEnvelope(BaseModel): + data: ScholarItemData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ScholarSearchCandidateData(BaseModel): + scholar_id: str + display_name: str + affiliation: str | None + email_domain: str | None + cited_by_count: int | None + interests: list[str] = Field(default_factory=list) + profile_url: str + profile_image_url: str | None + + model_config = ConfigDict(extra="forbid") + + +class ScholarSearchData(BaseModel): + query: str + state: str + state_reason: str + action_hint: str | None = None + candidates: list[ScholarSearchCandidateData] + warnings: list[str] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class ScholarSearchEnvelope(BaseModel): + data: ScholarSearchData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ScholarImageUrlUpdateRequest(BaseModel): + image_url: str + + model_config = ConfigDict(extra="forbid") + + +class ScholarExportItemData(BaseModel): + scholar_id: str + display_name: str | None = None + is_enabled: bool = True + profile_image_override_url: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class PublicationExportItemData(BaseModel): + scholar_id: str + cluster_id: str | None = None + fingerprint_sha256: str | None = None + title: str + year: int | None = None + citation_count: int = 0 + author_text: str | None = None + venue_text: str | None = None + pub_url: str | None = None + pdf_url: str | None = None + is_read: bool = False + + model_config = ConfigDict(extra="forbid") + + +class DataExportData(BaseModel): + schema_version: int + exported_at: str + scholars: list[ScholarExportItemData] + publications: list[PublicationExportItemData] + + model_config = ConfigDict(extra="forbid") + + +class DataExportEnvelope(BaseModel): + data: DataExportData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class DataImportRequest(BaseModel): + schema_version: int | None = None + scholars: list[ScholarExportItemData] = Field(default_factory=list) + publications: list[PublicationExportItemData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class DataImportResultData(BaseModel): + scholars_created: int + scholars_updated: int + publications_created: int + publications_updated: int + links_created: int + links_updated: int + skipped_records: int + + model_config = ConfigDict(extra="forbid") + + +class DataImportEnvelope(BaseModel): + data: DataImportResultData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class RunListItemData(BaseModel): + id: int + trigger_type: str + status: str + start_dt: datetime + end_dt: datetime | None + scholar_count: int + new_publication_count: int + failed_count: int + partial_count: int + + model_config = ConfigDict(extra="forbid") + + +class RunsListData(BaseModel): + runs: list[RunListItemData] + safety_state: ScrapeSafetyStateData + + model_config = ConfigDict(extra="forbid") + + +class RunsListEnvelope(BaseModel): + data: RunsListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class RunSummaryData(BaseModel): + succeeded_count: int + failed_count: int + partial_count: int + failed_state_counts: dict[str, int] = Field(default_factory=dict) + failed_reason_counts: dict[str, int] = Field(default_factory=dict) + scrape_failure_counts: dict[str, int] = Field(default_factory=dict) + retry_counts: dict[str, int] = Field(default_factory=dict) + alert_thresholds: dict[str, int] = Field(default_factory=dict) + alert_flags: dict[str, bool] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + +class ScrapeSafetyCountersData(BaseModel): + consecutive_blocked_runs: int = 0 + consecutive_network_runs: int = 0 + cooldown_entry_count: int = 0 + blocked_start_count: int = 0 + last_blocked_failure_count: int = 0 + last_network_failure_count: int = 0 + last_evaluated_run_id: int | None = None + + model_config = ConfigDict(extra="forbid") + + +class ScrapeSafetyStateData(BaseModel): + cooldown_active: bool + cooldown_reason: str | None = None + cooldown_reason_label: str | None = None + cooldown_until: datetime | None = None + cooldown_remaining_seconds: int = 0 + recommended_action: str | None = None + counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData) + + model_config = ConfigDict(extra="forbid") + + +class RunAttemptLogData(BaseModel): + attempt: int + cstart: int + state: str | None = None + state_reason: str | None = None + status_code: int | None = None + fetch_error: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class RunPageLogData(BaseModel): + page: int + cstart: int + state: str + state_reason: str | None = None + status_code: int | None = None + publication_count: int = 0 + attempt_count: int = 0 + has_show_more_button: bool = False + articles_range: str | None = None + warning_codes: list[str] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class RunDebugData(BaseModel): + status_code: int | None = None + final_url: str | None = None + fetch_error: str | None = None + body_sha256: str | None = None + body_length: int | None = None + has_show_more_button: bool | None = None + articles_range: str | None = None + state_reason: str | None = None + warning_codes: list[str] = Field(default_factory=list) + marker_counts_nonzero: dict[str, int] = Field(default_factory=dict) + page_logs: list[RunPageLogData] = Field(default_factory=list) + attempt_log: list[RunAttemptLogData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class RunScholarResultData(BaseModel): + scholar_profile_id: int + scholar_id: str + state: str + state_reason: str | None = None + outcome: str + attempt_count: int = 0 + publication_count: int = 0 + start_cstart: int = 0 + continuation_cstart: int | None = None + continuation_enqueued: bool = False + continuation_cleared: bool = False + warnings: list[str] = Field(default_factory=list) + error: str | None = None + debug: RunDebugData | None = None + + model_config = ConfigDict(extra="forbid") + + +class RunDetailData(BaseModel): + run: RunListItemData + summary: RunSummaryData + scholar_results: list[RunScholarResultData] + safety_state: ScrapeSafetyStateData + + model_config = ConfigDict(extra="forbid") + + +class RunDetailEnvelope(BaseModel): + data: RunDetailData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ManualRunData(BaseModel): + run_id: int + status: str + scholar_count: int + succeeded_count: int + failed_count: int + partial_count: int + new_publication_count: int + reused_existing_run: bool + idempotency_key: str | None = None + safety_state: ScrapeSafetyStateData + + model_config = ConfigDict(extra="forbid") + + +class ManualRunEnvelope(BaseModel): + data: ManualRunData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class QueueItemData(BaseModel): + id: int + scholar_profile_id: int + scholar_label: str + status: str + reason: str + dropped_reason: str | None + attempt_count: int + resume_cstart: int + next_attempt_dt: datetime | None + updated_at: datetime + last_error: str | None + last_run_id: int | None + + model_config = ConfigDict(extra="forbid") + + +class QueueListData(BaseModel): + queue_items: list[QueueItemData] + + model_config = ConfigDict(extra="forbid") + + +class QueueListEnvelope(BaseModel): + data: QueueListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class QueueItemEnvelope(BaseModel): + data: QueueItemData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class QueueClearData(BaseModel): + queue_item_id: int + previous_status: str + status: str + message: str + + model_config = ConfigDict(extra="forbid") + + +class QueueClearEnvelope(BaseModel): + data: QueueClearData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminUserData(BaseModel): + id: int + email: str + is_active: bool + is_admin: bool + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(extra="forbid") + + +class AdminUsersListData(BaseModel): + users: list[AdminUserData] + + model_config = ConfigDict(extra="forbid") + + +class AdminUsersListEnvelope(BaseModel): + data: AdminUsersListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminUserCreateRequest(BaseModel): + email: str + password: str + is_admin: bool = False + + model_config = ConfigDict(extra="forbid") + + +class AdminUserActiveUpdateRequest(BaseModel): + is_active: bool + + model_config = ConfigDict(extra="forbid") + + +class AdminUserEnvelope(BaseModel): + data: AdminUserData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminResetPasswordRequest(BaseModel): + new_password: str + + model_config = ConfigDict(extra="forbid") + + +class AdminDbIntegrityCheckData(BaseModel): + name: str + count: int + severity: str + message: str + + model_config = ConfigDict(extra="forbid") + + +class AdminDbIntegrityData(BaseModel): + status: str + checked_at: datetime + failures: list[str] = Field(default_factory=list) + warnings: list[str] = Field(default_factory=list) + checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class AdminDbIntegrityEnvelope(BaseModel): + data: AdminDbIntegrityData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminDbRepairJobData(BaseModel): + id: int + job_name: str + requested_by: str | None + dry_run: bool + status: str + scope: dict[str, Any] = Field(default_factory=dict) + summary: dict[str, Any] = Field(default_factory=dict) + error_text: str | None + started_at: datetime | None + finished_at: datetime | None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(extra="forbid") + + +class AdminDbRepairJobsData(BaseModel): + jobs: list[AdminDbRepairJobData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class AdminDbRepairJobsEnvelope(BaseModel): + data: AdminDbRepairJobsData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class DisplayIdentifierData(BaseModel): + kind: str + value: str + label: str + url: str | None + confidence_score: float = Field(ge=0.0, le=1.0) + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueItemData(BaseModel): + publication_id: int + title: str + display_identifier: DisplayIdentifierData | None = None + pdf_url: str | None + status: str + attempt_count: int + last_failure_reason: str | None + last_failure_detail: str | None + last_source: str | None + requested_by_user_id: int | None + requested_by_email: str | None + queued_at: datetime | None + last_attempt_at: datetime | None + resolved_at: datetime | None + updated_at: datetime + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueData(BaseModel): + items: list[AdminPdfQueueItemData] = Field(default_factory=list) + total_count: int = 0 + page: int = 1 + page_size: int = 100 + has_next: bool = False + has_prev: bool = False + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueEnvelope(BaseModel): + data: AdminPdfQueueData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueRequeueData(BaseModel): + publication_id: int + queued: bool + status: str + message: str + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueRequeueEnvelope(BaseModel): + data: AdminPdfQueueRequeueData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueBulkEnqueueData(BaseModel): + requested_count: int + queued_count: int + message: str + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueBulkEnqueueEnvelope(BaseModel): + data: AdminPdfQueueBulkEnqueueData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminRepairPublicationLinksRequest(BaseModel): + scope_mode: Literal["single_user", "all_users"] = "single_user" + user_id: int | None = Field(default=None, ge=1) + scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200) + dry_run: bool = True + gc_orphan_publications: bool = False + requested_by: str | None = None + confirmation_text: str | None = None + + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="after") + def validate_scope(self) -> "AdminRepairPublicationLinksRequest": + if self.scope_mode == "single_user" and self.user_id is None: + raise ValueError("user_id is required when scope_mode=single_user.") + if self.scope_mode == "all_users" and self.user_id is not None: + raise ValueError("user_id must be omitted when scope_mode=all_users.") + if not self.dry_run and self.scope_mode == "all_users": + expected = "REPAIR ALL USERS" + provided = (self.confirmation_text or "").strip() + if provided != expected: + raise ValueError( + "confirmation_text must equal 'REPAIR ALL USERS' " + "when applying a repair to all users." + ) + return self + + +class AdminRepairPublicationLinksResultData(BaseModel): + job_id: int + status: str + scope: dict[str, Any] = Field(default_factory=dict) + summary: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + +class AdminRepairPublicationLinksEnvelope(BaseModel): + data: AdminRepairPublicationLinksResultData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class SettingsPolicyData(BaseModel): + min_run_interval_minutes: int + min_request_delay_seconds: int + automation_allowed: bool + manual_run_allowed: bool + blocked_failure_threshold: int + network_failure_threshold: int + cooldown_blocked_seconds: int + cooldown_network_seconds: int + + model_config = ConfigDict(extra="forbid") + + +class SettingsData(BaseModel): + auto_run_enabled: bool + run_interval_minutes: int + request_delay_seconds: int + nav_visible_pages: list[str] + policy: SettingsPolicyData + safety_state: ScrapeSafetyStateData + + openalex_api_key: str | None = None + crossref_api_token: str | None = None + crossref_api_mailto: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class SettingsEnvelope(BaseModel): + data: SettingsData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class SettingsUpdateRequest(BaseModel): + auto_run_enabled: bool + run_interval_minutes: int + request_delay_seconds: int + nav_visible_pages: list[str] | None = None + + openalex_api_key: str | None = None + crossref_api_token: str | None = None + crossref_api_mailto: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class PublicationItemData(BaseModel): + publication_id: int + scholar_profile_id: int + scholar_label: str + title: str + year: int | None + citation_count: int + venue_text: str | None + pub_url: str | None + display_identifier: DisplayIdentifierData | None = None + pdf_url: str | None + pdf_status: str = "untracked" + pdf_attempt_count: int = 0 + pdf_failure_reason: str | None = None + pdf_failure_detail: str | None = None + is_read: bool + is_favorite: bool = False + first_seen_at: datetime + is_new_in_latest_run: bool + + model_config = ConfigDict(extra="forbid") + + +class PublicationsListData(BaseModel): + mode: str + favorite_only: bool = False + selected_scholar_profile_id: int | None + unread_count: int + favorites_count: int + latest_count: int + new_count: int + total_count: int + page: int + page_size: int + has_next: bool = False + has_prev: bool = False + publications: list[PublicationItemData] + + model_config = ConfigDict(extra="forbid") + + +class PublicationsListEnvelope(BaseModel): + data: PublicationsListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class MarkAllReadData(BaseModel): + message: str + updated_count: int + + model_config = ConfigDict(extra="forbid") + + +class MarkAllReadEnvelope(BaseModel): + data: MarkAllReadData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class PublicationSelectionItem(BaseModel): + scholar_profile_id: int + publication_id: int + + model_config = ConfigDict(extra="forbid") + + +class MarkSelectedReadRequest(BaseModel): + selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500) + + model_config = ConfigDict(extra="forbid") + + +class MarkSelectedReadData(BaseModel): + message: str + requested_count: int + updated_count: int + + model_config = ConfigDict(extra="forbid") + + +class MarkSelectedReadEnvelope(BaseModel): + data: MarkSelectedReadData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class RetryPublicationPdfRequest(BaseModel): + scholar_profile_id: int = Field(ge=1) + + model_config = ConfigDict(extra="forbid") + + +class RetryPublicationPdfData(BaseModel): + message: str + queued: bool + resolved_pdf: bool + publication: PublicationItemData + + model_config = ConfigDict(extra="forbid") + + +class RetryPublicationPdfEnvelope(BaseModel): + data: RetryPublicationPdfData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class TogglePublicationFavoriteRequest(BaseModel): + scholar_profile_id: int = Field(ge=1) + is_favorite: bool + + model_config = ConfigDict(extra="forbid") + + +class TogglePublicationFavoriteData(BaseModel): + message: str + publication: PublicationItemData + + model_config = ConfigDict(extra="forbid") + + +class TogglePublicationFavoriteEnvelope(BaseModel): + data: TogglePublicationFavoriteData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") diff --git a/build/lib/app/auth/__init__.py b/build/lib/app/auth/__init__.py new file mode 100644 index 0000000..79644a4 --- /dev/null +++ b/build/lib/app/auth/__init__.py @@ -0,0 +1,2 @@ +"""Authentication package for scholarr.""" + diff --git a/build/lib/app/auth/deps.py b/build/lib/app/auth/deps.py new file mode 100644 index 0000000..50424ae --- /dev/null +++ b/build/lib/app/auth/deps.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from functools import lru_cache + +from app.auth.rate_limit import SlidingWindowRateLimiter +from app.auth.security import PasswordService +from app.auth.service import AuthService +from app.settings import settings + + +@lru_cache +def get_password_service() -> PasswordService: + return PasswordService() + + +@lru_cache +def get_auth_service() -> AuthService: + return AuthService(password_service=get_password_service()) + + +@lru_cache +def get_login_rate_limiter() -> SlidingWindowRateLimiter: + return SlidingWindowRateLimiter( + max_attempts=settings.login_rate_limit_attempts, + window_seconds=settings.login_rate_limit_window_seconds, + ) + diff --git a/build/lib/app/auth/rate_limit.py b/build/lib/app/auth/rate_limit.py new file mode 100644 index 0000000..c3f5e8a --- /dev/null +++ b/build/lib/app/auth/rate_limit.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from collections import deque +from collections.abc import Callable +from dataclasses import dataclass +from math import ceil +from threading import Lock +from time import monotonic + + +@dataclass(frozen=True) +class RateLimitDecision: + allowed: bool + retry_after_seconds: int = 0 + + +class SlidingWindowRateLimiter: + def __init__( + self, + *, + max_attempts: int, + window_seconds: int, + now: Callable[[], float] = monotonic, + ) -> None: + self._max_attempts = max_attempts + self._window_seconds = window_seconds + self._now = now + self._attempts: dict[str, deque[float]] = {} + self._lock = Lock() + + def check(self, key: str) -> RateLimitDecision: + with self._lock: + now_value = self._now() + attempts = self._attempts.get(key) + if attempts is None: + return RateLimitDecision(allowed=True) + self._trim_expired(attempts, now_value) + if not attempts: + self._attempts.pop(key, None) + return RateLimitDecision(allowed=True) + if len(attempts) >= self._max_attempts: + retry_after = self._window_seconds - (now_value - attempts[0]) + return RateLimitDecision( + allowed=False, + retry_after_seconds=max(1, ceil(retry_after)), + ) + return RateLimitDecision(allowed=True) + + def record_failure(self, key: str) -> None: + with self._lock: + now_value = self._now() + attempts = self._attempts.get(key) + if attempts is None: + attempts = deque() + self._attempts[key] = attempts + self._trim_expired(attempts, now_value) + attempts.append(now_value) + + def reset(self, key: str) -> None: + with self._lock: + self._attempts.pop(key, None) + + def clear_all(self) -> None: + with self._lock: + self._attempts.clear() + + def _trim_expired(self, attempts: deque[float], now_value: float) -> None: + while attempts and now_value - attempts[0] >= self._window_seconds: + attempts.popleft() diff --git a/build/lib/app/auth/runtime.py b/build/lib/app/auth/runtime.py new file mode 100644 index 0000000..777e799 --- /dev/null +++ b/build/lib/app/auth/runtime.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import logging + +from fastapi import Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.session import clear_session_user, get_session_user, set_session_user +from app.db.models import User +from app.security.csrf import CSRF_SESSION_KEY +from app.services.domains.users import application as user_service + +logger = logging.getLogger(__name__) + + +def invalidate_session(request: Request) -> None: + clear_session_user(request) + request.session.pop(CSRF_SESSION_KEY, None) + + +async def get_authenticated_user( + request: Request, + db_session: AsyncSession, +) -> User | None: + session_user = get_session_user(request) + if session_user is None: + return None + + user = await user_service.get_user_by_id(db_session, session_user.id) + if user is None or not user.is_active: + logger.info( + "auth.session_invalidated", + extra={ + "event": "auth.session_invalidated", + "session_user_id": session_user.id, + }, + ) + invalidate_session(request) + return None + + if user.email != session_user.email or user.is_admin != session_user.is_admin: + set_session_user( + request, + user_id=user.id, + email=user.email, + is_admin=user.is_admin, + ) + + return user + + +def login_rate_limit_key(request: Request, email: str) -> str: + client_host = request.client.host if request.client is not None else "unknown" + normalized_email = email.strip().lower() + return f"{client_host}:{normalized_email or ''}" diff --git a/build/lib/app/auth/security.py b/build/lib/app/auth/security.py new file mode 100644 index 0000000..f89b122 --- /dev/null +++ b/build/lib/app/auth/security.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError + + +class PasswordService: + def __init__(self, hasher: PasswordHasher | None = None) -> None: + self._hasher = hasher or PasswordHasher() + + def hash_password(self, password: str) -> str: + return self._hasher.hash(password) + + def verify_password(self, password_hash: str, password: str) -> bool: + try: + return bool(self._hasher.verify(password_hash, password)) + except (InvalidHashError, VerificationError): + return False + diff --git a/build/lib/app/auth/service.py b/build/lib/app/auth/service.py new file mode 100644 index 0000000..9772358 --- /dev/null +++ b/build/lib/app/auth/service.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.security import PasswordService +from app.db.models import User + + +class AuthService: + def __init__(self, password_service: PasswordService) -> None: + self._password_service = password_service + + async def authenticate_user( + self, + db_session: AsyncSession, + *, + email: str, + password: str, + ) -> User | None: + normalized_email = email.strip().lower() + if not normalized_email or not password: + return None + result = await db_session.execute( + select(User).where(User.email == normalized_email) + ) + user = result.scalar_one_or_none() + if user is None or not user.is_active: + return None + if not self._password_service.verify_password(user.password_hash, password): + return None + return user + + def hash_password(self, password: str) -> str: + return self._password_service.hash_password(password) + + def verify_password(self, *, password_hash: str, password: str) -> bool: + return self._password_service.verify_password(password_hash, password) diff --git a/build/lib/app/auth/session.py b/build/lib/app/auth/session.py new file mode 100644 index 0000000..8c83197 --- /dev/null +++ b/build/lib/app/auth/session.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from starlette.requests import Request + + +SESSION_USER_ID_KEY = "auth_user_id" +SESSION_USER_EMAIL_KEY = "auth_user_email" +SESSION_USER_IS_ADMIN_KEY = "auth_user_is_admin" + + +@dataclass(frozen=True) +class SessionUser: + id: int + email: str + is_admin: bool + + +def get_session_user(request: Request) -> SessionUser | None: + user_id = request.session.get(SESSION_USER_ID_KEY) + email = request.session.get(SESSION_USER_EMAIL_KEY) + is_admin = request.session.get(SESSION_USER_IS_ADMIN_KEY) + if user_id is None or email is None or is_admin is None: + return None + try: + parsed_user_id = int(user_id) + except (TypeError, ValueError): + return None + if not isinstance(email, str): + return None + return SessionUser(id=parsed_user_id, email=email, is_admin=bool(is_admin)) + + +def set_session_user( + request: Request, + *, + user_id: int, + email: str, + is_admin: bool, +) -> None: + request.session[SESSION_USER_ID_KEY] = int(user_id) + request.session[SESSION_USER_EMAIL_KEY] = email + request.session[SESSION_USER_IS_ADMIN_KEY] = bool(is_admin) + + +def clear_session_user(request: Request) -> None: + request.session.pop(SESSION_USER_ID_KEY, None) + request.session.pop(SESSION_USER_EMAIL_KEY, None) + request.session.pop(SESSION_USER_IS_ADMIN_KEY, None) diff --git a/build/lib/app/db/__init__.py b/build/lib/app/db/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/build/lib/app/db/__init__.py @@ -0,0 +1 @@ + diff --git a/build/lib/app/db/base.py b/build/lib/app/db/base.py new file mode 100644 index 0000000..6fa2049 --- /dev/null +++ b/build/lib/app/db/base.py @@ -0,0 +1,27 @@ +from datetime import datetime, timezone + +from sqlalchemy import MetaData, event +from sqlalchemy.orm import DeclarativeBase + + +NAMING_CONVENTION = { + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + + +class Base(DeclarativeBase): + metadata = MetaData(naming_convention=NAMING_CONVENTION) + + +@event.listens_for(Base, "before_update", propagate=True) +def _set_updated_at_before_update(_mapper, _connection, target) -> None: + # Keep audit timestamps current for ORM-managed updates. + if hasattr(target, "updated_at"): + target.updated_at = datetime.now(timezone.utc) + + +metadata = Base.metadata diff --git a/build/lib/app/db/models.py b/build/lib/app/db/models.py new file mode 100644 index 0000000..1a8ad8a --- /dev/null +++ b/build/lib/app/db/models.py @@ -0,0 +1,546 @@ +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum + +from sqlalchemy import ( + Boolean, + CheckConstraint, + DateTime, + Enum, + Float, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, + func, + text, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base + + +class RunTriggerType(StrEnum): + MANUAL = "manual" + SCHEDULED = "scheduled" + + +class RunStatus(StrEnum): + RUNNING = "running" + SUCCESS = "success" + PARTIAL_FAILURE = "partial_failure" + FAILED = "failed" + CANCELED = "canceled" + + +class QueueItemStatus(StrEnum): + QUEUED = "queued" + RETRYING = "retrying" + DROPPED = "dropped" + + +RUN_STATUS_DB_ENUM = Enum( + RunStatus, + name="run_status", + values_callable=lambda members: [member.value for member in members], +) +RUN_TRIGGER_TYPE_DB_ENUM = Enum( + RunTriggerType, + name="run_trigger_type", + values_callable=lambda members: [member.value for member in members], +) + + +class User(Base): + __tablename__ = "users" + + id: Mapped[int] = mapped_column(primary_key=True) + email: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + password_hash: Mapped[str] = mapped_column(String(255), nullable=False) + is_active: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("true") + ) + is_admin: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class UserSetting(Base): + __tablename__ = "user_settings" + __table_args__ = ( + CheckConstraint( + "run_interval_minutes >= 15", + name="run_interval_minutes_min_15", + ), + CheckConstraint( + "request_delay_seconds >= 2", + name="request_delay_seconds_min_2", + ), + ) + + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), primary_key=True + ) + auto_run_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) + run_interval_minutes: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("1440") + ) + request_delay_seconds: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("10") + ) + nav_visible_pages: Mapped[list[str]] = mapped_column( + JSONB, + nullable=False, + server_default=text( + '\'["dashboard","scholars","publications","settings","style-guide","runs","users"]\'::jsonb' + ), + ) + scrape_safety_state: Mapped[dict] = mapped_column( + JSONB, + nullable=False, + server_default=text("'{}'::jsonb"), + ) + scrape_cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + scrape_cooldown_reason: Mapped[str | None] = mapped_column(String(64)) + + openalex_api_key: Mapped[str | None] = mapped_column(String(255)) + crossref_api_token: Mapped[str | None] = mapped_column(String(255)) + crossref_api_mailto: Mapped[str | None] = mapped_column(String(255)) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class ScholarProfile(Base): + __tablename__ = "scholar_profiles" + __table_args__ = ( + UniqueConstraint("user_id", "scholar_id", name="uq_scholar_profiles_user_scholar"), + Index("ix_scholar_profiles_user_enabled", "user_id", "is_enabled"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + scholar_id: Mapped[str] = mapped_column(String(64), nullable=False) + display_name: Mapped[str | None] = mapped_column(String(255)) + profile_image_url: Mapped[str | None] = mapped_column(Text) + profile_image_override_url: Mapped[str | None] = mapped_column(Text) + profile_image_upload_path: Mapped[str | None] = mapped_column(Text) + last_initial_page_fingerprint_sha256: Mapped[str | None] = mapped_column(String(64)) + last_initial_page_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + is_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("true") + ) + baseline_completed: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) + last_run_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_run_status: Mapped[RunStatus | None] = mapped_column( + RUN_STATUS_DB_ENUM, + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class CrawlRun(Base): + __tablename__ = "crawl_runs" + __table_args__ = ( + Index("ix_crawl_runs_user_start", "user_id", "start_dt"), + Index( + "uq_crawl_runs_user_manual_idempotency_key", + "user_id", + "idempotency_key", + unique=True, + postgresql_where=text( + "idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type" + ), + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + trigger_type: Mapped[RunTriggerType] = mapped_column( + RUN_TRIGGER_TYPE_DB_ENUM, nullable=False + ) + status: Mapped[RunStatus] = mapped_column( + RUN_STATUS_DB_ENUM, nullable=False + ) + start_dt: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + end_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + scholar_count: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("0") + ) + new_pub_count: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("0") + ) + idempotency_key: Mapped[str | None] = mapped_column(String(128)) + error_log: Mapped[dict] = mapped_column( + JSONB, nullable=False, server_default=text("'{}'::jsonb") + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class Publication(Base): + __tablename__ = "publications" + __table_args__ = ( + UniqueConstraint("fingerprint_sha256", name="uq_publications_fingerprint"), + Index( + "uq_publications_cluster_id_not_null", + "cluster_id", + unique=True, + postgresql_where=text("cluster_id IS NOT NULL"), + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + cluster_id: Mapped[str | None] = mapped_column(String(64)) + fingerprint_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + title_raw: Mapped[str] = mapped_column(Text, nullable=False) + title_normalized: Mapped[str] = mapped_column(Text, nullable=False) + year: Mapped[int | None] = mapped_column(Integer) + citation_count: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("0") + ) + author_text: Mapped[str | None] = mapped_column(Text) + venue_text: Mapped[str | None] = mapped_column(Text) + pub_url: Mapped[str | None] = mapped_column(Text) + pdf_url: Mapped[str | None] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class PublicationIdentifier(Base): + __tablename__ = "publication_identifiers" + __table_args__ = ( + UniqueConstraint( + "publication_id", + "kind", + "value_normalized", + name="uq_publication_identifiers_publication_kind_value", + ), + CheckConstraint( + "confidence_score >= 0 AND confidence_score <= 1", + name="publication_identifiers_confidence_score_range", + ), + Index( + "ix_publication_identifiers_kind_value", + "kind", + "value_normalized", + ), + Index( + "ix_publication_identifiers_publication_id", + "publication_id", + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + publication_id: Mapped[int] = mapped_column( + ForeignKey("publications.id", ondelete="CASCADE"), + nullable=False, + ) + kind: Mapped[str] = mapped_column(String(32), nullable=False) + value_raw: Mapped[str] = mapped_column(Text, nullable=False) + value_normalized: Mapped[str] = mapped_column(String(255), nullable=False) + source: Mapped[str] = mapped_column(String(64), nullable=False) + confidence_score: Mapped[float] = mapped_column( + Float, + nullable=False, + server_default=text("0"), + ) + evidence_url: Mapped[str | None] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class PublicationPdfJob(Base): + __tablename__ = "publication_pdf_jobs" + __table_args__ = ( + CheckConstraint( + "status IN ('queued', 'running', 'resolved', 'failed')", + name="publication_pdf_jobs_status_valid", + ), + Index("ix_publication_pdf_jobs_status", "status"), + Index("ix_publication_pdf_jobs_updated_at", "updated_at"), + Index("ix_publication_pdf_jobs_queued_at", "queued_at"), + ) + + publication_id: Mapped[int] = mapped_column( + ForeignKey("publications.id", ondelete="CASCADE"), + primary_key=True, + ) + status: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'queued'"), + ) + attempt_count: Mapped[int] = mapped_column( + Integer, + nullable=False, + server_default=text("0"), + ) + queued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_failure_reason: Mapped[str | None] = mapped_column(String(64)) + last_failure_detail: Mapped[str | None] = mapped_column(Text) + last_source: Mapped[str | None] = mapped_column(String(32)) + last_requested_by_user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class PublicationPdfJobEvent(Base): + __tablename__ = "publication_pdf_job_events" + __table_args__ = ( + Index("ix_publication_pdf_job_events_publication_created", "publication_id", "created_at"), + Index("ix_publication_pdf_job_events_created_at", "created_at"), + Index("ix_publication_pdf_job_events_event_type", "event_type"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + publication_id: Mapped[int] = mapped_column( + ForeignKey("publications.id", ondelete="CASCADE"), + nullable=False, + ) + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), + ) + event_type: Mapped[str] = mapped_column(String(32), nullable=False) + status: Mapped[str | None] = mapped_column(String(16)) + source: Mapped[str | None] = mapped_column(String(32)) + failure_reason: Mapped[str | None] = mapped_column(String(64)) + message: Mapped[str | None] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class ScholarPublication(Base): + __tablename__ = "scholar_publications" + __table_args__ = ( + Index("ix_scholar_publications_is_read", "is_read"), + Index("ix_scholar_publications_is_favorite", "is_favorite"), + ) + + scholar_profile_id: Mapped[int] = mapped_column( + ForeignKey("scholar_profiles.id", ondelete="CASCADE"), + primary_key=True, + ) + publication_id: Mapped[int] = mapped_column( + ForeignKey("publications.id", ondelete="CASCADE"), + primary_key=True, + ) + is_read: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) + is_favorite: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) + first_seen_run_id: Mapped[int | None] = mapped_column( + ForeignKey("crawl_runs.id", ondelete="SET NULL") + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class IngestionQueueItem(Base): + __tablename__ = "ingestion_queue_items" + __table_args__ = ( + UniqueConstraint( + "user_id", + "scholar_profile_id", + name="uq_ingestion_queue_user_scholar", + ), + CheckConstraint( + "status IN ('queued', 'retrying', 'dropped')", + name="ingestion_queue_status_valid", + ), + Index("ix_ingestion_queue_next_attempt", "next_attempt_dt"), + Index("ix_ingestion_queue_status_next_attempt", "status", "next_attempt_dt"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ) + scholar_profile_id: Mapped[int] = mapped_column( + ForeignKey("scholar_profiles.id", ondelete="CASCADE"), + nullable=False, + ) + resume_cstart: Mapped[int] = mapped_column( + Integer, + nullable=False, + server_default=text("0"), + ) + reason: Mapped[str] = mapped_column(String(64), nullable=False) + status: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'queued'"), + ) + attempt_count: Mapped[int] = mapped_column( + Integer, + nullable=False, + server_default=text("0"), + ) + next_attempt_dt: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + ) + last_run_id: Mapped[int | None] = mapped_column( + ForeignKey("crawl_runs.id", ondelete="SET NULL"), + ) + last_error: Mapped[str | None] = mapped_column(Text) + dropped_reason: Mapped[str | None] = mapped_column(String(128)) + dropped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class DataRepairJob(Base): + __tablename__ = "data_repair_jobs" + __table_args__ = ( + CheckConstraint( + "status IN ('planned', 'running', 'completed', 'failed')", + name="data_repair_jobs_status_valid", + ), + Index("ix_data_repair_jobs_created_at", "created_at"), + Index("ix_data_repair_jobs_job_name_created_at", "job_name", "created_at"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + job_name: Mapped[str] = mapped_column(String(64), nullable=False) + requested_by: Mapped[str | None] = mapped_column(String(255)) + scope: Mapped[dict] = mapped_column( + JSONB, + nullable=False, + server_default=text("'{}'::jsonb"), + ) + dry_run: Mapped[bool] = mapped_column( + Boolean, + nullable=False, + server_default=text("true"), + ) + status: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'planned'"), + ) + summary: Mapped[dict] = mapped_column( + JSONB, + nullable=False, + server_default=text("'{}'::jsonb"), + ) + error_text: Mapped[str | None] = mapped_column(Text) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class AuthorSearchRuntimeState(Base): + __tablename__ = "author_search_runtime_state" + + state_key: Mapped[str] = mapped_column(String(64), primary_key=True) + last_live_request_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + consecutive_blocked_count: Mapped[int] = mapped_column( + Integer, + nullable=False, + server_default=text("0"), + ) + cooldown_rejection_count: Mapped[int] = mapped_column( + Integer, + nullable=False, + server_default=text("0"), + ) + cooldown_alert_emitted: Mapped[bool] = mapped_column( + Boolean, + nullable=False, + server_default=text("false"), + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class AuthorSearchCacheEntry(Base): + __tablename__ = "author_search_cache_entries" + __table_args__ = ( + Index("ix_author_search_cache_expires_at", "expires_at"), + Index("ix_author_search_cache_cached_at", "cached_at"), + ) + + query_key: Mapped[str] = mapped_column(String(256), primary_key=True) + payload: Mapped[dict] = mapped_column( + JSONB, + nullable=False, + ) + expires_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + ) + cached_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/build/lib/app/db/session.py b/build/lib/app/db/session.py new file mode 100644 index 0000000..c05266e --- /dev/null +++ b/build/lib/app/db/session.py @@ -0,0 +1,99 @@ +from collections.abc import AsyncIterator +import logging +import os + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) +from sqlalchemy.pool import NullPool + +from app.settings import settings + +logger = logging.getLogger(__name__) + +_engine: AsyncEngine | None = None +_session_factory: async_sessionmaker[AsyncSession] | None = None + + +def _normalized_pool_mode(raw_mode: str) -> str: + mode = (raw_mode or "").strip().lower() + if mode == "auto": + if os.getenv("PYTEST_CURRENT_TEST"): + return "null" + app_env = (os.getenv("APP_ENV") or "").strip().lower() + if app_env in {"test", "development", "dev", "local"}: + return "null" + return "queue" + if mode in {"null", "queue"}: + return mode + logger.warning( + "db.invalid_pool_mode_fallback", + extra={ + "event": "db.invalid_pool_mode_fallback", + "database_pool_mode": raw_mode, + "fallback_mode": "queue", + }, + ) + return "queue" + + +def get_engine() -> AsyncEngine: + global _engine + if _engine is None: + pool_mode = _normalized_pool_mode(settings.database_pool_mode) + engine_kwargs: dict[str, object] = { + "pool_pre_ping": True, + } + if pool_mode == "null": + engine_kwargs["poolclass"] = NullPool + else: + engine_kwargs["pool_size"] = max(1, int(settings.database_pool_size)) + engine_kwargs["max_overflow"] = max(0, int(settings.database_pool_max_overflow)) + engine_kwargs["pool_timeout"] = max(1, int(settings.database_pool_timeout_seconds)) + + _engine = create_async_engine(settings.database_url, **engine_kwargs) + logger.info( + "db.engine_initialized", + extra={ + "event": "db.engine_initialized", + "pool_mode": pool_mode, + }, + ) + return _engine + + +def get_session_factory() -> async_sessionmaker[AsyncSession]: + global _session_factory + if _session_factory is None: + _session_factory = async_sessionmaker(get_engine(), expire_on_commit=False) + return _session_factory + + +async def get_db_session() -> AsyncIterator[AsyncSession]: + session_factory = get_session_factory() + async with session_factory() as session: + yield session + + +async def check_database() -> bool: + engine = get_engine() + try: + async with engine.connect() as conn: + result = await conn.execute(text("SELECT 1")) + return result.scalar_one() == 1 + except Exception: + logger.exception("db.healthcheck_failed", extra={"event": "db.healthcheck_failed"}) + return False + + +async def close_engine() -> None: + global _engine, _session_factory + if _engine is not None: + await _engine.dispose() + logger.info("db.engine_disposed", extra={"event": "db.engine_disposed"}) + _engine = None + _session_factory = None diff --git a/build/lib/app/http/__init__.py b/build/lib/app/http/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/build/lib/app/http/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/build/lib/app/http/middleware.py b/build/lib/app/http/middleware.py new file mode 100644 index 0000000..ebe72f8 --- /dev/null +++ b/build/lib/app/http/middleware.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +from secrets import token_urlsafe +import logging +import time + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response + +from app.logging_context import set_request_id + +REQUEST_ID_HEADER = "X-Request-ID" +DEFAULT_PERMISSIONS_POLICY = ( + "accelerometer=(), autoplay=(), camera=(), display-capture=(), " + "geolocation=(), gyroscope=(), microphone=(), payment=(), usb=()" +) +DEFAULT_CSP_POLICY = ( + "default-src 'self'; " + "base-uri 'self'; " + "form-action 'self'; " + "frame-ancestors 'none'; " + "img-src 'self' data: https:; " + "script-src 'self'; " + "style-src 'self' 'unsafe-inline'; " + "font-src 'self' data:; " + "connect-src 'self'; " + "object-src 'none'" +) +DEFAULT_CSP_DOCS_POLICY = ( + "default-src 'self'; " + "img-src 'self' data: https:; " + "script-src 'self' 'unsafe-inline'; " + "style-src 'self' 'unsafe-inline'; " + "font-src 'self' data:; " + "connect-src 'self' https:; " + "object-src 'none'; " + "frame-ancestors 'none'" +) + +logger = logging.getLogger(__name__) + + +class RequestLoggingMiddleware(BaseHTTPMiddleware): + def __init__( + self, + app, + *, + log_requests: bool = True, + skip_paths: tuple[str, ...] = (), + ) -> None: + super().__init__(app) + self._log_requests = log_requests + self._skip_paths = tuple(path for path in skip_paths if path) + + async def dispatch(self, request: Request, call_next) -> Response: + request_id = request.headers.get(REQUEST_ID_HEADER) or token_urlsafe(12) + request.state.request_id = request_id + set_request_id(request_id) + + start = time.perf_counter() + should_log = self._log_requests and not self._is_skipped_path(request.url.path) + if should_log: + logger.debug( + "request.started", + extra={ + "event": "request.started", + "method": request.method, + "path": request.url.path, + }, + ) + + try: + response = await call_next(request) + except Exception: + duration_ms = int((time.perf_counter() - start) * 1000) + logger.exception( + "request.failed", + extra={ + "event": "request.failed", + "method": request.method, + "path": request.url.path, + "duration_ms": duration_ms, + }, + ) + raise + else: + duration_ms = int((time.perf_counter() - start) * 1000) + response.headers[REQUEST_ID_HEADER] = request_id + if should_log: + logger.debug( + "request.completed", + extra={ + "event": "request.completed", + "method": request.method, + "path": request.url.path, + "status_code": response.status_code, + "duration_ms": duration_ms, + }, + ) + return response + finally: + set_request_id(None) + + def _is_skipped_path(self, path: str) -> bool: + return any(path.startswith(prefix) for prefix in self._skip_paths) + + +class SecurityHeadersMiddleware(BaseHTTPMiddleware): + def __init__( + self, + app, + *, + enabled: bool = True, + x_content_type_options: str = "nosniff", + x_frame_options: str = "DENY", + referrer_policy: str = "strict-origin-when-cross-origin", + permissions_policy: str = DEFAULT_PERMISSIONS_POLICY, + cross_origin_opener_policy: str = "same-origin", + cross_origin_resource_policy: str = "same-origin", + content_security_policy_enabled: bool = True, + content_security_policy: str = DEFAULT_CSP_POLICY, + content_security_policy_docs: str = DEFAULT_CSP_DOCS_POLICY, + content_security_policy_report_only: bool = False, + strict_transport_security_enabled: bool = False, + strict_transport_security_max_age: int = 31_536_000, + strict_transport_security_include_subdomains: bool = True, + strict_transport_security_preload: bool = False, + ) -> None: + super().__init__(app) + self._enabled = enabled + self._x_content_type_options = x_content_type_options.strip() + self._x_frame_options = x_frame_options.strip() + self._referrer_policy = referrer_policy.strip() + self._permissions_policy = permissions_policy.strip() + self._cross_origin_opener_policy = cross_origin_opener_policy.strip() + self._cross_origin_resource_policy = cross_origin_resource_policy.strip() + self._csp_enabled = content_security_policy_enabled + self._csp_policy = content_security_policy.strip() + self._csp_docs_policy = content_security_policy_docs.strip() + self._csp_report_only = content_security_policy_report_only + self._hsts_enabled = strict_transport_security_enabled + self._hsts_max_age = max(0, strict_transport_security_max_age) + self._hsts_include_subdomains = strict_transport_security_include_subdomains + self._hsts_preload = strict_transport_security_preload + + async def dispatch(self, request: Request, call_next) -> Response: + response = await call_next(request) + if not self._enabled: + return response + + if self._x_content_type_options: + response.headers.setdefault("X-Content-Type-Options", self._x_content_type_options) + if self._x_frame_options: + response.headers.setdefault("X-Frame-Options", self._x_frame_options) + if self._referrer_policy: + response.headers.setdefault("Referrer-Policy", self._referrer_policy) + if self._permissions_policy: + response.headers.setdefault("Permissions-Policy", self._permissions_policy) + if self._cross_origin_opener_policy: + response.headers.setdefault( + "Cross-Origin-Opener-Policy", + self._cross_origin_opener_policy, + ) + if self._cross_origin_resource_policy: + response.headers.setdefault( + "Cross-Origin-Resource-Policy", + self._cross_origin_resource_policy, + ) + + csp_policy = self._csp_policy_for_path(request.url.path) + if self._csp_enabled and csp_policy: + csp_header = ( + "Content-Security-Policy-Report-Only" + if self._csp_report_only + else "Content-Security-Policy" + ) + response.headers.setdefault(csp_header, csp_policy) + + hsts = self._strict_transport_security_value() + if hsts: + response.headers.setdefault("Strict-Transport-Security", hsts) + + return response + + def _csp_policy_for_path(self, path: str) -> str: + if path.startswith("/docs") or path.startswith("/redoc"): + return self._csp_docs_policy or self._csp_policy + return self._csp_policy + + def _strict_transport_security_value(self) -> str: + if not self._hsts_enabled: + return "" + + directives = [f"max-age={self._hsts_max_age}"] + if self._hsts_include_subdomains: + directives.append("includeSubDomains") + if self._hsts_preload: + directives.append("preload") + return "; ".join(directives) + + +def parse_skip_paths(raw_value: str) -> tuple[str, ...]: + parts = [part.strip() for part in raw_value.split(",")] + return tuple(part for part in parts if part) diff --git a/build/lib/app/logging_config.py b/build/lib/app/logging_config.py new file mode 100644 index 0000000..22907be --- /dev/null +++ b/build/lib/app/logging_config.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +from datetime import datetime, timezone +import json +import logging +import sys +from typing import Any + +from app.logging_context import get_request_id + +DEFAULT_REDACT_FIELDS = { + "authorization", + "cookie", + "csrf_token", + "new_password", + "password", + "password_hash", + "session", + "session_secret_key", +} + +_BASE_RECORD = logging.makeLogRecord({}) +_STANDARD_RECORD_FIELDS = set(_BASE_RECORD.__dict__.keys()) | {"message", "asctime"} +_NOISY_RECORD_FIELDS = {"color_message"} + + +def parse_redact_fields(raw: str | None) -> set[str]: + fields = {field.strip().lower() for field in (raw or "").split(",") if field.strip()} + return DEFAULT_REDACT_FIELDS | fields + + +def configure_logging( + *, + level: str, + log_format: str, + redact_fields: set[str], + include_uvicorn_access: bool, +) -> None: + normalized_level = _normalize_level(level) + root_logger = logging.getLogger() + root_logger.handlers.clear() + root_logger.setLevel(normalized_level) + + handler = logging.StreamHandler(stream=sys.stdout) + handler.setLevel(normalized_level) + handler.addFilter(RequestContextFilter()) + + normalized_format = log_format.strip().lower() + if normalized_format == "json": + handler.setFormatter(JsonLogFormatter(redact_fields=redact_fields)) + else: + handler.setFormatter(ConsoleLogFormatter(redact_fields=redact_fields)) + + root_logger.addHandler(handler) + + # Route server/framework logs through our single root handler. + for logger_name in ("uvicorn", "uvicorn.error"): + framework_logger = logging.getLogger(logger_name) + framework_logger.handlers.clear() + framework_logger.propagate = True + framework_logger.setLevel(normalized_level) + + access_logger = logging.getLogger("uvicorn.access") + access_logger.handlers.clear() + access_logger.propagate = True + access_logger.setLevel(normalized_level if include_uvicorn_access else logging.WARNING) + + +class RequestContextFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + if not getattr(record, "request_id", None): + request_id = get_request_id() + if request_id: + record.request_id = request_id + return True + + +class JsonLogFormatter(logging.Formatter): + def __init__(self, *, redact_fields: set[str]) -> None: + super().__init__() + self._redact_fields = {field.lower() for field in redact_fields} + + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, Any] = { + "timestamp": _format_timestamp(record.created), + "level": record.levelname.lower(), + "logger": record.name, + "event": getattr(record, "event", record.getMessage()), + } + request_id = getattr(record, "request_id", None) + if request_id: + payload["request_id"] = request_id + payload.update(self._redact_mapping(_extra_fields(record))) + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + return json.dumps(payload, ensure_ascii=True, default=str) + + def _redact_mapping(self, value: dict[str, Any]) -> dict[str, Any]: + return {key: self._redact_value(key, item) for key, item in value.items()} + + def _redact_value(self, key: str, value: Any) -> Any: + if key.lower() in self._redact_fields: + return "[REDACTED]" + if isinstance(value, dict): + return {nested_key: self._redact_value(nested_key, nested_value) for nested_key, nested_value in value.items()} + if isinstance(value, (list, tuple)): + return [self._redact_value(key, item) for item in value] + return value + + +class ConsoleLogFormatter(logging.Formatter): + def __init__(self, *, redact_fields: set[str]) -> None: + super().__init__() + self._json_formatter = JsonLogFormatter(redact_fields=redact_fields) + + def format(self, record: logging.LogRecord) -> str: + payload = json.loads(self._json_formatter.format(record)) + timestamp = payload.get("timestamp", "") + level = _short_level(payload.get("level", "info")) + logger_name = str(payload.get("logger", "app")) + event = str(payload.get("event", "")) + + parts = [timestamp, level, logger_name, event] + + request_id = payload.pop("request_id", None) + method = payload.pop("method", None) + path = payload.pop("path", None) + status_code = payload.pop("status_code", None) + duration_ms = payload.pop("duration_ms", None) + + if request_id: + parts.append(f"rid={request_id}") + if method and path: + parts.append(f"{method} {path}") + if status_code is not None: + parts.append(str(status_code)) + if duration_ms is not None: + parts.append(f"{duration_ms}ms") + + for key in sorted(payload.keys()): + if key in {"timestamp", "level", "logger", "event", "exception"}: + continue + parts.append(f"{key}={payload[key]}") + + if "exception" in payload: + parts.append(f"exception={payload['exception']}") + + return " | ".join(str(part) for part in parts if part) + + +def _extra_fields(record: logging.LogRecord) -> dict[str, Any]: + extras: dict[str, Any] = {} + for key, value in record.__dict__.items(): + if key in _STANDARD_RECORD_FIELDS or key.startswith("_"): + continue + if key in _NOISY_RECORD_FIELDS: + continue + extras[key] = value + return extras + + +def _normalize_level(level: str) -> int: + normalized = level.strip().upper() + mapping = logging.getLevelNamesMapping() + if normalized not in mapping: + return logging.INFO + return mapping[normalized] + + +def _format_timestamp(created_ts: float) -> str: + dt = datetime.fromtimestamp(created_ts, tz=timezone.utc) + return dt.strftime("%Y-%m-%d %H:%M:%SZ") + + +def _short_level(level: str) -> str: + mapping = { + "debug": "DBG", + "info": "INF", + "warning": "WRN", + "error": "ERR", + "critical": "CRT", + } + return mapping.get(level.lower(), level[:3].upper()) diff --git a/build/lib/app/logging_context.py b/build/lib/app/logging_context.py new file mode 100644 index 0000000..ee578fb --- /dev/null +++ b/build/lib/app/logging_context.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from contextvars import ContextVar + + +_request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None) + + +def get_request_id() -> str | None: + return _request_id_ctx.get() + + +def set_request_id(value: str | None) -> None: + _request_id_ctx.set(value) + diff --git a/build/lib/app/main.py b/build/lib/app/main.py new file mode 100644 index 0000000..e628bd9 --- /dev/null +++ b/build/lib/app/main.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +from contextlib import asynccontextmanager +import logging +from pathlib import Path + +from fastapi import FastAPI, HTTPException +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles +from starlette.middleware.sessions import SessionMiddleware + +from app.api.errors import register_api_exception_handlers +from app.api.media import router as media_router +from app.api.router import router as api_router +from app.api.runtime_deps import get_ingestion_service, get_scholar_source +from app.db.session import check_database +from app.db.session import close_engine +from app.http.middleware import ( + RequestLoggingMiddleware, + SecurityHeadersMiddleware, + parse_skip_paths, +) +from app.logging_config import configure_logging, parse_redact_fields +from app.security.csrf import CSRFMiddleware +from app.services.domains.ingestion.scheduler import SchedulerService +from app.settings import settings + +logger = logging.getLogger(__name__) +BUILD_MARKER = "2026-02-19.phase2.direct-pdf-import-export-dashboard-sync" + +configure_logging( + level=settings.log_level, + log_format=settings.log_format, + redact_fields=parse_redact_fields(settings.log_redact_fields), + include_uvicorn_access=settings.log_uvicorn_access, +) + +scheduler_service = SchedulerService( + enabled=settings.scheduler_enabled, + tick_seconds=settings.scheduler_tick_seconds, + network_error_retries=settings.ingestion_network_error_retries, + retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, + max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, + page_size=settings.ingestion_page_size, + continuation_queue_enabled=settings.ingestion_continuation_queue_enabled, + continuation_base_delay_seconds=settings.ingestion_continuation_base_delay_seconds, + continuation_max_delay_seconds=settings.ingestion_continuation_max_delay_seconds, + continuation_max_attempts=settings.ingestion_continuation_max_attempts, + queue_batch_size=settings.scheduler_queue_batch_size, +) + + +def _log_startup_build_marker() -> None: + logger.info( + "app.startup_build_marker", + extra={ + "event": "app.startup_build_marker", + "build_marker": BUILD_MARKER, + "frontend_enabled": settings.frontend_enabled, + "scheduler_enabled": settings.scheduler_enabled, + "log_format": settings.log_format, + }, + ) + + +@asynccontextmanager +async def lifespan(_: FastAPI): + _log_startup_build_marker() + + from app.db.session import get_session_factory + from sqlalchemy import text + try: + session_factory = get_session_factory() + async with session_factory() as session: + await session.execute(text("UPDATE crawl_runs SET status = 'failed' WHERE status = 'running'")) + await session.commit() + logger.info("app.startup_orphaned_runs_cleaned", extra={"event": "app.startup_orphaned_runs_cleaned"}) + except Exception as e: + logger.error(f"Failed to clean orphaned runs at startup: {e}") + + await scheduler_service.start() + yield + await scheduler_service.stop() + await close_engine() + + +app = FastAPI(title=settings.app_name, lifespan=lifespan) +register_api_exception_handlers(app) +app.add_middleware(CSRFMiddleware) +app.add_middleware( + SessionMiddleware, + secret_key=settings.session_secret_key, + same_site="lax", + https_only=settings.session_cookie_secure, +) +app.add_middleware( + RequestLoggingMiddleware, + log_requests=settings.log_requests, + skip_paths=parse_skip_paths(settings.log_request_skip_paths), +) +app.add_middleware( + SecurityHeadersMiddleware, + enabled=settings.security_headers_enabled, + x_content_type_options=settings.security_x_content_type_options, + x_frame_options=settings.security_x_frame_options, + referrer_policy=settings.security_referrer_policy, + permissions_policy=settings.security_permissions_policy, + cross_origin_opener_policy=settings.security_cross_origin_opener_policy, + cross_origin_resource_policy=settings.security_cross_origin_resource_policy, + content_security_policy_enabled=settings.security_csp_enabled, + content_security_policy=settings.security_csp_policy, + content_security_policy_docs=settings.security_csp_docs_policy, + content_security_policy_report_only=settings.security_csp_report_only, + strict_transport_security_enabled=settings.security_strict_transport_security_enabled, + strict_transport_security_max_age=settings.security_strict_transport_security_max_age, + strict_transport_security_include_subdomains=( + settings.security_strict_transport_security_include_subdomains + ), + strict_transport_security_preload=settings.security_strict_transport_security_preload, +) +app.include_router(api_router) +app.include_router(media_router) + + +@app.get("/healthz") +async def healthz() -> dict[str, str]: + if await check_database(): + return {"status": "ok"} + raise HTTPException(status_code=500, detail="database unavailable") + + +def _configure_frontend_routes(application: FastAPI) -> None: + if not settings.frontend_enabled: + return + + dist_dir = Path(settings.frontend_dist_dir) + index_file = dist_dir / "index.html" + + if not index_file.is_file(): + return + + assets_dir = dist_dir / "assets" + if assets_dir.is_dir(): + application.mount("/assets", StaticFiles(directory=assets_dir), name="frontend-assets") + + @application.get("/", include_in_schema=False) + async def frontend_index() -> FileResponse: + return FileResponse(index_file) + + @application.get("/{full_path:path}", include_in_schema=False) + async def frontend_entry(full_path: str) -> FileResponse: + normalized = full_path.lstrip("/") + if normalized in {"healthz", "api"} or normalized.startswith("api/"): + raise HTTPException(status_code=404, detail="Not Found") + + candidate = (dist_dir / normalized).resolve() + try: + candidate.relative_to(dist_dir.resolve()) + except ValueError as error: + raise HTTPException(status_code=404, detail="Not Found") from error + + if candidate.is_file(): + return FileResponse(candidate) + + return FileResponse(index_file) + + +_configure_frontend_routes(app) diff --git a/build/lib/app/security/__init__.py b/build/lib/app/security/__init__.py new file mode 100644 index 0000000..4033692 --- /dev/null +++ b/build/lib/app/security/__init__.py @@ -0,0 +1,2 @@ +"""Security middleware and helpers for scholarr.""" + diff --git a/build/lib/app/security/csrf.py b/build/lib/app/security/csrf.py new file mode 100644 index 0000000..70eecac --- /dev/null +++ b/build/lib/app/security/csrf.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import logging +from secrets import compare_digest, token_urlsafe +from urllib.parse import parse_qs + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import JSONResponse, PlainTextResponse, Response +from starlette.types import Message + + +CSRF_SESSION_KEY = "csrf_token" +CSRF_FORM_FIELD = "csrf_token" +CSRF_HEADER_NAME = "x-csrf-token" +SAFE_METHODS = {"GET", "HEAD", "OPTIONS", "TRACE"} +logger = logging.getLogger(__name__) + + +def ensure_csrf_token(request: Request) -> str: + token = request.session.get(CSRF_SESSION_KEY) + if token is None: + token = token_urlsafe(32) + request.session[CSRF_SESSION_KEY] = token + return str(token) + + +class CSRFMiddleware(BaseHTTPMiddleware): + def __init__(self, app, *, exempt_paths: set[str] | None = None) -> None: + super().__init__(app) + self._exempt_paths = exempt_paths or set() + + async def dispatch(self, request: Request, call_next) -> Response: + if self._should_skip(request): + return await call_next(request) + + session_token = request.session.get(CSRF_SESSION_KEY) + if not session_token: + logger.warning( + "csrf.missing_session_token", + extra={ + "event": "csrf.missing_session_token", + "method": request.method, + "path": request.url.path, + }, + ) + return self._csrf_error_response( + request, + code="csrf_missing", + message="CSRF token missing.", + ) + + request_token = request.headers.get(CSRF_HEADER_NAME) + if request_token is None and self._is_form_payload(request): + body = await request.body() + request_token = self._token_from_form_body(request, body) + request._receive = self._build_receive(body) # type: ignore[attr-defined] + + if not request_token or not compare_digest(str(session_token), str(request_token)): + logger.warning( + "csrf.invalid_token", + extra={ + "event": "csrf.invalid_token", + "method": request.method, + "path": request.url.path, + }, + ) + return self._csrf_error_response( + request, + code="csrf_invalid", + message="CSRF token invalid.", + ) + + return await call_next(request) + + def _should_skip(self, request: Request) -> bool: + if request.method in SAFE_METHODS: + return True + path = request.url.path + return path in self._exempt_paths + + def _is_form_payload(self, request: Request) -> bool: + content_type = request.headers.get("content-type", "") + return content_type.startswith("application/x-www-form-urlencoded") or ( + content_type.startswith("multipart/form-data") + ) + + def _token_from_form_body(self, request: Request, body: bytes) -> str | None: + content_type = request.headers.get("content-type", "") + if not content_type.startswith("application/x-www-form-urlencoded"): + return None + parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True) + values = parsed.get(CSRF_FORM_FIELD) + if not values: + return None + return values[0] + + def _build_receive(self, body: bytes): + consumed = False + + async def receive() -> Message: + nonlocal consumed + if consumed: + return {"type": "http.request", "body": b"", "more_body": False} + consumed = True + return {"type": "http.request", "body": body, "more_body": False} + + return receive + + def _csrf_error_response( + self, + request: Request, + *, + code: str, + message: str, + ) -> Response: + if request.url.path.startswith("/api/"): + request_state = getattr(request, "state", None) + request_id = getattr(request_state, "request_id", None) if request_state else None + return JSONResponse( + { + "error": { + "code": code, + "message": message, + "details": None, + }, + "meta": { + "request_id": request_id, + }, + }, + status_code=403, + ) + return PlainTextResponse(message, status_code=403) diff --git a/build/lib/app/services/__init__.py b/build/lib/app/services/__init__.py new file mode 100644 index 0000000..167f869 --- /dev/null +++ b/build/lib/app/services/__init__.py @@ -0,0 +1,2 @@ +"""Service layer for scholarr application workflows.""" + diff --git a/build/lib/app/services/domains/__init__.py b/build/lib/app/services/domains/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/build/lib/app/services/domains/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/build/lib/app/services/domains/arxiv/application.py b/build/lib/app/services/domains/arxiv/application.py new file mode 100644 index 0000000..1581a7d --- /dev/null +++ b/build/lib/app/services/domains/arxiv/application.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import asyncio +import logging +from typing import TYPE_CHECKING +import xml.etree.ElementTree as ET + +import httpx + +from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id +from app.settings import settings + +if TYPE_CHECKING: + from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem + +logger = logging.getLogger(__name__) + + +def _build_arxiv_query(title: str, author_surname: str | None) -> str | None: + parts = [] + if title: + # arXiv api allows strict title searching using ti: + clean_title = title.replace('"', '').replace("'", "") + parts.append(f'ti:"{clean_title}"') + if author_surname: + parts.append(f'au:"{author_surname}"') + + if not parts: + return None + return " AND ".join(parts) + + +async def discover_arxiv_id_for_publication( + *, + item: PublicationListItem | UnreadPublicationItem, + request_email: str | None = None, + timeout_seconds: float = 3.0, +) -> str | None: + title = (item.title or "").strip() + if not title: + return None + + author_surname = None + if item.scholar_label: + tokens = [t for t in item.scholar_label.strip().split() if t] + if tokens: + author_surname = tokens[-1].lower() + + query = _build_arxiv_query(title, author_surname) + if not query: + return None + + url = "https://export.arxiv.org/api/query" + params = { + "search_query": query, + "start": 0, + "max_results": 3, + } + + headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{request_email or settings.crossref_api_mailto or 'unknown@example.com'})"} + + try: + async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True, headers=headers) as client: + response = await client.get(url, params=params) + response.raise_for_status() + + root = ET.fromstring(response.text) + namespace = {'atom': 'http://www.w3.org/2005/Atom'} + + for entry in root.findall('atom:entry', namespace): + id_elem = entry.find('atom:id', namespace) + if id_elem is not None and id_elem.text: + candidate = str(id_elem.text) + if '/abs/' in candidate: + candidate = candidate.split('/abs/')[-1] + normalized = normalize_arxiv_id(candidate) + if normalized: + logger.debug("arxiv.id_discovered", extra={"event": "arxiv.id_discovered", "arxiv_id": normalized}) + return normalized + + except Exception as exc: + logger.debug(f"Failed to query arXiv API: {exc}") + + return None diff --git a/build/lib/app/services/domains/crossref/__init__.py b/build/lib/app/services/domains/crossref/__init__.py new file mode 100644 index 0000000..8adb7ba --- /dev/null +++ b/build/lib/app/services/domains/crossref/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from app.services.domains.crossref.application import discover_doi_for_publication + +__all__ = ["discover_doi_for_publication"] diff --git a/build/lib/app/services/domains/crossref/application.py b/build/lib/app/services/domains/crossref/application.py new file mode 100644 index 0000000..f0e1311 --- /dev/null +++ b/build/lib/app/services/domains/crossref/application.py @@ -0,0 +1,382 @@ +from __future__ import annotations + +import asyncio +import logging +import re +import threading +import time +from typing import TYPE_CHECKING + +from crossref.restful import Etiquette, Works + +from app.services.domains.doi.normalize import normalize_doi +from app.settings import settings + +if TYPE_CHECKING: + from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem + +TOKEN_RE = re.compile(r"[a-z0-9]+") +NON_ALNUM_RE = re.compile(r"[^a-z0-9\\s]+") +STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis"} +_RATE_LOCK = threading.Lock() +_LAST_REQUEST_AT = 0.0 +logger = logging.getLogger(__name__) +STRICT_TITLE_MATCH_THRESHOLD = 0.75 +RELAXED_TITLE_MATCH_THRESHOLD = 0.85 + + +def _rate_limit_wait(min_interval_seconds: float) -> None: + global _LAST_REQUEST_AT + interval = max(float(min_interval_seconds), 0.0) + with _RATE_LOCK: + elapsed = time.monotonic() - _LAST_REQUEST_AT + remaining = interval - elapsed + if remaining > 0: + time.sleep(remaining) + _LAST_REQUEST_AT = time.monotonic() + + +def _normalized_tokens(value: str) -> list[str]: + lowered = value.lower().replace("’", "'").replace("“", "\"").replace("”", "\"") + lowered = NON_ALNUM_RE.sub(" ", lowered) + return [token for token in TOKEN_RE.findall(lowered) if len(token) >= 3] + + +def _normalized_query(value: str) -> str: + tokens = [token for token in _normalized_tokens(value) if token not in STOP_WORDS] + if len(tokens) < 3: + tokens = _normalized_tokens(value) + if len(tokens) < 3: + return "" + return " ".join(tokens[:12]).strip() + + +def _query_author(value: str) -> str | None: + tokens = [token for token in value.strip().split() if token] + if len(tokens) < 2: + return None + return " ".join(tokens[:2])[:64] + + +def _author_surname(value: str) -> str | None: + tokens = [token for token in value.strip().split() if token] + if not tokens: + return None + return NON_ALNUM_RE.sub("", tokens[-1].lower()) or None + + +def _query_filters(year: int | None) -> list[tuple[str, str] | None]: + if year is None: + return [None] + return [ + (f"{year - 1}-01-01", f"{year + 1}-12-31"), + (f"{year}-01-01", f"{year}-12-31"), + None, + ] + + +def _candidate_title(item: dict) -> str: + titles = item.get("title") + if isinstance(titles, list) and titles: + return str(titles[0] or "") + return str(item.get("title") or "") + + +def _title_match_score(source: str, candidate: str) -> float: + source_tokens = {token for token in _normalized_tokens(source) if len(token) >= 3} + candidate_tokens = {token for token in _normalized_tokens(candidate) if len(token) >= 3} + if not source_tokens or not candidate_tokens: + return 0.0 + return len(source_tokens & candidate_tokens) / float(len(source_tokens)) + + +def _candidate_year(item: dict) -> int | None: + issued = item.get("issued") + if not isinstance(issued, dict): + return None + date_parts = issued.get("date-parts") + if not isinstance(date_parts, list) or not date_parts: + return None + first = date_parts[0] + if not isinstance(first, list) or not first: + return None + try: + return int(first[0]) + except (TypeError, ValueError): + return None + + +def _candidate_author_match(item: dict, surname: str | None) -> bool: + if not surname: + return True + authors = item.get("author") + if not isinstance(authors, list): + return False + for author in authors: + if not isinstance(author, dict): + continue + family = NON_ALNUM_RE.sub("", str(author.get("family") or "").lower()) + if family and family == surname: + return True + return False + + +def _candidate_rank(*, title: str, year: int | None, item: dict) -> tuple[float, str | None]: + doi = normalize_doi(str(item.get("DOI") or "")) + if doi is None: + return 0.0, None + score = _title_match_score(title, _candidate_title(item)) + candidate_year = _candidate_year(item) + if year is not None and candidate_year is not None: + if abs(year - candidate_year) > 1: + return 0.0, None + score += 0.1 + return score, doi + + +def _year_delta(source_year: int | None, candidate_year: int | None) -> int | None: + if source_year is None or candidate_year is None: + return None + return abs(int(source_year) - int(candidate_year)) + + +def _candidate_rank_relaxed( + *, + title: str, + year: int | None, + item: dict, + author_surname: str | None, +) -> tuple[float, str | None]: + doi = normalize_doi(str(item.get("DOI") or "")) + if doi is None: + return 0.0, None + score = _title_match_score(title, _candidate_title(item)) + if score <= 0: + return 0.0, None + candidate_year = _candidate_year(item) + delta = _year_delta(year, candidate_year) + if delta is not None: + if delta <= 1: + score += 0.05 + elif delta <= 3: + score += 0.0 + elif delta <= 5: + score -= 0.03 + else: + score -= 0.08 + if _candidate_author_match(item, author_surname): + score += 0.03 + return score, doi + + +def _best_candidate_doi_strict( + *, + title: str, + year: int | None, + items: list[dict], + author_surname: str | None, +) -> str | None: + best_score = 0.0 + best_doi: str | None = None + best_year: int | None = None + for item in items: + if not isinstance(item, dict): + continue + if not _candidate_author_match(item, author_surname): + continue + score, doi = _candidate_rank(title=title, year=year, item=item) + candidate_year = _candidate_year(item) + if doi is None or score < STRICT_TITLE_MATCH_THRESHOLD: + continue + if score > best_score: + best_score = score + best_doi = doi + best_year = candidate_year + continue + if abs(score - best_score) > 0.02: + continue + if best_year is None or candidate_year is None: + continue + if candidate_year < best_year: + best_doi = doi + best_year = candidate_year + return best_doi + + +def _best_candidate_doi_relaxed( + *, + title: str, + year: int | None, + items: list[dict], + author_surname: str | None, +) -> str | None: + best_score = 0.0 + best_doi: str | None = None + best_author_match = False + best_delta: int | None = None + best_year: int | None = None + for item in items: + if not isinstance(item, dict): + continue + score, doi = _candidate_rank_relaxed( + title=title, + year=year, + item=item, + author_surname=author_surname, + ) + if doi is None or score < RELAXED_TITLE_MATCH_THRESHOLD: + continue + candidate_year = _candidate_year(item) + candidate_author_match = _candidate_author_match(item, author_surname) + candidate_delta = _year_delta(year, candidate_year) + if score > best_score: + best_score = score + best_doi = doi + best_author_match = candidate_author_match + best_delta = candidate_delta + best_year = candidate_year + continue + if abs(score - best_score) > 0.02: + continue + if candidate_author_match and not best_author_match: + best_doi = doi + best_author_match = True + best_delta = candidate_delta + best_year = candidate_year + continue + if best_delta is None and candidate_delta is not None: + best_doi = doi + best_author_match = candidate_author_match + best_delta = candidate_delta + best_year = candidate_year + continue + if best_delta is not None and candidate_delta is not None and candidate_delta < best_delta: + best_doi = doi + best_author_match = candidate_author_match + best_delta = candidate_delta + best_year = candidate_year + continue + if best_year is None or candidate_year is None: + continue + if candidate_year < best_year: + best_doi = doi + best_author_match = candidate_author_match + best_delta = candidate_delta + best_year = candidate_year + return best_doi + + +def _best_candidate_doi( + *, + title: str, + year: int | None, + items: list[dict], + author_surname: str | None, +) -> str | None: + strict_match = _best_candidate_doi_strict( + title=title, + year=year, + items=items, + author_surname=author_surname, + ) + if strict_match: + return strict_match + return _best_candidate_doi_relaxed( + title=title, + year=year, + items=items, + author_surname=author_surname, + ) + + +def _works_client(email: str | None) -> Works: + if email: + etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email) + return Works(etiquette=etiquette) + return Works() + + +def _fetch_items_sync( + *, + query: str, + author: str | None, + date_range: tuple[str, str] | None, + max_rows: int, + email: str | None, + min_interval_seconds: float, +) -> list[dict]: + _rate_limit_wait(min_interval_seconds) + works = _works_client(email) + params = {"bibliographic": query} + if author: + params["author"] = author + request = works.query(**params) + if date_range is not None: + from_date, until_date = date_range + request = request.filter(from_pub_date=from_date, until_pub_date=until_date) + request = request.select(["DOI", "title", "issued", "score", "author"]) + items: list[dict] = [] + for entry in request: + if isinstance(entry, dict): + items.append(entry) + if len(items) >= max(max_rows, 1): + break + return items + + +async def _fetch_items( + *, + query: str, + author: str | None, + date_range: tuple[str, str] | None, + max_rows: int, + email: str | None, +) -> list[dict]: + timeout = max(float(settings.crossref_timeout_seconds), 0.5) + try: + return await asyncio.wait_for( + asyncio.to_thread( + _fetch_items_sync, + query=query, + author=author, + date_range=date_range, + max_rows=max_rows, + email=email, + min_interval_seconds=settings.crossref_min_interval_seconds, + ), + timeout=timeout, + ) + except Exception: + return [] + + +async def discover_doi_for_publication( + *, + item: PublicationListItem | UnreadPublicationItem, + max_rows: int = 10, + email: str | None = None, +) -> str | None: + title = (item.title or "").strip() + query = _normalized_query(title) + if not query: + return None + author = _query_author(item.scholar_label) + author_surname = _author_surname(item.scholar_label) + for date_range in _query_filters(item.year): + items = await _fetch_items( + query=query, + author=author, + date_range=date_range, + max_rows=max_rows, + email=email, + ) + doi = _best_candidate_doi( + title=title, + year=item.year, + items=items, + author_surname=author_surname, + ) + if doi: + logger.debug("crossref.doi_discovered", extra={"event": "crossref.doi_discovered"}) + return doi + return None diff --git a/build/lib/app/services/domains/dbops/__init__.py b/build/lib/app/services/domains/dbops/__init__.py new file mode 100644 index 0000000..f5b02e4 --- /dev/null +++ b/build/lib/app/services/domains/dbops/__init__.py @@ -0,0 +1,5 @@ +from app.services.domains.dbops.application import run_publication_link_repair +from app.services.domains.dbops.integrity import collect_integrity_report +from app.services.domains.dbops.query import list_repair_jobs + +__all__ = ["collect_integrity_report", "list_repair_jobs", "run_publication_link_repair"] diff --git a/build/lib/app/services/domains/dbops/application.py b/build/lib/app/services/domains/dbops/application.py new file mode 100644 index 0000000..e7ace77 --- /dev/null +++ b/build/lib/app/services/domains/dbops/application.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy import delete, exists, func, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication + + +REPAIR_STATUS_PLANNED = "planned" +REPAIR_STATUS_RUNNING = "running" +REPAIR_STATUS_COMPLETED = "completed" +REPAIR_STATUS_FAILED = "failed" +SCOPE_MODE_SINGLE_USER = "single_user" +SCOPE_MODE_ALL_USERS = "all_users" + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def _normalize_scope_mode(scope_mode: str) -> str: + normalized = scope_mode.strip().lower() + if normalized in {SCOPE_MODE_SINGLE_USER, SCOPE_MODE_ALL_USERS}: + return normalized + raise ValueError("Unknown scope mode.") + + +def _scope_user_id(*, scope_mode: str, user_id: int | None) -> int | None: + if scope_mode == SCOPE_MODE_SINGLE_USER: + if user_id is None: + raise ValueError("user_id is required when scope_mode=single_user.") + return int(user_id) + if user_id is not None: + raise ValueError("user_id must be omitted when scope_mode=all_users.") + return None + + +def _scope_payload( + *, + scope_mode: str, + user_id: int | None, + target_scholar_profile_ids: list[int], + orphan_gc: bool, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "scope_mode": scope_mode, + "scholar_profile_ids": [int(value) for value in target_scholar_profile_ids], + "gc_orphan_publications": bool(orphan_gc), + } + if user_id is not None: + payload["user_id"] = int(user_id) + return payload + + +async def _target_scholar_profile_ids( + db_session: AsyncSession, + *, + scope_mode: str, + user_id: int | None, + scholar_profile_ids: list[int] | None, +) -> list[int]: + stmt = select(ScholarProfile.id) + if scope_mode == SCOPE_MODE_SINGLE_USER: + stmt = stmt.where(ScholarProfile.user_id == user_id) + if scholar_profile_ids: + normalized_ids = [int(value) for value in scholar_profile_ids] + stmt = stmt.where(ScholarProfile.id.in_(normalized_ids)) + result = await db_session.execute(stmt.order_by(ScholarProfile.id.asc())) + ids = [int(row[0]) for row in result.all()] + if not ids: + raise ValueError("No target scholar profiles found for the requested scope.") + return ids + + +async def _count_scope( + db_session: AsyncSession, + *, + user_id: int | None, + target_scholar_profile_ids: list[int], +) -> dict[str, int]: + links_result = await db_session.execute( + select(func.count()) + .select_from(ScholarPublication) + .where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids)) + ) + queue_stmt = ( + select(func.count()) + .select_from(IngestionQueueItem) + .where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids)) + ) + if user_id is not None: + queue_stmt = queue_stmt.where(IngestionQueueItem.user_id == user_id) + queue_result = await db_session.execute(queue_stmt) + return { + "target_scholar_count": len(target_scholar_profile_ids), + "links_in_scope": int(links_result.scalar_one() or 0), + "queue_items_in_scope": int(queue_result.scalar_one() or 0), + } + + +async def _count_orphan_publications(db_session: AsyncSession) -> int: + stmt = ( + select(func.count()) + .select_from(Publication) + .where( + ~exists( + select(1).where(ScholarPublication.publication_id == Publication.id) + ) + ) + ) + result = await db_session.execute(stmt) + return int(result.scalar_one() or 0) + + +async def _create_job( + db_session: AsyncSession, + *, + requested_by: str | None, + scope: dict[str, Any], + dry_run: bool, +) -> DataRepairJob: + job = DataRepairJob( + job_name="repair_publication_links", + requested_by=(requested_by or "").strip() or None, + scope=scope, + dry_run=dry_run, + status=REPAIR_STATUS_PLANNED, + summary={}, + ) + db_session.add(job) + await db_session.flush() + return job + + +def _job_summary( + *, + counts: dict[str, int], + dry_run: bool, + links_deleted: int, + queue_items_deleted: int, + scholars_reset: int, + orphan_publications_before: int, + orphan_publications_deleted: int, +) -> dict[str, Any]: + return { + **counts, + "dry_run": bool(dry_run), + "links_deleted": int(links_deleted), + "queue_items_deleted": int(queue_items_deleted), + "scholars_reset": int(scholars_reset), + "orphan_publications_before": int(orphan_publications_before), + "orphan_publications_deleted": int(orphan_publications_deleted), + } + + +async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int: + result = await db_session.execute( + delete(ScholarPublication).where( + ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids) + ) + ) + return int(result.rowcount or 0) + + +async def _delete_queue_for_targets( + db_session: AsyncSession, + *, + user_id: int | None, + target_scholar_profile_ids: list[int], +) -> int: + stmt = delete(IngestionQueueItem).where( + IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids) + ) + if user_id is not None: + stmt = stmt.where(IngestionQueueItem.user_id == user_id) + result = await db_session.execute(stmt) + return int(result.rowcount or 0) + + +async def _reset_scholar_tracking_state( + db_session: AsyncSession, + *, + user_id: int | None, + target_scholar_profile_ids: list[int], +) -> int: + stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids)) + if user_id is not None: + stmt = stmt.where(ScholarProfile.user_id == user_id) + result = await db_session.execute( + stmt.values( + baseline_completed=False, + last_initial_page_fingerprint_sha256=None, + last_initial_page_checked_at=None, + last_run_dt=None, + last_run_status=None, + ) + ) + return int(result.rowcount or 0) + + +async def _delete_orphan_publications(db_session: AsyncSession) -> int: + result = await db_session.execute( + delete(Publication).where( + ~exists(select(1).where(ScholarPublication.publication_id == Publication.id)) + ) + ) + return int(result.rowcount or 0) + + +async def _mutation_counts( + db_session: AsyncSession, + *, + user_id: int | None, + target_ids: list[int], + dry_run: bool, + gc_orphan_publications: bool, +) -> tuple[int, int, int, int]: + if dry_run: + return 0, 0, 0, 0 + links_deleted = await _delete_links_for_targets( + db_session, + target_scholar_profile_ids=target_ids, + ) + queue_deleted = await _delete_queue_for_targets( + db_session, + user_id=user_id, + target_scholar_profile_ids=target_ids, + ) + scholars_reset = await _reset_scholar_tracking_state( + db_session, + user_id=user_id, + target_scholar_profile_ids=target_ids, + ) + orphan_deleted = 0 + if gc_orphan_publications: + orphan_deleted = await _delete_orphan_publications(db_session) + return links_deleted, queue_deleted, scholars_reset, orphan_deleted + + +def _result_payload(*, job: DataRepairJob, scope: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]: + return { + "job_id": int(job.id), + "status": job.status, + "scope": scope, + "summary": summary, + } + + +async def _complete_job( + db_session: AsyncSession, + *, + job: DataRepairJob, + summary: dict[str, Any], + scope: dict[str, Any], +) -> dict[str, Any]: + job.summary = summary + job.status = REPAIR_STATUS_COMPLETED + job.finished_at = _utcnow() + await db_session.commit() + return _result_payload(job=job, scope=scope, summary=summary) + + +async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None: + await db_session.rollback() + job.status = REPAIR_STATUS_FAILED + job.error_text = str(error) + job.finished_at = _utcnow() + db_session.add(job) + await db_session.commit() + + +async def _prepare_repair_job( + db_session: AsyncSession, + *, + scope_mode: str, + user_id: int | None, + scholar_profile_ids: list[int] | None, + dry_run: bool, + gc_orphan_publications: bool, + requested_by: str | None, +) -> tuple[int | None, list[int], dict[str, Any], DataRepairJob]: + normalized_scope = _normalize_scope_mode(scope_mode) + scope_user_id = _scope_user_id(scope_mode=normalized_scope, user_id=user_id) + target_ids = await _target_scholar_profile_ids( + db_session, + scope_mode=normalized_scope, + user_id=scope_user_id, + scholar_profile_ids=scholar_profile_ids, + ) + scope = _scope_payload( + scope_mode=normalized_scope, + user_id=scope_user_id, + target_scholar_profile_ids=target_ids, + orphan_gc=gc_orphan_publications, + ) + job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run) + job.status = REPAIR_STATUS_RUNNING + job.started_at = _utcnow() + return scope_user_id, target_ids, scope, job + + +async def _build_repair_summary( + db_session: AsyncSession, + *, + scope_user_id: int | None, + target_ids: list[int], + dry_run: bool, + gc_orphan_publications: bool, +) -> dict[str, Any]: + counts = await _count_scope(db_session, user_id=scope_user_id, target_scholar_profile_ids=target_ids) + orphan_before = await _count_orphan_publications(db_session) + links_deleted, queue_deleted, scholars_reset, orphan_deleted = await _mutation_counts( + db_session, + user_id=scope_user_id, + target_ids=target_ids, + dry_run=dry_run, + gc_orphan_publications=gc_orphan_publications, + ) + return _job_summary( + counts=counts, + dry_run=dry_run, + links_deleted=links_deleted, + queue_items_deleted=queue_deleted, + scholars_reset=scholars_reset, + orphan_publications_before=orphan_before, + orphan_publications_deleted=orphan_deleted, + ) + + +async def run_publication_link_repair( + db_session: AsyncSession, + *, + scope_mode: str = SCOPE_MODE_SINGLE_USER, + user_id: int | None = None, + scholar_profile_ids: list[int] | None = None, + dry_run: bool = True, + gc_orphan_publications: bool = False, + requested_by: str | None = None, +) -> dict[str, Any]: + scope_user_id, target_ids, scope, job = await _prepare_repair_job( + db_session, + scope_mode=scope_mode, + user_id=user_id, + scholar_profile_ids=scholar_profile_ids, + dry_run=dry_run, + gc_orphan_publications=gc_orphan_publications, + requested_by=requested_by, + ) + + try: + summary = await _build_repair_summary( + db_session, + scope_user_id=scope_user_id, + target_ids=target_ids, + dry_run=dry_run, + gc_orphan_publications=gc_orphan_publications, + ) + return await _complete_job(db_session, job=job, summary=summary, scope=scope) + except Exception as exc: + await _fail_job(db_session, job=job, error=exc) + raise diff --git a/build/lib/app/services/domains/dbops/integrity.py b/build/lib/app/services/domains/dbops/integrity.py new file mode 100644 index 0000000..4d9c43c --- /dev/null +++ b/build/lib/app/services/domains/dbops/integrity.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy import func, select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Publication + +INTEGRITY_CHECK_DEFS = ( + ( + "legacy_cluster_id_format", + "warning", + "Publications with non-namespaced cluster IDs.", + ), + ( + "negative_citation_count", + "failure", + "Publications with negative citation counts.", + ), + ( + "orphan_publications", + "warning", + "Publications with no scholar links.", + ), + ( + "orphan_scholar_publication_links", + "failure", + "Link rows missing parent scholar/publication.", + ), + ( + "duplicate_fingerprint_keys", + "failure", + "Duplicate publication fingerprint keys.", + ), + ( + "duplicate_cluster_ids", + "failure", + "Duplicate non-null publication cluster IDs.", + ), + ( + "missing_pdf_url", + "metric", + "Publications without a resolved PDF URL.", + ), +) + + +async def _legacy_cluster_id_count(db_session: AsyncSession) -> int: + stmt = ( + select(func.count()) + .select_from(Publication) + .where(Publication.cluster_id.is_not(None)) + .where(~Publication.cluster_id.like("cfv:%")) + .where(~Publication.cluster_id.like("cluster:%")) + ) + result = await db_session.execute(stmt) + return int(result.scalar_one() or 0) + + +async def _negative_citation_count(db_session: AsyncSession) -> int: + result = await db_session.execute( + select(func.count()).select_from(Publication).where(Publication.citation_count < 0) + ) + return int(result.scalar_one() or 0) + + +async def _missing_pdf_url_count(db_session: AsyncSession) -> int: + result = await db_session.execute( + select(func.count()).select_from(Publication).where(Publication.pdf_url.is_(None)) + ) + return int(result.scalar_one() or 0) + + +async def _count_from_sql(db_session: AsyncSession, *, sql: str) -> int: + result = await db_session.execute(text(sql)) + return int(result.scalar_one() or 0) + + +def _issues_for_severity(*, checks: list[dict[str, Any]], severity: str) -> list[str]: + return [row["name"] for row in checks if row["severity"] == severity and row["count"] > 0] + + +def _check_row(*, name: str, count: int, severity: str, message: str) -> dict[str, Any]: + return { + "name": name, + "count": int(count), + "severity": severity, + "message": message, + } + + +async def _collect_counts(db_session: AsyncSession) -> dict[str, int]: + orphan_publications = await _count_from_sql( + db_session, + sql=( + "SELECT count(*) FROM publications p " + "LEFT JOIN scholar_publications sp ON sp.publication_id = p.id " + "WHERE sp.publication_id IS NULL" + ), + ) + orphan_links = await _count_from_sql( + db_session, + sql=( + "SELECT count(*) FROM scholar_publications sp " + "LEFT JOIN publications p ON p.id = sp.publication_id " + "LEFT JOIN scholar_profiles s ON s.id = sp.scholar_profile_id " + "WHERE p.id IS NULL OR s.id IS NULL" + ), + ) + duplicate_fingerprints = await _count_from_sql( + db_session, + sql=( + "SELECT count(*) FROM (" + "SELECT fingerprint_sha256 FROM publications " + "GROUP BY fingerprint_sha256 HAVING count(*) > 1" + ") dup" + ), + ) + duplicate_cluster_ids = await _count_from_sql( + db_session, + sql=( + "SELECT count(*) FROM (" + "SELECT cluster_id FROM publications " + "WHERE cluster_id IS NOT NULL " + "GROUP BY cluster_id HAVING count(*) > 1" + ") dup" + ), + ) + return { + "legacy_cluster_id_format": await _legacy_cluster_id_count(db_session), + "negative_citation_count": await _negative_citation_count(db_session), + "orphan_publications": orphan_publications, + "orphan_scholar_publication_links": orphan_links, + "duplicate_fingerprint_keys": duplicate_fingerprints, + "duplicate_cluster_ids": duplicate_cluster_ids, + "missing_pdf_url": await _missing_pdf_url_count(db_session), + } + + +def _build_checks(*, counts: dict[str, int]) -> list[dict[str, Any]]: + return [ + _check_row( + name=name, + count=counts[name], + severity=severity, + message=message, + ) + for name, severity, message in INTEGRITY_CHECK_DEFS + ] + + +def _status_from_issues(*, failures: list[str], warnings: list[str]) -> str: + status = "ok" + if failures: + status = "failed" + elif warnings: + status = "warning" + return status + + +async def collect_integrity_report(db_session: AsyncSession) -> dict[str, Any]: + counts = await _collect_counts(db_session) + checks = _build_checks(counts=counts) + failures = _issues_for_severity(checks=checks, severity="failure") + warnings = _issues_for_severity(checks=checks, severity="warning") + + return { + "status": _status_from_issues(failures=failures, warnings=warnings), + "checked_at": datetime.now(timezone.utc).isoformat(), + "failures": failures, + "warnings": warnings, + "checks": checks, + } diff --git a/build/lib/app/services/domains/dbops/query.py b/build/lib/app/services/domains/dbops/query.py new file mode 100644 index 0000000..2e95e31 --- /dev/null +++ b/build/lib/app/services/domains/dbops/query.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import DataRepairJob + + +def _bounded_limit(limit: int) -> int: + return max(1, min(int(limit), 200)) + + +async def list_repair_jobs( + db_session: AsyncSession, + *, + limit: int = 50, +) -> list[DataRepairJob]: + bounded = _bounded_limit(limit) + result = await db_session.execute( + select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded) + ) + return list(result.scalars()) diff --git a/build/lib/app/services/domains/doi/normalize.py b/build/lib/app/services/domains/doi/normalize.py new file mode 100644 index 0000000..4719cf3 --- /dev/null +++ b/build/lib/app/services/domains/doi/normalize.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import re +from urllib.parse import unquote + +DOI_RE = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I) + + +def normalize_doi(value: str | None) -> str | None: + if not value: + return None + match = DOI_RE.search(unquote(value)) + if not match: + return None + return match.group(0).rstrip(" .;,)").lower() + + +def first_doi_from_texts(*values: str | None) -> str | None: + for value in values: + doi = normalize_doi(value) + if doi: + return doi + return None diff --git a/build/lib/app/services/domains/ingestion/__init__.py b/build/lib/app/services/domains/ingestion/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/build/lib/app/services/domains/ingestion/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/build/lib/app/services/domains/ingestion/application.py b/build/lib/app/services/domains/ingestion/application.py new file mode 100644 index 0000000..63656b4 --- /dev/null +++ b/build/lib/app/services/domains/ingestion/application.py @@ -0,0 +1,2413 @@ +from __future__ import annotations + +import asyncio +from datetime import datetime, timezone +import hashlib +import logging +from typing import Any + +from sqlalchemy import select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + CrawlRun, + Publication, + RunStatus, + RunTriggerType, + ScholarProfile, + ScholarPublication, +) +from app.services.domains.ingestion.constants import ( + FAILED_STATES, + FAILURE_BUCKET_BLOCKED, + FAILURE_BUCKET_INGESTION, + FAILURE_BUCKET_LAYOUT, + FAILURE_BUCKET_NETWORK, + FAILURE_BUCKET_OTHER, + RESUMABLE_PARTIAL_REASON_PREFIXES, + RESUMABLE_PARTIAL_REASONS, + RUN_LOCK_NAMESPACE, +) +from app.services.domains.doi.normalize import first_doi_from_texts +from app.services.domains.publication_identifiers import application as identifier_service +from app.services.domains.ingestion.fingerprints import ( + _build_body_excerpt, + _dedupe_publication_candidates, + _next_cstart_value, + build_initial_page_fingerprint, + build_publication_fingerprint, + build_publication_url, + normalize_title, +) +from app.services.domains.ingestion import queue as queue_service +from app.services.domains.ingestion import safety as run_safety_service +from app.services.domains.ingestion.types import ( + PagedLoopState, + PagedParseResult, + RunAlertSummary, + RunAlreadyInProgressError, + RunBlockedBySafetyPolicyError, + RunExecutionSummary, + RunFailureSummary, + RunProgress, + ScholarProcessingOutcome, +) +from app.services.domains.settings import application as user_settings_service +from app.services.domains.runs.events import run_events +from app.services.domains.scholar.parser import ( + ParseState, + ParsedProfilePage, + PublicationCandidate, + ScholarParserError, + parse_profile_page, +) +from app.services.domains.scholar.source import FetchResult, ScholarSource +from app.settings import settings + +logger = logging.getLogger(__name__) + + +def _int_or_default(value: Any, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _classify_failure_bucket(*, state: str, state_reason: str) -> str: + reason = state_reason.strip().lower() + normalized_state = state.strip().lower() + + if normalized_state == ParseState.BLOCKED_OR_CAPTCHA.value or reason.startswith("blocked_"): + return FAILURE_BUCKET_BLOCKED + if normalized_state == ParseState.NETWORK_ERROR.value or reason.startswith("network_"): + return FAILURE_BUCKET_NETWORK + if normalized_state == ParseState.LAYOUT_CHANGED.value: + return FAILURE_BUCKET_LAYOUT + if normalized_state == "ingestion_error": + return FAILURE_BUCKET_INGESTION + return FAILURE_BUCKET_OTHER + + +class ScholarIngestionService: + def __init__(self, *, source: ScholarSource) -> None: + self._source = source + + @staticmethod + def _effective_request_delay_seconds(value: int) -> int: + policy_minimum = user_settings_service.resolve_request_delay_minimum( + settings.ingestion_min_request_delay_seconds + ) + return max(policy_minimum, _int_or_default(value, policy_minimum)) + + @staticmethod + def _log_request_delay_coercion( + *, + user_id: int, + requested_request_delay_seconds: int, + effective_request_delay_seconds: int, + ) -> None: + logger.warning( + "ingestion.request_delay_coerced_to_policy_floor", + extra={ + "event": "ingestion.request_delay_coerced_to_policy_floor", + "user_id": user_id, + "requested_request_delay_seconds": requested_request_delay_seconds, + "effective_request_delay_seconds": effective_request_delay_seconds, + "policy_minimum_request_delay_seconds": user_settings_service.resolve_request_delay_minimum( + settings.ingestion_min_request_delay_seconds + ), + "metric_name": "ingestion_request_delay_coerced_total", + "metric_value": 1, + }, + ) + + async def _load_user_settings_for_run( + self, + db_session: AsyncSession, + *, + user_id: int, + trigger_type: RunTriggerType, + ): + user_settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=user_id, + ) + await self._enforce_safety_gate( + db_session, + user_settings=user_settings, + user_id=user_id, + trigger_type=trigger_type, + ) + return user_settings + + async def _enforce_safety_gate( + self, + db_session: AsyncSession, + *, + user_settings, + user_id: int, + trigger_type: RunTriggerType, + ) -> None: + now_utc = datetime.now(timezone.utc) + previous = run_safety_service.get_safety_event_context(user_settings, now_utc=now_utc) + if run_safety_service.clear_expired_cooldown(user_settings, now_utc=now_utc): + await db_session.commit() + await db_session.refresh(user_settings) + logger.info( + "ingestion.safety_cooldown_cleared", + extra={ + "event": "ingestion.safety_cooldown_cleared", + "user_id": user_id, + "reason": previous.get("cooldown_reason"), + "cooldown_until": previous.get("cooldown_until"), + "metric_name": "ingestion_safety_cooldown_cleared_total", + "metric_value": 1, + }, + ) + now_utc = datetime.now(timezone.utc) + if run_safety_service.is_cooldown_active(user_settings, now_utc=now_utc): + await self._raise_safety_blocked_start( + db_session, + user_settings=user_settings, + user_id=user_id, + trigger_type=trigger_type, + now_utc=now_utc, + ) + + async def _raise_safety_blocked_start( + self, + db_session: AsyncSession, + *, + user_settings, + user_id: int, + trigger_type: RunTriggerType, + now_utc: datetime, + ) -> None: + safety_state = run_safety_service.register_cooldown_blocked_start( + user_settings, + now_utc=now_utc, + ) + await db_session.commit() + logger.warning( + "ingestion.safety_policy_blocked_run_start", + extra={ + "event": "ingestion.safety_policy_blocked_run_start", + "user_id": user_id, + "trigger_type": trigger_type.value, + "reason": safety_state.get("cooldown_reason"), + "cooldown_until": safety_state.get("cooldown_until"), + "cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"), + "blocked_start_count": ((safety_state.get("counters") or {}).get("blocked_start_count")), + "metric_name": "ingestion_safety_run_start_blocked_total", + "metric_value": 1, + }, + ) + raise RunBlockedBySafetyPolicyError( + code="scrape_cooldown_active", + message="Scrape safety cooldown is active; run start is temporarily blocked.", + safety_state=safety_state, + ) + + @staticmethod + def _normalize_run_targets( + *, + scholar_profile_ids: set[int] | None, + start_cstart_by_scholar_id: dict[int, int] | None, + ) -> tuple[set[int] | None, dict[int, int]]: + filtered_scholar_ids = ( + {int(value) for value in scholar_profile_ids} + if scholar_profile_ids is not None + else None + ) + start_cstart_map = { + int(key): max(0, int(value)) + for key, value in (start_cstart_by_scholar_id or {}).items() + } + return filtered_scholar_ids, start_cstart_map + + async def _load_target_scholars( + self, + db_session: AsyncSession, + *, + user_id: int, + filtered_scholar_ids: set[int] | None, + ) -> list[ScholarProfile]: + scholars_stmt = ( + select(ScholarProfile) + .where(ScholarProfile.user_id == user_id, ScholarProfile.is_enabled.is_(True)) + .order_by(ScholarProfile.created_at.asc(), ScholarProfile.id.asc()) + ) + if filtered_scholar_ids is not None: + scholars_stmt = scholars_stmt.where(ScholarProfile.id.in_(filtered_scholar_ids)) + scholars_result = await db_session.execute(scholars_stmt) + scholars = list(scholars_result.scalars().all()) + await self._clear_missing_filtered_jobs( + db_session, + user_id=user_id, + filtered_scholar_ids=filtered_scholar_ids, + scholars=scholars, + ) + return scholars + + async def _clear_missing_filtered_jobs( + self, + db_session: AsyncSession, + *, + user_id: int, + filtered_scholar_ids: set[int] | None, + scholars: list[ScholarProfile], + ) -> None: + if filtered_scholar_ids is None: + return + found_ids = {int(scholar.id) for scholar in scholars} + missing_ids = filtered_scholar_ids - found_ids + for scholar_profile_id in missing_ids: + await queue_service.clear_job_for_scholar( + db_session, + user_id=user_id, + scholar_profile_id=scholar_profile_id, + ) + + @staticmethod + def _create_running_run( + *, + user_id: int, + trigger_type: RunTriggerType, + scholar_count: int, + idempotency_key: str | None, + ) -> CrawlRun: + return CrawlRun( + user_id=user_id, + trigger_type=trigger_type, + status=RunStatus.RUNNING, + scholar_count=scholar_count, + new_pub_count=0, + idempotency_key=idempotency_key, + error_log={}, + ) + + @staticmethod + def _log_run_started( + *, + user_id: int, + trigger_type: RunTriggerType, + scholar_count: int, + filtered: bool, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + idempotency_key: str | None, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> None: + logger.info( + "ingestion.run_started", + extra={ + "event": "ingestion.run_started", + "user_id": user_id, + "trigger_type": trigger_type.value, + "scholar_count": scholar_count, + "is_filtered_run": filtered, + "request_delay_seconds": request_delay_seconds, + "network_error_retries": network_error_retries, + "retry_backoff_seconds": retry_backoff_seconds, + "max_pages_per_scholar": max_pages_per_scholar, + "page_size": page_size, + "idempotency_key": idempotency_key, + "alert_blocked_failure_threshold": alert_blocked_failure_threshold, + "alert_network_failure_threshold": alert_network_failure_threshold, + "alert_retry_scheduled_threshold": alert_retry_scheduled_threshold, + }, + ) + + @staticmethod + async def _wait_between_scholars(*, index: int, request_delay_seconds: int) -> None: + if index <= 0 or request_delay_seconds <= 0: + return + await asyncio.sleep(float(request_delay_seconds)) + + @staticmethod + def _assert_valid_paged_parse_result( + *, + scholar_id: str, + paged_parse_result: PagedParseResult, + ) -> None: + parsed_page = paged_parse_result.parsed_page + if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS}: + if any(code.startswith("layout_") for code in parsed_page.warnings): + raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.") + for publication in paged_parse_result.publications: + if not publication.title.strip(): + raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.") + if publication.citation_count is not None and int(publication.citation_count) < 0: + raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.") + + @staticmethod + def _apply_first_page_profile_metadata( + *, + scholar: ScholarProfile, + paged_parse_result: PagedParseResult, + run_dt: datetime, + ) -> None: + first_page = paged_parse_result.first_page_parsed_page + if first_page.profile_name and not (scholar.display_name or "").strip(): + scholar.display_name = first_page.profile_name + if first_page.profile_image_url: + scholar.profile_image_url = first_page.profile_image_url + scholar.last_initial_page_checked_at = run_dt + + @staticmethod + def _log_scholar_parsed( + *, + user_id: int, + run_id: int, + scholar: ScholarProfile, + paged_parse_result: PagedParseResult, + ) -> None: + parsed_page = paged_parse_result.parsed_page + logger.info( + "ingestion.scholar_parsed", + extra={ + "event": "ingestion.scholar_parsed", + "user_id": user_id, + "crawl_run_id": run_id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": parsed_page.state.value, + "publication_count": len(paged_parse_result.publications), + "has_show_more_button": parsed_page.has_show_more_button, + "pages_fetched": paged_parse_result.pages_fetched, + "pages_attempted": paged_parse_result.pages_attempted, + "has_more_remaining": paged_parse_result.has_more_remaining, + "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, + "warning_count": len(parsed_page.warnings), + "skipped_no_change": paged_parse_result.skipped_no_change, + }, + ) + + @staticmethod + def _build_result_entry( + *, + scholar: ScholarProfile, + start_cstart: int, + paged_parse_result: PagedParseResult, + ) -> dict[str, Any]: + parsed_page = paged_parse_result.parsed_page + return { + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "outcome": "failed", + "attempt_count": len(paged_parse_result.attempt_log), + "publication_count": len(paged_parse_result.publications), + "start_cstart": start_cstart, + "articles_range": parsed_page.articles_range, + "warnings": parsed_page.warnings, + "has_show_more_button": parsed_page.has_show_more_button, + "pages_fetched": paged_parse_result.pages_fetched, + "pages_attempted": paged_parse_result.pages_attempted, + "has_more_remaining": paged_parse_result.has_more_remaining, + "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, + "continuation_cstart": paged_parse_result.continuation_cstart, + "skipped_no_change": paged_parse_result.skipped_no_change, + "initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, + } + + def _skipped_no_change_outcome( + self, + *, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + ) -> ScholarProcessingOutcome: + first_page = paged_parse_result.first_page_parsed_page + scholar.last_run_status = RunStatus.SUCCESS + scholar.last_run_dt = run_dt + result_entry["state"] = first_page.state.value + result_entry["state_reason"] = "no_change_initial_page_signature" + result_entry["outcome"] = "success" + result_entry["publication_count"] = 0 + result_entry["warnings"] = first_page.warnings + result_entry["debug"] = { + "state_reason": "no_change_initial_page_signature", + "first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, + "attempt_log": paged_parse_result.attempt_log, + "page_logs": paged_parse_result.page_logs, + } + return ScholarProcessingOutcome( + result_entry=result_entry, + succeeded_count_delta=1, + failed_count_delta=0, + partial_count_delta=0, + discovered_publication_count=0, + ) + + async def _upsert_publications_outcome( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + ) -> ScholarProcessingOutcome: + parsed_page = paged_parse_result.parsed_page + publications = paged_parse_result.publications + had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS} + has_partial_set = len(publications) > 0 and had_page_failure + if (not had_page_failure) or has_partial_set: + return await self._upsert_success_or_exception( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + has_partial_publication_set=has_partial_set, + ) + return self._parse_failure_outcome( + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + + async def _upsert_success_or_exception( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + has_partial_publication_set: bool, + ) -> ScholarProcessingOutcome: + try: + return await self._upsert_success( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + has_partial_publication_set=has_partial_publication_set, + ) + except Exception as exc: + return self._upsert_exception_outcome( + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + exc=exc, + ) + + async def _upsert_success( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + has_partial_publication_set: bool, + ) -> ScholarProcessingOutcome: + # We no longer aggregate and upsert the publications here. + # Eager DB insertions have already saved and committed them inside `_fetch_and_parse_all_pages_with_retry` and `_paginate_loop`. + discovered_count = paged_parse_result.discovered_publication_count + is_partial = ( + paged_parse_result.has_more_remaining + or paged_parse_result.pagination_truncated_reason is not None + or has_partial_publication_set + ) + scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS + scholar.last_run_dt = run_dt + if not is_partial and paged_parse_result.first_page_fingerprint_sha256: + scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256 + result_entry["outcome"] = "partial" if is_partial else "success" + if is_partial: + result_entry["debug"] = self._build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + ) + return ScholarProcessingOutcome( + result_entry=result_entry, + succeeded_count_delta=1, + failed_count_delta=0, + partial_count_delta=1 if is_partial else 0, + discovered_publication_count=discovered_count, + ) + + def _upsert_exception_outcome( + self, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + exc: Exception, + ) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = run_dt + result_entry["state"] = "ingestion_error" + result_entry["state_reason"] = "publication_upsert_exception" + result_entry["outcome"] = "failed" + result_entry["error"] = str(exc) + result_entry["debug"] = self._build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + exception=exc, + ) + logger.exception( + "ingestion.scholar_failed", + extra={ + "event": "ingestion.scholar_failed", + "crawl_run_id": run.id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + }, + ) + return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) + + def _parse_failure_outcome( + self, + *, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + ) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = run_dt + result_entry["debug"] = self._build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + ) + logger.warning( + "ingestion.scholar_parse_failed", + extra={ + "event": "ingestion.scholar_parse_failed", + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": paged_parse_result.parsed_page.state.value, + "state_reason": paged_parse_result.parsed_page.state_reason, + "status_code": paged_parse_result.fetch_result.status_code, + }, + ) + return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) + + async def _sync_continuation_queue( + self, + db_session: AsyncSession, + *, + user_id: int, + scholar: ScholarProfile, + run: CrawlRun, + start_cstart: int, + result_entry: dict[str, Any], + paged_parse_result: PagedParseResult, + auto_queue_continuations: bool, + queue_delay_seconds: int, + ) -> None: + queue_reason, queue_cstart = self._resolve_continuation_queue_target( + outcome=str(result_entry.get("outcome", "")), + state=str(result_entry.get("state", "")), + pagination_truncated_reason=paged_parse_result.pagination_truncated_reason, + continuation_cstart=paged_parse_result.continuation_cstart, + fallback_cstart=start_cstart, + ) + if auto_queue_continuations and queue_reason is not None: + await queue_service.upsert_job( + db_session, + user_id=user_id, + scholar_profile_id=scholar.id, + resume_cstart=queue_cstart, + reason=queue_reason, + run_id=run.id, + delay_seconds=queue_delay_seconds, + ) + result_entry["continuation_enqueued"] = True + result_entry["continuation_reason"] = queue_reason + result_entry["continuation_cstart"] = queue_cstart + return + if await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=scholar.id): + result_entry["continuation_cleared"] = True + + async def _process_scholar( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + user_id: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + start_cstart: int, + auto_queue_continuations: bool, + queue_delay_seconds: int, + ) -> ScholarProcessingOutcome: + try: + return await self._process_scholar_inner( + db_session, + run=run, + scholar=scholar, + user_id=user_id, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + start_cstart=start_cstart, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + ) + except Exception as exc: + return self._unexpected_scholar_exception_outcome( + run=run, + scholar=scholar, + start_cstart=start_cstart, + exc=exc, + ) + + async def _process_scholar_inner( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + user_id: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + start_cstart: int, + auto_queue_continuations: bool, + queue_delay_seconds: int, + ) -> ScholarProcessingOutcome: + run_dt, paged_parse_result, result_entry = await self._fetch_and_prepare_scholar_result( + db_session, + run=run, + scholar=scholar, + user_id=user_id, + start_cstart=start_cstart, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + ) + outcome = await self._resolve_scholar_outcome( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + await self._sync_continuation_queue( + db_session, + user_id=user_id, + scholar=scholar, + run=run, + start_cstart=start_cstart, + result_entry=outcome.result_entry, + paged_parse_result=paged_parse_result, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + ) + return outcome + + async def _fetch_and_prepare_scholar_result( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + user_id: int, + start_cstart: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + ) -> tuple[datetime, PagedParseResult, dict[str, Any]]: + run_dt = datetime.now(timezone.utc) + paged_parse_result = await self._fetch_and_parse_all_pages_with_retry( + scholar=scholar, + run=run, + db_session=db_session, + start_cstart=start_cstart, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + max_pages=max_pages_per_scholar, + page_size=page_size, + previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256, + ) + self._assert_valid_paged_parse_result(scholar_id=scholar.scholar_id, paged_parse_result=paged_parse_result) + self._apply_first_page_profile_metadata(scholar=scholar, paged_parse_result=paged_parse_result, run_dt=run_dt) + self._log_scholar_parsed(user_id=user_id, run_id=run.id, scholar=scholar, paged_parse_result=paged_parse_result) + result_entry = self._build_result_entry( + scholar=scholar, + start_cstart=start_cstart, + paged_parse_result=paged_parse_result, + ) + return run_dt, paged_parse_result, result_entry + + async def _resolve_scholar_outcome( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + ) -> ScholarProcessingOutcome: + if paged_parse_result.skipped_no_change: + return self._skipped_no_change_outcome( + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + return await self._upsert_publications_outcome( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + + @staticmethod + def _apply_outcome_to_progress( + *, + progress: RunProgress, + run: CrawlRun, + outcome: ScholarProcessingOutcome, + ) -> None: + progress.succeeded_count += outcome.succeeded_count_delta + progress.failed_count += outcome.failed_count_delta + progress.partial_count += outcome.partial_count_delta + run.new_pub_count = int(run.new_pub_count or 0) + outcome.discovered_publication_count + progress.scholar_results.append(outcome.result_entry) + + def _unexpected_scholar_exception_outcome( + self, + *, + run: CrawlRun, + scholar: ScholarProfile, + start_cstart: int, + exc: Exception, + ) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = datetime.now(timezone.utc) + logger.exception( + "ingestion.scholar_unexpected_failure", + extra={ + "event": "ingestion.scholar_unexpected_failure", + "crawl_run_id": run.id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + }, + ) + return ScholarProcessingOutcome( + result_entry={ + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": "ingestion_error", + "state_reason": "scholar_processing_exception", + "outcome": "failed", + "attempt_count": 0, + "publication_count": 0, + "start_cstart": start_cstart, + "warnings": [], + "error": str(exc), + "debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)}, + }, + succeeded_count_delta=0, + failed_count_delta=1, + partial_count_delta=0, + discovered_publication_count=0, + ) + + @staticmethod + def _summarize_failures( + *, + scholar_results: list[dict[str, Any]], + ) -> RunFailureSummary: + failed_state_counts: dict[str, int] = {} + failed_reason_counts: dict[str, int] = {} + scrape_failure_counts: dict[str, int] = {} + retries_scheduled_count = 0 + scholars_with_retries_count = 0 + retry_exhausted_count = 0 + for entry in scholar_results: + retries_for_entry = max(0, _int_or_default(entry.get("attempt_count"), 0) - 1) + if retries_for_entry > 0: + retries_scheduled_count += retries_for_entry + scholars_with_retries_count += 1 + if str(entry.get("outcome", "")) != "failed": + continue + state = str(entry.get("state", "")).strip() + if state not in FAILED_STATES: + continue + failed_state_counts[state] = failed_state_counts.get(state, 0) + 1 + reason = str(entry.get("state_reason", "")).strip() + if reason: + failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1 + bucket = _classify_failure_bucket(state=state, state_reason=reason) + scrape_failure_counts[bucket] = scrape_failure_counts.get(bucket, 0) + 1 + if state == ParseState.NETWORK_ERROR.value and retries_for_entry > 0: + retry_exhausted_count += 1 + return RunFailureSummary( + failed_state_counts=failed_state_counts, + failed_reason_counts=failed_reason_counts, + scrape_failure_counts=scrape_failure_counts, + retries_scheduled_count=retries_scheduled_count, + scholars_with_retries_count=scholars_with_retries_count, + retry_exhausted_count=retry_exhausted_count, + ) + + @staticmethod + def _build_alert_summary( + *, + failure_summary: RunFailureSummary, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> RunAlertSummary: + blocked_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0)) + network_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0)) + blocked_threshold = max(1, int(alert_blocked_failure_threshold)) + network_threshold = max(1, int(alert_network_failure_threshold)) + retry_threshold = max(1, int(alert_retry_scheduled_threshold)) + alert_flags = { + "blocked_failure_threshold_exceeded": blocked_failure_count >= blocked_threshold, + "network_failure_threshold_exceeded": network_failure_count >= network_threshold, + "retry_scheduled_threshold_exceeded": failure_summary.retries_scheduled_count >= retry_threshold, + } + return RunAlertSummary( + blocked_failure_count=blocked_failure_count, + network_failure_count=network_failure_count, + blocked_failure_threshold=blocked_threshold, + network_failure_threshold=network_threshold, + retry_scheduled_threshold=retry_threshold, + alert_flags=alert_flags, + ) + + @staticmethod + def _log_alert_thresholds( + *, + user_id: int, + run_id: int, + failure_summary: RunFailureSummary, + alert_summary: RunAlertSummary, + ) -> None: + if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]: + logger.warning( + "ingestion.alert_blocked_failure_threshold_exceeded", + extra={ + "event": "ingestion.alert_blocked_failure_threshold_exceeded", + "user_id": user_id, + "crawl_run_id": run_id, + "blocked_failure_count": alert_summary.blocked_failure_count, + "threshold": alert_summary.blocked_failure_threshold, + "metric_name": "ingestion_blocked_failure_threshold_exceeded_total", + "metric_value": 1, + }, + ) + if alert_summary.alert_flags["network_failure_threshold_exceeded"]: + logger.warning( + "ingestion.alert_network_failure_threshold_exceeded", + extra={ + "event": "ingestion.alert_network_failure_threshold_exceeded", + "user_id": user_id, + "crawl_run_id": run_id, + "network_failure_count": alert_summary.network_failure_count, + "threshold": alert_summary.network_failure_threshold, + "metric_name": "ingestion_network_failure_threshold_exceeded_total", + "metric_value": 1, + }, + ) + if alert_summary.alert_flags["retry_scheduled_threshold_exceeded"]: + logger.warning( + "ingestion.alert_retry_scheduled_threshold_exceeded", + extra={ + "event": "ingestion.alert_retry_scheduled_threshold_exceeded", + "user_id": user_id, + "crawl_run_id": run_id, + "retries_scheduled_count": failure_summary.retries_scheduled_count, + "threshold": alert_summary.retry_scheduled_threshold, + "metric_name": "ingestion_retry_scheduled_threshold_exceeded_total", + "metric_value": 1, + }, + ) + + @staticmethod + def _apply_safety_outcome( + *, + user_settings, + run: CrawlRun, + user_id: int, + alert_summary: RunAlertSummary, + ) -> None: + pre_apply_state = run_safety_service.get_safety_event_context( + user_settings, + now_utc=datetime.now(timezone.utc), + ) + safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome( + user_settings, + run_id=int(run.id), + blocked_failure_count=alert_summary.blocked_failure_count, + network_failure_count=alert_summary.network_failure_count, + blocked_failure_threshold=alert_summary.blocked_failure_threshold, + network_failure_threshold=alert_summary.network_failure_threshold, + blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds, + network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds, + now_utc=datetime.now(timezone.utc), + ) + ScholarIngestionService._log_safety_transition( + user_id=user_id, + run_id=int(run.id), + alert_summary=alert_summary, + pre_apply_state=pre_apply_state, + safety_state=safety_state, + cooldown_trigger_reason=cooldown_trigger_reason, + ) + + @staticmethod + def _log_safety_transition( + *, + user_id: int, + run_id: int, + alert_summary: RunAlertSummary, + pre_apply_state: dict[str, Any], + safety_state: dict[str, Any], + cooldown_trigger_reason: str | None, + ) -> None: + if cooldown_trigger_reason is not None: + logger.warning( + "ingestion.safety_cooldown_entered", + extra={ + "event": "ingestion.safety_cooldown_entered", + "user_id": user_id, + "crawl_run_id": run_id, + "reason": cooldown_trigger_reason, + "blocked_failure_count": alert_summary.blocked_failure_count, + "network_failure_count": alert_summary.network_failure_count, + "blocked_failure_threshold": alert_summary.blocked_failure_threshold, + "network_failure_threshold": alert_summary.network_failure_threshold, + "cooldown_until": safety_state.get("cooldown_until"), + "cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"), + "safety_counters": safety_state.get("counters", {}), + "metric_name": "ingestion_safety_cooldown_entered_total", + "metric_value": 1, + }, + ) + elif pre_apply_state.get("cooldown_active") and not safety_state.get("cooldown_active"): + logger.info( + "ingestion.safety_cooldown_cleared", + extra={ + "event": "ingestion.safety_cooldown_cleared", + "user_id": user_id, + "crawl_run_id": run_id, + "reason": pre_apply_state.get("cooldown_reason"), + "cooldown_until": pre_apply_state.get("cooldown_until"), + "metric_name": "ingestion_safety_cooldown_cleared_total", + "metric_value": 1, + }, + ) + + def _finalize_run_record( + self, + *, + run: CrawlRun, + scholars: list[ScholarProfile], + progress: RunProgress, + failure_summary: RunFailureSummary, + alert_summary: RunAlertSummary, + idempotency_key: str | None, + ) -> None: + run.end_dt = datetime.now(timezone.utc) + if run.status != RunStatus.CANCELED: + run.status = self._resolve_run_status( + scholar_count=len(scholars), + succeeded_count=progress.succeeded_count, + failed_count=progress.failed_count, + partial_count=progress.partial_count, + ) + run.error_log = { + "scholar_results": progress.scholar_results, + "summary": { + "succeeded_count": progress.succeeded_count, + "failed_count": progress.failed_count, + "partial_count": progress.partial_count, + "failed_state_counts": failure_summary.failed_state_counts, + "failed_reason_counts": failure_summary.failed_reason_counts, + "scrape_failure_counts": failure_summary.scrape_failure_counts, + "retry_counts": { + "retries_scheduled_count": failure_summary.retries_scheduled_count, + "scholars_with_retries_count": failure_summary.scholars_with_retries_count, + "retry_exhausted_count": failure_summary.retry_exhausted_count, + }, + "alert_thresholds": { + "blocked_failure_threshold": alert_summary.blocked_failure_threshold, + "network_failure_threshold": alert_summary.network_failure_threshold, + "retry_scheduled_threshold": alert_summary.retry_scheduled_threshold, + }, + "alert_flags": alert_summary.alert_flags, + }, + "meta": {"idempotency_key": idempotency_key} if idempotency_key else {}, + } + + @staticmethod + def _log_run_completed( + *, + user_id: int, + run: CrawlRun, + scholars: list[ScholarProfile], + progress: RunProgress, + alert_summary: RunAlertSummary, + failure_summary: RunFailureSummary, + ) -> None: + logger.info( + "ingestion.run_completed", + extra={ + "event": "ingestion.run_completed", + "user_id": user_id, + "crawl_run_id": run.id, + "status": run.status.value, + "scholar_count": len(scholars), + "succeeded_count": progress.succeeded_count, + "failed_count": progress.failed_count, + "partial_count": progress.partial_count, + "new_publication_count": run.new_pub_count, + "blocked_failure_count": alert_summary.blocked_failure_count, + "network_failure_count": alert_summary.network_failure_count, + "retries_scheduled_count": failure_summary.retries_scheduled_count, + "alert_flags": alert_summary.alert_flags, + }, + ) + + @staticmethod + def _paging_kwargs( + *, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + ) -> dict[str, Any]: + return { + "request_delay_seconds": request_delay_seconds, + "network_error_retries": network_error_retries, + "retry_backoff_seconds": retry_backoff_seconds, + "rate_limit_retries": rate_limit_retries, + "rate_limit_backoff_seconds": rate_limit_backoff_seconds, + "max_pages_per_scholar": max_pages_per_scholar, + "page_size": page_size, + } + + @staticmethod + def _threshold_kwargs( + *, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> dict[str, Any]: + return { + "alert_blocked_failure_threshold": alert_blocked_failure_threshold, + "alert_network_failure_threshold": alert_network_failure_threshold, + "alert_retry_scheduled_threshold": alert_retry_scheduled_threshold, + } + + @staticmethod + def _run_execution_summary( + *, + run: CrawlRun, + scholars: list[ScholarProfile], + progress: RunProgress, + ) -> RunExecutionSummary: + return RunExecutionSummary( + crawl_run_id=run.id, + status=run.status, + scholar_count=len(scholars), + succeeded_count=progress.succeeded_count, + failed_count=progress.failed_count, + partial_count=progress.partial_count, + new_publication_count=run.new_pub_count, + ) + + async def _initialize_run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, scholar_profile_ids: set[int] | None, start_cstart_by_scholar_id: dict[int, int] | None, request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, rate_limit_retries: int, rate_limit_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, idempotency_key: str | None, alert_blocked_failure_threshold: int, alert_network_failure_threshold: int, alert_retry_scheduled_threshold: int) -> tuple[Any, CrawlRun, list[ScholarProfile], dict[int, int]]: + user_settings = await self._load_user_settings_for_run( + db_session, + user_id=user_id, + trigger_type=trigger_type, + ) + if not await self._try_acquire_user_lock(db_session, user_id=user_id): + raise RunAlreadyInProgressError(f"Run already in progress for user_id={user_id}.") + filtered_scholar_ids, start_cstart_map = self._normalize_run_targets( + scholar_profile_ids=scholar_profile_ids, + start_cstart_by_scholar_id=start_cstart_by_scholar_id, + ) + scholars = await self._load_target_scholars( + db_session, + user_id=user_id, + filtered_scholar_ids=filtered_scholar_ids, + ) + run = await self._start_run_record_for_targets( + db_session, + user_id=user_id, + trigger_type=trigger_type, + scholars=scholars, + filtered=filtered_scholar_ids is not None, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + idempotency_key=idempotency_key, + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + return user_settings, run, scholars, start_cstart_map + + async def _start_run_record_for_targets( + self, + db_session: AsyncSession, + *, + user_id: int, + trigger_type: RunTriggerType, + scholars: list[ScholarProfile], + filtered: bool, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + idempotency_key: str | None, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> CrawlRun: + self._log_run_started( + user_id=user_id, + trigger_type=trigger_type, + scholar_count=len(scholars), + filtered=filtered, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + idempotency_key=idempotency_key, + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + run = self._create_running_run( + user_id=user_id, + trigger_type=trigger_type, + scholar_count=len(scholars), + idempotency_key=idempotency_key, + ) + db_session.add(run) + await db_session.flush() + return run + + async def _run_scholar_iteration( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholars: list[ScholarProfile], + user_id: int, + start_cstart_map: dict[int, int], + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + auto_queue_continuations: bool, + queue_delay_seconds: int, + ) -> RunProgress: + from app.db.models import RunStatus + + progress = RunProgress() + for index, scholar in enumerate(scholars): + await db_session.refresh(run) + if run.status == RunStatus.CANCELED: + logger.info( + "ingestion.run_canceled", + extra={"event": "ingestion.run_canceled", "run_id": run.id, "user_id": user_id}, + ) + break + await self._wait_between_scholars(index=index, request_delay_seconds=request_delay_seconds) + start_cstart = int(start_cstart_map.get(int(scholar.id), 0)) + outcome = await self._process_scholar( + db_session, + run=run, + scholar=scholar, + user_id=user_id, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + start_cstart=start_cstart, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + ) + self._apply_outcome_to_progress(progress=progress, run=run, outcome=outcome) + return progress + + def _complete_run_for_user( + self, + *, + user_settings: Any, + run: CrawlRun, + scholars: list[ScholarProfile], + user_id: int, + progress: RunProgress, + idempotency_key: str | None, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> tuple[RunFailureSummary, RunAlertSummary]: + failure_summary = self._summarize_failures(scholar_results=progress.scholar_results) + alert_summary = self._build_alert_summary( + failure_summary=failure_summary, + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + self._log_alert_thresholds( + user_id=user_id, + run_id=int(run.id), + failure_summary=failure_summary, + alert_summary=alert_summary, + ) + self._apply_safety_outcome(user_settings=user_settings, run=run, user_id=user_id, alert_summary=alert_summary) + self._finalize_run_record( + run=run, + scholars=scholars, + progress=progress, + failure_summary=failure_summary, + alert_summary=alert_summary, + idempotency_key=idempotency_key, + ) + return failure_summary, alert_summary + + async def run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, request_delay_seconds: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, max_pages_per_scholar: int = 30, page_size: int = 100, scholar_profile_ids: set[int] | None = None, start_cstart_by_scholar_id: dict[int, int] | None = None, auto_queue_continuations: bool = True, queue_delay_seconds: int = 60, idempotency_key: str | None = None, alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3) -> RunExecutionSummary: + effective_request_delay_seconds = self._effective_request_delay_seconds(request_delay_seconds) + if effective_request_delay_seconds != _int_or_default(request_delay_seconds, effective_request_delay_seconds): + self._log_request_delay_coercion( + user_id=user_id, + requested_request_delay_seconds=_int_or_default(request_delay_seconds, 0), + effective_request_delay_seconds=effective_request_delay_seconds, + ) + paging_kwargs = self._paging_kwargs(request_delay_seconds=effective_request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, rate_limit_retries=settings.ingestion_rate_limit_retries, rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size) + threshold_kwargs = self._threshold_kwargs(alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold) + user_settings, run, scholars, start_cstart_map = await self._initialize_run_for_user( + db_session, + user_id=user_id, + trigger_type=trigger_type, + scholar_profile_ids=scholar_profile_ids, + start_cstart_by_scholar_id=start_cstart_by_scholar_id, + idempotency_key=idempotency_key, + **paging_kwargs, + **threshold_kwargs, + ) + progress = await self._run_scholar_iteration( + db_session, + run=run, + scholars=scholars, + user_id=user_id, + start_cstart_map=start_cstart_map, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + **paging_kwargs, + ) + failure_summary, alert_summary = self._complete_run_for_user( + user_settings=user_settings, + run=run, + scholars=scholars, + user_id=user_id, + progress=progress, + idempotency_key=idempotency_key, + **threshold_kwargs, + ) + await db_session.commit() + self._log_run_completed(user_id=user_id, run=run, scholars=scholars, progress=progress, alert_summary=alert_summary, failure_summary=failure_summary) + return self._run_execution_summary(run=run, scholars=scholars, progress=progress) + + async def _fetch_profile_page( + self, + *, + scholar_id: str, + cstart: int, + page_size: int, + ) -> FetchResult: + try: + page_fetcher = getattr(self._source, "fetch_profile_page_html", None) + if callable(page_fetcher): + return await page_fetcher( + scholar_id, + cstart=cstart, + pagesize=page_size, + ) + if cstart <= 0: + return await self._source.fetch_profile_html(scholar_id) + return FetchResult( + requested_url=( + "https://scholar.google.com/citations" + f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}" + ), + status_code=None, + final_url=None, + body="", + error="source_does_not_support_pagination", + ) + except Exception as exc: + logger.exception( + "ingestion.fetch_unexpected_error", + extra={ + "event": "ingestion.fetch_unexpected_error", + "scholar_id": scholar_id, + "cstart": cstart, + "page_size": page_size, + }, + ) + return FetchResult( + requested_url=( + "https://scholar.google.com/citations" + f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}" + ), + status_code=None, + final_url=None, + body="", + error=str(exc), + ) + + @staticmethod + def _attempt_log_entry( + *, + attempt: int, + cstart: int, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + ) -> dict[str, Any]: + return { + "attempt": attempt, + "cstart": cstart, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "status_code": fetch_result.status_code, + "fetch_error": fetch_result.error, + } + + @staticmethod + def _should_retry_page_fetch( + *, + parsed_page: ParsedProfilePage, + network_attempt_count: int, + rate_limit_attempt_count: int, + network_error_retries: int, + rate_limit_retries: int, + ) -> bool: + if parsed_page.state == ParseState.NETWORK_ERROR: + return network_attempt_count <= network_error_retries + if parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA and parsed_page.state_reason == "blocked_http_429_rate_limited": + return rate_limit_attempt_count <= rate_limit_retries + return False + + @staticmethod + async def _sleep_retry_backoff( + *, + scholar_id: str, + cstart: int, + network_attempt_count: int, + rate_limit_attempt_count: int, + retry_backoff_seconds: float, + rate_limit_backoff_seconds: float, + state_reason: str, + ) -> None: + if state_reason == "blocked_http_429_rate_limited": + sleep_seconds = rate_limit_backoff_seconds * rate_limit_attempt_count + attempt_label = rate_limit_attempt_count + else: + sleep_seconds = retry_backoff_seconds * (2 ** (network_attempt_count - 1)) + attempt_label = network_attempt_count + + logger.warning( + "ingestion.scholar_retry_scheduled", + extra={ + "event": "ingestion.scholar_retry_scheduled", + "scholar_id": scholar_id, + "cstart": cstart, + "attempt_count": attempt_label, + "sleep_seconds": sleep_seconds, + "state_reason": state_reason, + }, + ) + if sleep_seconds > 0: + await asyncio.sleep(sleep_seconds) + + async def _fetch_and_parse_page_with_retry( + self, + *, + scholar_id: str, + cstart: int, + page_size: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]: + network_attempts = 0 + rate_limit_attempts = 0 + attempt_log: list[dict[str, Any]] = [] + fetch_result: FetchResult | None = None + parsed_page: ParsedProfilePage | None = None + + while True: + fetch_result = await self._fetch_profile_page( + scholar_id=scholar_id, + cstart=cstart, + page_size=page_size, + ) + parsed_page = self._parse_profile_page_or_layout_error(fetch_result=fetch_result) + + if parsed_page.state == ParseState.NETWORK_ERROR: + network_attempts += 1 + total_attempts = network_attempts + elif parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA and parsed_page.state_reason == "blocked_http_429_rate_limited": + rate_limit_attempts += 1 + total_attempts = rate_limit_attempts + else: + total_attempts = network_attempts + rate_limit_attempts + 1 + + attempt_log.append( + self._attempt_log_entry( + attempt=total_attempts, + cstart=cstart, + fetch_result=fetch_result, + parsed_page=parsed_page, + ) + ) + + if not self._should_retry_page_fetch( + parsed_page=parsed_page, + network_attempt_count=network_attempts, + rate_limit_attempt_count=rate_limit_attempts, + network_error_retries=network_error_retries, + rate_limit_retries=rate_limit_retries, + ): + break + + await self._sleep_retry_backoff( + scholar_id=scholar_id, + cstart=cstart, + network_attempt_count=network_attempts, + rate_limit_attempt_count=rate_limit_attempts, + retry_backoff_seconds=max(float(retry_backoff_seconds), 0.0), + rate_limit_backoff_seconds=max(float(rate_limit_backoff_seconds), 0.0), + state_reason=parsed_page.state_reason, + ) + + if fetch_result is None or parsed_page is None: + raise RuntimeError("Fetch-and-parse retry loop produced no result.") + return fetch_result, parsed_page, attempt_log + + def _parse_profile_page_or_layout_error( + self, + *, + fetch_result: FetchResult, + ) -> ParsedProfilePage: + try: + return parse_profile_page(fetch_result) + except ScholarParserError as exc: + return self._parsed_page_from_parser_error( + fetch_result=fetch_result, + code=exc.code, + ) + + @staticmethod + def _parsed_page_from_parser_error( + *, + fetch_result: FetchResult, + code: str, + ) -> ParsedProfilePage: + return ParsedProfilePage( + state=ParseState.LAYOUT_CHANGED, + state_reason=code, + profile_name=None, + profile_image_url=None, + publications=[], + marker_counts={}, + warnings=[code], + has_show_more_button=False, + has_operation_error_banner=False, + articles_range=None, + ) + + @staticmethod + def _page_log_entry( + *, + page_number: int, + cstart: int, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + attempt_count: int, + ) -> dict[str, Any]: + return { + "page": page_number, + "cstart": cstart, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "status_code": fetch_result.status_code, + "publication_count": len(parsed_page.publications), + "articles_range": parsed_page.articles_range, + "has_show_more_button": parsed_page.has_show_more_button, + "warning_codes": parsed_page.warnings, + "attempt_count": attempt_count, + } + + @staticmethod + def _should_skip_no_change( + *, + start_cstart: int, + first_page_fingerprint_sha256: str | None, + previous_initial_page_fingerprint_sha256: str | None, + parsed_page: ParsedProfilePage, + ) -> bool: + return ( + start_cstart <= 0 + and first_page_fingerprint_sha256 is not None + and previous_initial_page_fingerprint_sha256 is not None + and first_page_fingerprint_sha256 == previous_initial_page_fingerprint_sha256 + and parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} + ) + + @staticmethod + def _skip_no_change_result( + *, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + first_page_fingerprint_sha256: str | None, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]], + ) -> PagedParseResult: + return PagedParseResult( + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fetch_result=fetch_result, + first_page_parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + publications=[], + attempt_log=attempt_log, + page_logs=page_logs, + pages_fetched=1, + pages_attempted=1, + has_more_remaining=False, + pagination_truncated_reason=None, + continuation_cstart=None, + skipped_no_change=True, + discovered_publication_count=0, + ) + + @staticmethod + def _initial_failure_result( + *, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + first_page_fingerprint_sha256: str | None, + start_cstart: int, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]], + ) -> PagedParseResult: + continuation_cstart = start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None + return PagedParseResult( + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fetch_result=fetch_result, + first_page_parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + publications=[], + attempt_log=attempt_log, + page_logs=page_logs, + pages_fetched=0, + pages_attempted=1, + has_more_remaining=False, + pagination_truncated_reason=None, + continuation_cstart=continuation_cstart, + skipped_no_change=False, + discovered_publication_count=0, + ) + + @staticmethod + def _build_loop_state( + *, + start_cstart: int, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]], + ) -> PagedLoopState: + next_cstart = _next_cstart_value( + articles_range=parsed_page.articles_range, + fallback=start_cstart + len(parsed_page.publications), + ) + return PagedLoopState( + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_log=attempt_log, + page_logs=page_logs, + publications=list(parsed_page.publications), + pages_fetched=1, + pages_attempted=1, + current_cstart=start_cstart, + next_cstart=next_cstart, + ) + + @staticmethod + def _set_truncated_state( + *, + state: PagedLoopState, + reason: str, + continuation_cstart: int, + ) -> None: + state.has_more_remaining = True + state.pagination_truncated_reason = reason + state.continuation_cstart = continuation_cstart + + def _should_stop_pagination(self, *, state: PagedLoopState, bounded_max_pages: int) -> bool: + if state.pages_fetched >= bounded_max_pages: + self._set_truncated_state( + state=state, + reason="max_pages_reached", + continuation_cstart=( + state.next_cstart if state.next_cstart > state.current_cstart else state.current_cstart + ), + ) + return True + if state.next_cstart <= state.current_cstart: + self._set_truncated_state( + state=state, + reason="pagination_cursor_stalled", + continuation_cstart=state.current_cstart, + ) + return True + return False + + async def _fetch_next_page( + self, + *, + scholar_id: str, + state: PagedLoopState, + request_delay_seconds: int, + bounded_page_size: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]: + if request_delay_seconds > 0: + await asyncio.sleep(float(request_delay_seconds)) + state.current_cstart = state.next_cstart + return await self._fetch_and_parse_page_with_retry( + scholar_id=scholar_id, + cstart=state.current_cstart, + page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + ) + + @staticmethod + def _record_next_page( + *, + state: PagedLoopState, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + page_attempt_log: list[dict[str, Any]], + ) -> None: + state.pages_attempted += 1 + state.attempt_log.extend(page_attempt_log) + state.page_logs.append( + ScholarIngestionService._page_log_entry( + page_number=state.pages_attempted, + cstart=state.current_cstart, + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_count=len(page_attempt_log), + ) + ) + state.fetch_result = fetch_result + state.parsed_page = parsed_page + + @staticmethod + def _handle_page_state_transition(*, state: PagedLoopState) -> bool: + if state.parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: + ScholarIngestionService._set_truncated_state( + state=state, + reason=f"page_state_{state.parsed_page.state.value}", + continuation_cstart=state.current_cstart, + ) + return True + if state.parsed_page.state == ParseState.NO_RESULTS and len(state.parsed_page.publications) == 0: + state.pages_fetched += 1 + return True + state.pages_fetched += 1 + state.publications.extend(state.parsed_page.publications) + state.next_cstart = _next_cstart_value( + articles_range=state.parsed_page.articles_range, + fallback=state.current_cstart + len(state.parsed_page.publications), + ) + return False + + async def _fetch_initial_page_context( + self, + *, + scholar_id: str, + start_cstart: int, + bounded_page_size: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + ) -> tuple[FetchResult, ParsedProfilePage, str | None, list[dict[str, Any]], list[dict[str, Any]]]: + fetch_result, parsed_page, first_attempt_log = await self._fetch_and_parse_page_with_retry( + scholar_id=scholar_id, + cstart=start_cstart, + page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + ) + first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page) + attempt_log = list(first_attempt_log) + page_logs = [ + self._page_log_entry( + page_number=1, + cstart=start_cstart, + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_count=len(first_attempt_log), + ) + ] + return fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs + + async def _paginate_loop( + self, + *, + scholar: ScholarProfile, + run: CrawlRun, + db_session: AsyncSession, + state: PagedLoopState, + bounded_max_pages: int, + request_delay_seconds: int, + bounded_page_size: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + ) -> None: + if state.parsed_page.publications: + discovered_count = await self._upsert_profile_publications( + db_session, run=run, scholar=scholar, publications=list(state.parsed_page.publications) + ) + state.discovered_publication_count += discovered_count + await db_session.commit() + + while state.parsed_page.has_show_more_button: + from app.db.models import RunStatus + await db_session.refresh(run) + if run.status == RunStatus.CANCELED: + logger.info( + "ingestion.pagination_canceled", + extra={"event": "ingestion.pagination_canceled", "run_id": run.id}, + ) + self._set_truncated_state( + state=state, + reason="run_canceled", + continuation_cstart=state.current_cstart, + ) + return + + if self._should_stop_pagination(state=state, bounded_max_pages=bounded_max_pages): + return + next_fetch_result, next_parsed_page, next_attempt_log = await self._fetch_next_page( + scholar_id=scholar.scholar_id, + state=state, + request_delay_seconds=request_delay_seconds, + bounded_page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + ) + self._record_next_page( + state=state, + fetch_result=next_fetch_result, + parsed_page=next_parsed_page, + page_attempt_log=next_attempt_log, + ) + + # Immediately commit the new page's publications to the DB + if next_parsed_page.publications: + discovered_count = await self._upsert_profile_publications( + db_session, run=run, scholar=scholar, publications=list(next_parsed_page.publications) + ) + state.discovered_publication_count += discovered_count + await db_session.commit() + + if self._handle_page_state_transition(state=state): + return + + @staticmethod + def _result_from_pagination_state( + *, + state: PagedLoopState, + first_page_fetch_result: FetchResult, + first_page_parsed_page: ParsedProfilePage, + first_page_fingerprint_sha256: str | None, + ) -> PagedParseResult: + return PagedParseResult( + fetch_result=state.fetch_result, + parsed_page=state.parsed_page, + first_page_fetch_result=first_page_fetch_result, + first_page_parsed_page=first_page_parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + publications=_dedupe_publication_candidates(state.publications), + attempt_log=state.attempt_log, + page_logs=state.page_logs, + pages_fetched=state.pages_fetched, + pages_attempted=state.pages_attempted, + has_more_remaining=state.has_more_remaining, + pagination_truncated_reason=state.pagination_truncated_reason, + continuation_cstart=state.continuation_cstart, + skipped_no_change=False, + discovered_publication_count=state.discovered_publication_count, + ) + + def _short_circuit_initial_page( + self, + *, + start_cstart: int, + previous_initial_page_fingerprint_sha256: str | None, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + first_page_fingerprint_sha256: str | None, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]], + ) -> PagedParseResult | None: + if self._should_skip_no_change( + start_cstart=start_cstart, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256, + parsed_page=parsed_page, + ): + return self._skip_no_change_result( + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + attempt_log=attempt_log, + page_logs=page_logs, + ) + if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: + return self._initial_failure_result( + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + start_cstart=start_cstart, + attempt_log=attempt_log, + page_logs=page_logs, + ) + return None + + async def _fetch_and_parse_all_pages_with_retry( + self, + *, + scholar: ScholarProfile, + run: CrawlRun, + db_session: AsyncSession, + start_cstart: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + max_pages: int, + page_size: int, + previous_initial_page_fingerprint_sha256: str | None = None + ) -> PagedParseResult: + bounded_max_pages = max(1, int(max_pages)) + bounded_page_size = max(1, int(page_size)) + fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs = ( + await self._fetch_initial_page_context( + scholar_id=scholar.scholar_id, + start_cstart=start_cstart, + bounded_page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + )) + shortcut_result = self._short_circuit_initial_page( + start_cstart=start_cstart, + previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256, + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + attempt_log=attempt_log, + page_logs=page_logs, + ) + if shortcut_result is not None: + return shortcut_result + state = self._build_loop_state( + start_cstart=start_cstart, + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_log=attempt_log, + page_logs=page_logs, + ) + await self._paginate_loop( + scholar=scholar, + run=run, + db_session=db_session, + state=state, + bounded_max_pages=bounded_max_pages, + request_delay_seconds=request_delay_seconds, + bounded_page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + ) + return self._result_from_pagination_state( + state=state, + first_page_fetch_result=fetch_result, + first_page_parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + ) + + async def _enrich_publications_with_openalex( + self, + scholar: ScholarProfile, + publications: list[PublicationCandidate], + ) -> list[PublicationCandidate]: + if not publications: + return publications + + from app.services.domains.openalex.client import OpenAlexClient + from app.services.domains.openalex.matching import find_best_match + + client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto) + + # Batch requests by 25 to stay within URL length limits + batch_size = 25 + enriched: list[PublicationCandidate] = [] + + import re + for i in range(0, len(publications), batch_size): + batch = publications[i:i + batch_size] + + titles = [] + for p in batch: + if not p.title: + continue + # Strip all non-alphanumeric words to prevent OpenAlex API injection crashes + # and minimize punctuation-based duplicate misses. + safe_title = re.sub(r"[^\w\s]", " ", p.title) + safe_title = " ".join(safe_title.split()) + if safe_title: + titles.append(safe_title) + + if not titles: + enriched.extend(batch) + continue + + query = "|".join(t for t in titles) + try: + openalex_works = await client.get_works_by_filter( + {"title.search": query}, limit=batch_size * 3 + ) + except Exception as e: + logger.warning( + "ingestion.openalex_enrichment_failed", + extra={"error": str(e), "batch_size": len(batch), "scholar_id": scholar.id} + ) + openalex_works = [] + + for p in batch: + match = find_best_match( + target_title=p.title, + target_year=p.year, + target_authors=p.authors_text or scholar.name, + candidates=openalex_works + ) + if match: + new_p = PublicationCandidate( + title=p.title, + title_url=p.title_url, + cluster_id=p.cluster_id, + year=match.publication_year or p.year, + citation_count=match.cited_by_count, + authors_text=p.authors_text, + venue_text=p.venue_text, + pdf_url=match.oa_url or p.pdf_url, + ) + enriched.append(new_p) + else: + enriched.append(p) + + return enriched + + async def _upsert_profile_publications( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + publications: list[PublicationCandidate], + ) -> int: + publications = await self._enrich_publications_with_openalex(scholar, publications) + + seen_publication_ids: set[int] = set() + discovered_count = 0 + + for candidate in publications: + publication = await self._resolve_publication(db_session, candidate) + if publication.id in seen_publication_ids: + continue + seen_publication_ids.add(publication.id) + + link_result = await db_session.execute( + select(ScholarPublication).where( + ScholarPublication.scholar_profile_id == scholar.id, + ScholarPublication.publication_id == publication.id, + ) + ) + link = link_result.scalar_one_or_none() + if link is not None: + continue + + link = ScholarPublication( + scholar_profile_id=scholar.id, + publication_id=publication.id, + is_read=False, + first_seen_run_id=run.id, + ) + db_session.add(link) + discovered_count += 1 + + logger.debug( + "ingestion.publication_discovered", + extra={ + "event": "ingestion.publication_discovered", + "scholar_profile_id": scholar.id, + "publication_id": publication.id, + "crawl_run_id": run.id, + }, + ) + + # Commit immediately to ensure the row becomes visible to the UI immediately + await db_session.commit() + + # Emit Server-Sent Event for live UI progress updates + await run_events.publish( + run_id=run.id, + event_type="publication_discovered", + data={ + "publication_id": publication.id, + "title": publication.title_raw, + "pub_url": publication.pub_url, + "scholar_profile_id": scholar.id, + "scholar_label": scholar.display_name or scholar.scholar_id, + "first_seen_at": datetime.now(timezone.utc).isoformat(), + } + ) + + if not scholar.baseline_completed: + scholar.baseline_completed = True + + return discovered_count + + @staticmethod + def _validate_publication_candidate(candidate: PublicationCandidate) -> None: + if not candidate.title.strip(): + raise RuntimeError("Publication candidate is missing title.") + if candidate.citation_count is not None and int(candidate.citation_count) < 0: + raise RuntimeError("Publication candidate has negative citation_count.") + + async def _find_publication_by_cluster( + self, + db_session: AsyncSession, + *, + cluster_id: str | None, + ) -> Publication | None: + if not cluster_id: + return None + result = await db_session.execute( + select(Publication).where(Publication.cluster_id == cluster_id) + ) + return result.scalar_one_or_none() + + async def _find_publication_by_fingerprint( + self, + db_session: AsyncSession, + *, + fingerprint: str, + ) -> Publication | None: + result = await db_session.execute( + select(Publication).where(Publication.fingerprint_sha256 == fingerprint) + ) + return result.scalar_one_or_none() + + @staticmethod + def _select_existing_publication( + *, + cluster_publication: Publication | None, + fingerprint_publication: Publication | None, + ) -> Publication | None: + if cluster_publication is not None: + return cluster_publication + return fingerprint_publication + + async def _create_publication( + self, + db_session: AsyncSession, + *, + candidate: PublicationCandidate, + fingerprint: str, + ) -> Publication: + publication = Publication( + cluster_id=candidate.cluster_id, + fingerprint_sha256=fingerprint, + title_raw=candidate.title, + title_normalized=normalize_title(candidate.title), + year=candidate.year, + citation_count=int(candidate.citation_count or 0), + author_text=candidate.authors_text, + venue_text=candidate.venue_text, + pub_url=build_publication_url(candidate.title_url), + pdf_url=None, + ) + db_session.add(publication) + await db_session.flush() + logger.debug( + "ingestion.publication_created", + extra={ + "event": "ingestion.publication_created", + "publication_id": publication.id, + "cluster_id": publication.cluster_id, + }, + ) + return publication + + @staticmethod + def _update_existing_publication( + *, + publication: Publication, + candidate: PublicationCandidate, + ) -> None: + if candidate.cluster_id and publication.cluster_id is None: + publication.cluster_id = candidate.cluster_id + publication.title_raw = candidate.title + publication.title_normalized = normalize_title(candidate.title) + if candidate.year is not None: + publication.year = candidate.year + if candidate.citation_count is not None: + publication.citation_count = int(candidate.citation_count) + if candidate.authors_text: + publication.author_text = candidate.authors_text + if candidate.venue_text: + publication.venue_text = candidate.venue_text + if candidate.title_url: + publication.pub_url = build_publication_url(candidate.title_url) + local_doi = first_doi_from_texts(candidate.title_url, candidate.venue_text, candidate.title) + + async def _resolve_publication( + self, + db_session: AsyncSession, + candidate: PublicationCandidate, + ) -> Publication: + self._validate_publication_candidate(candidate) + fingerprint = build_publication_fingerprint(candidate) + cluster_publication = await self._find_publication_by_cluster( + db_session, + cluster_id=candidate.cluster_id, + ) + fingerprint_publication = await self._find_publication_by_fingerprint( + db_session, + fingerprint=fingerprint, + ) + publication = self._select_existing_publication( + cluster_publication=cluster_publication, + fingerprint_publication=fingerprint_publication, + ) + if publication is None: + created = await self._create_publication( + db_session, + candidate=candidate, + fingerprint=fingerprint, + ) + await identifier_service.discover_and_sync_identifiers_for_publication( + db_session, + publication=created, + scholar_label=candidate.authors_text or "", + ) + return created + self._update_existing_publication( + publication=publication, + candidate=candidate, + ) + await identifier_service.discover_and_sync_identifiers_for_publication( + db_session, + publication=publication, + scholar_label=candidate.authors_text or "", + ) + return publication + + def _resolve_run_status( + self, + *, + scholar_count: int, + succeeded_count: int, + failed_count: int, + partial_count: int, + ) -> RunStatus: + if scholar_count == 0: + return RunStatus.SUCCESS + if failed_count == scholar_count: + return RunStatus.FAILED + if failed_count > 0 or partial_count > 0: + return RunStatus.PARTIAL_FAILURE + if succeeded_count > 0: + return RunStatus.SUCCESS + return RunStatus.FAILED + + def _resolve_continuation_queue_target( + self, + *, + outcome: str, + state: str, + pagination_truncated_reason: str | None, + continuation_cstart: int | None, + fallback_cstart: int, + ) -> tuple[str | None, int]: + if outcome == "partial": + reason = (pagination_truncated_reason or "").strip() + if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith( + RESUMABLE_PARTIAL_REASON_PREFIXES + ): + return reason, queue_service.normalize_cstart( + continuation_cstart if continuation_cstart is not None else fallback_cstart + ) + return None, queue_service.normalize_cstart(fallback_cstart) + + if outcome == "failed" and state == ParseState.NETWORK_ERROR.value: + return "network_error_retry", queue_service.normalize_cstart( + continuation_cstart if continuation_cstart is not None else fallback_cstart + ) + + return None, queue_service.normalize_cstart(fallback_cstart) + + def _build_failure_debug_context( + self, + *, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]] | None = None, + exception: Exception | None = None, + ) -> dict[str, Any]: + context: dict[str, Any] = { + "requested_url": fetch_result.requested_url, + "final_url": fetch_result.final_url, + "status_code": fetch_result.status_code, + "fetch_error": fetch_result.error, + "state_reason": parsed_page.state_reason, + "profile_name": parsed_page.profile_name, + "profile_image_url": parsed_page.profile_image_url, + "articles_range": parsed_page.articles_range, + "has_show_more_button": parsed_page.has_show_more_button, + "has_operation_error_banner": parsed_page.has_operation_error_banner, + "warning_codes": parsed_page.warnings, + "marker_counts_nonzero": { + key: value for key, value in parsed_page.marker_counts.items() if value > 0 + }, + "body_length": len(fetch_result.body), + "body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest() + if fetch_result.body + else None, + "body_excerpt": _build_body_excerpt(fetch_result.body), + "attempt_log": attempt_log, + } + if page_logs: + context["page_logs"] = page_logs + if exception is not None: + context["exception_type"] = type(exception).__name__ + context["exception_message"] = str(exception) + return context + + async def _try_acquire_user_lock( + self, + db_session: AsyncSession, + *, + user_id: int, + ) -> bool: + result = await db_session.execute( + text( + "SELECT pg_try_advisory_xact_lock(:namespace, :user_key)" + ), + { + "namespace": RUN_LOCK_NAMESPACE, + "user_key": int(user_id), + }, + ) + return bool(result.scalar_one()) diff --git a/build/lib/app/services/domains/ingestion/constants.py b/build/lib/app/services/domains/ingestion/constants.py new file mode 100644 index 0000000..cfe2ba0 --- /dev/null +++ b/build/lib/app/services/domains/ingestion/constants.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import re + +from app.services.domains.scholar.parser import ParseState + +TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+") +WORD_RE = re.compile(r"[a-z0-9]+") +HTML_TAG_RE = re.compile(r"<[^>]+>", re.S) +SPACE_RE = re.compile(r"\s+") + +FAILED_STATES = { + ParseState.BLOCKED_OR_CAPTCHA.value, + ParseState.LAYOUT_CHANGED.value, + ParseState.NETWORK_ERROR.value, + "ingestion_error", +} + +FAILURE_BUCKET_BLOCKED = "blocked_or_captcha" +FAILURE_BUCKET_NETWORK = "network_error" +FAILURE_BUCKET_LAYOUT = "layout_changed" +FAILURE_BUCKET_INGESTION = "ingestion_error" +FAILURE_BUCKET_OTHER = "other_failure" + +RUN_LOCK_NAMESPACE = 8217 +RESUMABLE_PARTIAL_REASONS = { + "max_pages_reached", + "pagination_cursor_stalled", +} +RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",) +INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30 diff --git a/build/lib/app/services/domains/ingestion/fingerprints.py b/build/lib/app/services/domains/ingestion/fingerprints.py new file mode 100644 index 0000000..b59710e --- /dev/null +++ b/build/lib/app/services/domains/ingestion/fingerprints.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import hashlib +import json +import re +from typing import Any +from urllib.parse import urljoin + +from app.services.domains.ingestion.constants import ( + HTML_TAG_RE, + INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS, + SPACE_RE, + TITLE_ALNUM_RE, + WORD_RE, +) +from app.services.domains.scholar.parser import ParseState, ParsedProfilePage, PublicationCandidate + + +def normalize_title(value: str) -> str: + lowered = value.lower() + return TITLE_ALNUM_RE.sub("", lowered) + + +def _first_author_last_name(authors_text: str | None) -> str: + if not authors_text: + return "" + first_author = authors_text.split(",", maxsplit=1)[0].strip().lower() + words = WORD_RE.findall(first_author) + if not words: + return "" + return words[-1] + + +def _first_venue_word(venue_text: str | None) -> str: + if not venue_text: + return "" + words = WORD_RE.findall(venue_text.lower()) + if not words: + return "" + return words[0] + + +def build_publication_fingerprint(candidate: PublicationCandidate) -> str: + canonical = "|".join( + [ + normalize_title(candidate.title), + str(candidate.year) if candidate.year is not None else "", + _first_author_last_name(candidate.authors_text), + _first_venue_word(candidate.venue_text), + ] + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None: + if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: + return None + + normalized_rows: list[dict[str, Any]] = [] + for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]: + normalized_rows.append( + { + "cluster_id": publication.cluster_id or "", + "title_normalized": normalize_title(publication.title), + "year": publication.year, + "citation_count": publication.citation_count, + } + ) + + payload = { + "state": parsed_page.state.value, + "articles_range": parsed_page.articles_range or "", + "has_show_more_button": parsed_page.has_show_more_button, + "profile_name": parsed_page.profile_name or "", + "publications": normalized_rows, + } + canonical = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def build_publication_url(path_or_url: str | None) -> str | None: + if not path_or_url: + return None + return urljoin("https://scholar.google.com", path_or_url) + + +def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int: + if articles_range: + numbers = re.findall(r"\d+", articles_range) + if len(numbers) >= 2: + try: + return int(numbers[1]) + except ValueError: + pass + return int(fallback) + + +def _title_tokens(value: str) -> set[str]: + """Extract normalized word tokens for fuzzy title comparison.""" + return set(WORD_RE.findall(value.lower())) + + +def fuzzy_titles_match( + title_a: str, + title_b: str, + *, + threshold: float = 0.85, +) -> bool: + """Return True if two titles are near-duplicates by token-level Jaccard similarity. + + A threshold of 0.85 catches common academic duplicate patterns: + differences in punctuation, minor word variations, subtitle changes. + """ + tokens_a = _title_tokens(title_a) + tokens_b = _title_tokens(title_b) + if not tokens_a or not tokens_b: + return False + intersection = tokens_a & tokens_b + union = tokens_a | tokens_b + return (len(intersection) / len(union)) >= threshold + + +def _dedupe_publication_candidates( + publications: list[PublicationCandidate], +) -> list[PublicationCandidate]: + deduped: list[PublicationCandidate] = [] + seen: set[str] = set() + seen_titles: list[tuple[str, int]] = [] # (normalized_title, index into deduped) + + for publication in publications: + if publication.cluster_id: + identity = f"cluster:{publication.cluster_id}" + else: + identity = "|".join( + [ + "fallback", + normalize_title(publication.title), + str(publication.year) if publication.year is not None else "", + _first_author_last_name(publication.authors_text), + _first_venue_word(publication.venue_text), + ] + ) + if identity in seen: + continue + + # Fuzzy title check — catch near-identical titles not caught by exact fingerprint + norm_title = normalize_title(publication.title) + is_fuzzy_dup = False + for existing_title, _idx in seen_titles: + if fuzzy_titles_match(norm_title, existing_title): + is_fuzzy_dup = True + break + if is_fuzzy_dup: + continue + + seen.add(identity) + seen_titles.append((norm_title, len(deduped))) + deduped.append(publication) + return deduped + + +def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None: + if not body: + return None + flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip() + if not flattened: + return None + if len(flattened) <= max_chars: + return flattened + return f"{flattened[:max_chars - 1]}..." diff --git a/build/lib/app/services/domains/ingestion/queue.py b/build/lib/app/services/domains/ingestion/queue.py new file mode 100644 index 0000000..7494420 --- /dev/null +++ b/build/lib/app/services/domains/ingestion/queue.py @@ -0,0 +1,348 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import IngestionQueueItem, QueueItemStatus + + +@dataclass(frozen=True) +class ContinuationQueueJob: + id: int + user_id: int + scholar_profile_id: int + resume_cstart: int + reason: str + status: str + attempt_count: int + next_attempt_dt: datetime + + +ACTIVE_QUEUE_STATUSES: tuple[str, ...] = ( + QueueItemStatus.QUEUED.value, + QueueItemStatus.RETRYING.value, +) + + +def normalize_cstart(value: int | None) -> int: + if value is None: + return 0 + return max(0, int(value)) + + +def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int: + base = max(1, int(base_seconds)) + attempts = max(1, int(attempt_count)) + maximum = max(base, int(max_seconds)) + seconds = base * (2 ** max(0, attempts - 1)) + return min(seconds, maximum) + + +async def _get_item_for_user_scholar( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, +) -> IngestionQueueItem | None: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.user_id == user_id, + IngestionQueueItem.scholar_profile_id == scholar_profile_id, + ) + ) + return result.scalar_one_or_none() + + +def _build_queue_item( + *, + now: datetime, + user_id: int, + scholar_profile_id: int, + normalized_cstart: int, + reason: str, + run_id: int | None, + next_attempt_dt: datetime, +) -> IngestionQueueItem: + return IngestionQueueItem( + user_id=user_id, + scholar_profile_id=scholar_profile_id, + resume_cstart=normalized_cstart, + reason=reason, + status=QueueItemStatus.QUEUED.value, + attempt_count=0, + next_attempt_dt=next_attempt_dt, + last_run_id=run_id, + last_error=None, + dropped_reason=None, + dropped_at=None, + created_at=now, + updated_at=now, + ) + + +def _update_existing_queue_item( + *, + item: IngestionQueueItem, + now: datetime, + normalized_cstart: int, + reason: str, + run_id: int | None, + next_attempt_dt: datetime, +) -> None: + item.resume_cstart = normalized_cstart + item.reason = reason + if item.status == QueueItemStatus.DROPPED.value: + item.attempt_count = 0 + item.status = QueueItemStatus.QUEUED.value + item.next_attempt_dt = next_attempt_dt + item.last_run_id = run_id + item.last_error = None + item.dropped_reason = None + item.dropped_at = None + item.updated_at = now + + +async def upsert_job( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, + resume_cstart: int, + reason: str, + run_id: int | None, + delay_seconds: int, +) -> IngestionQueueItem: + now = datetime.now(timezone.utc) + next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds))) + item = await _get_item_for_user_scholar( + db_session, + user_id=user_id, + scholar_profile_id=scholar_profile_id, + ) + normalized_cstart = normalize_cstart(resume_cstart) + if item is None: + item = _build_queue_item( + now=now, + user_id=user_id, + scholar_profile_id=scholar_profile_id, + normalized_cstart=normalized_cstart, + reason=reason, + run_id=run_id, + next_attempt_dt=next_attempt_dt, + ) + db_session.add(item) + return item + + _update_existing_queue_item( + item=item, + now=now, + normalized_cstart=normalized_cstart, + reason=reason, + run_id=run_id, + next_attempt_dt=next_attempt_dt, + ) + return item + + +async def clear_job_for_scholar( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, +) -> bool: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.user_id == user_id, + IngestionQueueItem.scholar_profile_id == scholar_profile_id, + ) + ) + item = result.scalar_one_or_none() + if item is None: + return False + await db_session.delete(item) + return True + + +async def delete_job_by_id( + db_session: AsyncSession, + *, + job_id: int, +) -> bool: + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return False + await db_session.delete(item) + return True + + +async def list_due_jobs( + db_session: AsyncSession, + *, + now: datetime, + limit: int, +) -> list[ContinuationQueueJob]: + result = await db_session.execute( + select(IngestionQueueItem) + .where( + IngestionQueueItem.next_attempt_dt <= now, + IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES), + ) + .order_by( + IngestionQueueItem.next_attempt_dt.asc(), + IngestionQueueItem.id.asc(), + ) + .limit(limit) + ) + rows = list(result.scalars().all()) + jobs: list[ContinuationQueueJob] = [] + for row in rows: + jobs.append( + ContinuationQueueJob( + id=int(row.id), + user_id=int(row.user_id), + scholar_profile_id=int(row.scholar_profile_id), + resume_cstart=normalize_cstart(row.resume_cstart), + reason=row.reason, + status=row.status, + attempt_count=int(row.attempt_count), + next_attempt_dt=row.next_attempt_dt, + ) + ) + return jobs + + +async def increment_attempt_count( + db_session: AsyncSession, + *, + job_id: int, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.attempt_count = int(item.attempt_count or 0) + 1 + item.updated_at = now + return item + + +async def reset_attempt_count( + db_session: AsyncSession, + *, + job_id: int, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.attempt_count = 0 + item.updated_at = now + return item + + +async def reschedule_job( + db_session: AsyncSession, + *, + job_id: int, + delay_seconds: int, + reason: str, + error: str | None = None, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds))) + item.status = QueueItemStatus.QUEUED.value + item.reason = reason + item.last_error = error + item.dropped_reason = None + item.dropped_at = None + item.updated_at = now + return item + + +async def mark_retrying( + db_session: AsyncSession, + *, + job_id: int, + reason: str | None = None, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + if item.status == QueueItemStatus.DROPPED.value: + return item + item.status = QueueItemStatus.RETRYING.value + if reason: + item.reason = reason + item.updated_at = now + return item + + +async def mark_dropped( + db_session: AsyncSession, + *, + job_id: int, + reason: str, + error: str | None = None, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.status = QueueItemStatus.DROPPED.value + item.reason = "dropped" + item.dropped_reason = reason + item.dropped_at = now + if error is not None: + item.last_error = error + item.updated_at = now + return item + + +async def mark_queued_now( + db_session: AsyncSession, + *, + job_id: int, + reason: str, + reset_attempt_count: bool = False, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.status = QueueItemStatus.QUEUED.value + item.reason = reason + item.next_attempt_dt = now + if reset_attempt_count: + item.attempt_count = 0 + item.last_error = None + item.dropped_reason = None + item.dropped_at = None + item.updated_at = now + return item diff --git a/build/lib/app/services/domains/ingestion/safety.py b/build/lib/app/services/domains/ingestion/safety.py new file mode 100644 index 0000000..debb8c5 --- /dev/null +++ b/build/lib/app/services/domains/ingestion/safety.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import Any + +from app.db.models import UserSetting + +COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD = "blocked_failure_threshold_exceeded" +COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD = "network_failure_threshold_exceeded" + +_COUNTER_CONSECUTIVE_BLOCKED_RUNS = "consecutive_blocked_runs" +_COUNTER_CONSECUTIVE_NETWORK_RUNS = "consecutive_network_runs" +_COUNTER_COOLDOWN_ENTRY_COUNT = "cooldown_entry_count" +_COUNTER_BLOCKED_START_COUNT = "blocked_start_count" +_COUNTER_LAST_BLOCKED_FAILURE_COUNT = "last_blocked_failure_count" +_COUNTER_LAST_NETWORK_FAILURE_COUNT = "last_network_failure_count" +_COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id" + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def _safe_int(value: Any, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _safe_optional_int(value: Any) -> int | None: + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _normalize_datetime(value: datetime | None) -> datetime | None: + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def _state_dict(settings: UserSetting) -> dict[str, Any]: + state = settings.scrape_safety_state + if isinstance(state, dict): + return state + return {} + + +def _counters_from_state(settings: UserSetting) -> dict[str, Any]: + state = _state_dict(settings) + return { + _COUNTER_CONSECUTIVE_BLOCKED_RUNS: max( + 0, + _safe_int(state.get(_COUNTER_CONSECUTIVE_BLOCKED_RUNS), 0), + ), + _COUNTER_CONSECUTIVE_NETWORK_RUNS: max( + 0, + _safe_int(state.get(_COUNTER_CONSECUTIVE_NETWORK_RUNS), 0), + ), + _COUNTER_COOLDOWN_ENTRY_COUNT: max( + 0, + _safe_int(state.get(_COUNTER_COOLDOWN_ENTRY_COUNT), 0), + ), + _COUNTER_BLOCKED_START_COUNT: max( + 0, + _safe_int(state.get(_COUNTER_BLOCKED_START_COUNT), 0), + ), + _COUNTER_LAST_BLOCKED_FAILURE_COUNT: max( + 0, + _safe_int(state.get(_COUNTER_LAST_BLOCKED_FAILURE_COUNT), 0), + ), + _COUNTER_LAST_NETWORK_FAILURE_COUNT: max( + 0, + _safe_int(state.get(_COUNTER_LAST_NETWORK_FAILURE_COUNT), 0), + ), + _COUNTER_LAST_EVALUATED_RUN_ID: _safe_optional_int( + state.get(_COUNTER_LAST_EVALUATED_RUN_ID), + ), + } + + +def _cooldown_reason_label(reason: str | None) -> str | None: + if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD: + return "Blocked responses exceeded safety threshold" + if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD: + return "Network failures exceeded safety threshold" + return None + + +def _recommended_action(reason: str | None) -> str | None: + if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD: + return ( + "Google Scholar appears to be blocking requests. Wait for cooldown to expire, " + "increase request delay, and avoid repeated manual retries." + ) + if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD: + return ( + "Network failures crossed the threshold. Verify connectivity and retry after cooldown." + ) + return None + + +def is_cooldown_active( + settings: UserSetting, + *, + now_utc: datetime | None = None, +) -> bool: + now = now_utc or _utcnow() + cooldown_until = _normalize_datetime(settings.scrape_cooldown_until) + if cooldown_until is None: + return False + return cooldown_until > now + + +def clear_expired_cooldown( + settings: UserSetting, + *, + now_utc: datetime | None = None, +) -> bool: + now = now_utc or _utcnow() + cooldown_until = _normalize_datetime(settings.scrape_cooldown_until) + if cooldown_until is None: + return False + if cooldown_until > now: + return False + settings.scrape_cooldown_until = None + settings.scrape_cooldown_reason = None + return True + + +def register_cooldown_blocked_start( + settings: UserSetting, + *, + now_utc: datetime | None = None, +) -> dict[str, Any]: + now = now_utc or _utcnow() + counters = _counters_from_state(settings) + counters[_COUNTER_BLOCKED_START_COUNT] = int(counters[_COUNTER_BLOCKED_START_COUNT]) + 1 + settings.scrape_safety_state = counters + return get_safety_state_payload(settings, now_utc=now) + + +def _update_run_counters( + *, + counters: dict[str, Any], + run_id: int, + blocked_failure_count: int, + network_failure_count: int, +) -> tuple[int, int]: + bounded_blocked_failures = max(0, int(blocked_failure_count)) + bounded_network_failures = max(0, int(network_failure_count)) + counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures + counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures + counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id) + counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = ( + int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1 + if bounded_blocked_failures > 0 + else 0 + ) + counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = ( + int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1 + if bounded_network_failures > 0 + else 0 + ) + return bounded_blocked_failures, bounded_network_failures + + +def _resolve_cooldown_trigger( + *, + blocked_failures: int, + network_failures: int, + blocked_failure_threshold: int, + network_failure_threshold: int, + blocked_cooldown_seconds: int, + network_cooldown_seconds: int, +) -> tuple[str | None, int]: + if blocked_failures >= max(1, int(blocked_failure_threshold)): + return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds)) + if network_failures >= max(1, int(network_failure_threshold)): + return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds)) + return None, 0 + + +def _apply_cooldown_decision( + *, + settings: UserSetting, + counters: dict[str, Any], + now: datetime, + reason: str | None, + cooldown_seconds: int, +) -> None: + if reason is None: + clear_expired_cooldown(settings, now_utc=now) + return + settings.scrape_cooldown_reason = reason + settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds))) + counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1 + + +def apply_run_safety_outcome( + settings: UserSetting, + *, + run_id: int, + blocked_failure_count: int, + network_failure_count: int, + blocked_failure_threshold: int, + network_failure_threshold: int, + blocked_cooldown_seconds: int, + network_cooldown_seconds: int, + now_utc: datetime | None = None, +) -> tuple[dict[str, Any], str | None]: + now = now_utc or _utcnow() + counters = _counters_from_state(settings) + blocked_failures, network_failures = _update_run_counters( + counters=counters, + run_id=run_id, + blocked_failure_count=blocked_failure_count, + network_failure_count=network_failure_count, + ) + reason, cooldown_seconds = _resolve_cooldown_trigger( + blocked_failures=blocked_failures, + network_failures=network_failures, + blocked_failure_threshold=blocked_failure_threshold, + network_failure_threshold=network_failure_threshold, + blocked_cooldown_seconds=blocked_cooldown_seconds, + network_cooldown_seconds=network_cooldown_seconds, + ) + _apply_cooldown_decision( + settings=settings, + counters=counters, + now=now, + reason=reason, + cooldown_seconds=cooldown_seconds, + ) + settings.scrape_safety_state = counters + return get_safety_state_payload(settings, now_utc=now), reason + + +def get_safety_state_payload( + settings: UserSetting, + *, + now_utc: datetime | None = None, +) -> dict[str, Any]: + now = now_utc or _utcnow() + cooldown_until = _normalize_datetime(settings.scrape_cooldown_until) + cooldown_active = bool(cooldown_until is not None and cooldown_until > now) + cooldown_remaining_seconds = 0 + if cooldown_active and cooldown_until is not None: + cooldown_remaining_seconds = max(0, int((cooldown_until - now).total_seconds())) + + reason = settings.scrape_cooldown_reason if cooldown_active else None + + return { + "cooldown_active": cooldown_active, + "cooldown_reason": reason, + "cooldown_reason_label": _cooldown_reason_label(reason), + "cooldown_until": cooldown_until, + "cooldown_remaining_seconds": cooldown_remaining_seconds, + "recommended_action": _recommended_action(reason), + "counters": _counters_from_state(settings), + } + + +def get_safety_event_context( + settings: UserSetting, + *, + now_utc: datetime | None = None, +) -> dict[str, Any]: + payload = get_safety_state_payload(settings, now_utc=now_utc) + return { + "cooldown_active": bool(payload.get("cooldown_active")), + "cooldown_reason": payload.get("cooldown_reason"), + "cooldown_until": payload.get("cooldown_until"), + "cooldown_remaining_seconds": int(payload.get("cooldown_remaining_seconds") or 0), + "safety_counters": payload.get("counters", {}), + } diff --git a/build/lib/app/services/domains/ingestion/scheduler.py b/build/lib/app/services/domains/ingestion/scheduler.py new file mode 100644 index 0000000..b05e84d --- /dev/null +++ b/build/lib/app/services/domains/ingestion/scheduler.py @@ -0,0 +1,622 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +import logging + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + CrawlRun, + QueueItemStatus, + RunTriggerType, + ScholarProfile, + User, + UserSetting, +) +from app.db.session import get_session_factory +from app.services.domains.ingestion import queue as queue_service +from app.services.domains.ingestion.application import ( + RunAlreadyInProgressError, + RunBlockedBySafetyPolicyError, + ScholarIngestionService, +) +from app.services.domains.settings import application as user_settings_service +from app.services.domains.scholar.source import LiveScholarSource +from app.settings import settings + +logger = logging.getLogger(__name__) + + +def _request_delay_floor_seconds() -> int: + return user_settings_service.resolve_request_delay_minimum( + settings.ingestion_min_request_delay_seconds + ) + + +def _effective_request_delay_seconds(value: int | None) -> int: + floor = _request_delay_floor_seconds() + try: + parsed = int(value) if value is not None else floor + except (TypeError, ValueError): + parsed = floor + return max(floor, parsed) + + +@dataclass(frozen=True) +class _AutoRunCandidate: + user_id: int + run_interval_minutes: int + request_delay_seconds: int + cooldown_until: datetime | None + cooldown_reason: str | None + + +class SchedulerService: + def __init__( + self, + *, + enabled: bool, + tick_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + continuation_queue_enabled: bool, + continuation_base_delay_seconds: int, + continuation_max_delay_seconds: int, + continuation_max_attempts: int, + queue_batch_size: int, + ) -> None: + self._enabled = enabled + self._tick_seconds = max(5, int(tick_seconds)) + self._network_error_retries = max(0, int(network_error_retries)) + self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds)) + self._max_pages_per_scholar = max(1, int(max_pages_per_scholar)) + self._page_size = max(1, int(page_size)) + self._continuation_queue_enabled = bool(continuation_queue_enabled) + self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds)) + self._continuation_max_delay_seconds = max( + self._continuation_base_delay_seconds, + int(continuation_max_delay_seconds), + ) + self._continuation_max_attempts = max(1, int(continuation_max_attempts)) + self._queue_batch_size = max(1, int(queue_batch_size)) + self._task: asyncio.Task[None] | None = None + self._source = LiveScholarSource() + + async def start(self) -> None: + if not self._enabled: + logger.info( + "scheduler.disabled", + extra={ + "event": "scheduler.disabled", + }, + ) + return + if self._task is not None: + return + self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler") + logger.info( + "scheduler.started", + extra={ + "event": "scheduler.started", + "tick_seconds": self._tick_seconds, + "network_error_retries": self._network_error_retries, + "retry_backoff_seconds": self._retry_backoff_seconds, + "max_pages_per_scholar": self._max_pages_per_scholar, + "page_size": self._page_size, + "continuation_queue_enabled": self._continuation_queue_enabled, + "continuation_base_delay_seconds": self._continuation_base_delay_seconds, + "continuation_max_delay_seconds": self._continuation_max_delay_seconds, + "continuation_max_attempts": self._continuation_max_attempts, + "queue_batch_size": self._queue_batch_size, + }, + ) + + async def stop(self) -> None: + if self._task is None: + return + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + finally: + self._task = None + logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"}) + + async def _run_loop(self) -> None: + while True: + try: + await self._tick_once() + except asyncio.CancelledError: + raise + except Exception: + logger.exception( + "scheduler.tick_failed", + extra={ + "event": "scheduler.tick_failed", + }, + ) + await asyncio.sleep(float(self._tick_seconds)) + + async def _tick_once(self) -> None: + if self._continuation_queue_enabled: + await self._drain_continuation_queue() + + await self._drain_pdf_queue() + + candidates = await self._load_candidates() + if not candidates: + return + now = datetime.now(timezone.utc) + for candidate in candidates: + if not await self._is_due(candidate, now=now): + continue + await self._run_candidate(candidate) + + async def _load_candidate_rows(self) -> list[tuple]: + session_factory = get_session_factory() + async with session_factory() as session: + result = await session.execute( + select( + UserSetting.user_id, + UserSetting.run_interval_minutes, + UserSetting.request_delay_seconds, + UserSetting.scrape_cooldown_until, + UserSetting.scrape_cooldown_reason, + ) + .join(User, User.id == UserSetting.user_id) + .where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True)) + .order_by(UserSetting.user_id.asc()) + ) + return result.all() + + @staticmethod + def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None: + user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row + if cooldown_until is not None and cooldown_until.tzinfo is None: + cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) + if cooldown_until is not None and cooldown_until > now_utc: + logger.info( + "scheduler.run_skipped_safety_cooldown_precheck", + extra={ + "event": "scheduler.run_skipped_safety_cooldown_precheck", + "user_id": int(user_id), + "reason": cooldown_reason, + "cooldown_until": cooldown_until, + "cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()), + "metric_name": "scheduler_run_skipped_safety_cooldown_total", + "metric_value": 1, + }, + ) + return None + return _AutoRunCandidate( + user_id=int(user_id), + run_interval_minutes=int(run_interval_minutes), + request_delay_seconds=_effective_request_delay_seconds(request_delay_seconds), + cooldown_until=cooldown_until, + cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None), + ) + + async def _load_candidates(self) -> list[_AutoRunCandidate]: + if not settings.ingestion_automation_allowed: + return [] + rows = await self._load_candidate_rows() + now_utc = datetime.now(timezone.utc) + candidates: list[_AutoRunCandidate] = [] + for row in rows: + candidate = self._candidate_from_row(row, now_utc=now_utc) + if candidate is not None: + candidates.append(candidate) + return candidates + + async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool: + session_factory = get_session_factory() + async with session_factory() as session: + result = await session.execute( + select(CrawlRun.start_dt) + .where( + CrawlRun.user_id == candidate.user_id, + ) + .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) + .limit(1) + ) + last_run = result.scalar_one_or_none() + + if last_run is None: + return True + + next_due_dt = last_run + timedelta( + minutes=candidate.run_interval_minutes + ) + return now >= next_due_dt + + async def _run_candidate_ingestion( + self, + *, + candidate: _AutoRunCandidate, + ): + session_factory = get_session_factory() + async with session_factory() as session: + ingestion = ScholarIngestionService(source=self._source) + try: + return await ingestion.run_for_user( + session, + user_id=candidate.user_id, + trigger_type=RunTriggerType.SCHEDULED, + request_delay_seconds=candidate.request_delay_seconds, + network_error_retries=self._network_error_retries, + retry_backoff_seconds=self._retry_backoff_seconds, + max_pages_per_scholar=self._max_pages_per_scholar, + page_size=self._page_size, + auto_queue_continuations=self._continuation_queue_enabled, + queue_delay_seconds=self._continuation_base_delay_seconds, + alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, + alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, + alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, + ) + except RunAlreadyInProgressError: + await session.rollback() + logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id}) + return None + except RunBlockedBySafetyPolicyError as exc: + await session.rollback() + logger.info( + "scheduler.run_skipped_safety_cooldown", + extra={ + "event": "scheduler.run_skipped_safety_cooldown", + "user_id": candidate.user_id, + "reason": exc.safety_state.get("cooldown_reason"), + "cooldown_until": exc.safety_state.get("cooldown_until"), + "cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"), + "metric_name": "scheduler_run_skipped_safety_cooldown_total", + "metric_value": 1, + }, + ) + return None + except Exception: + await session.rollback() + logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id}) + return None + + async def _run_candidate(self, candidate: _AutoRunCandidate) -> None: + run_summary = await self._run_candidate_ingestion(candidate=candidate) + if run_summary is None: + return + logger.info( + "scheduler.run_completed", + extra={ + "event": "scheduler.run_completed", + "user_id": candidate.user_id, + "run_id": run_summary.crawl_run_id, + "status": run_summary.status.value, + "scholar_count": run_summary.scholar_count, + "new_publication_count": run_summary.new_publication_count, + }, + ) + + async def _drain_continuation_queue(self) -> None: + now = datetime.now(timezone.utc) + session_factory = get_session_factory() + async with session_factory() as session: + jobs = await queue_service.list_due_jobs( + session, + now=now, + limit=self._queue_batch_size, + ) + for job in jobs: + await self._run_queue_job(job) + + async def _drop_queue_job_if_max_attempts( + self, + job: queue_service.ContinuationQueueJob, + ) -> bool: + if job.attempt_count < self._continuation_max_attempts: + return False + session_factory = get_session_factory() + async with session_factory() as session: + dropped = await queue_service.mark_dropped( + session, + job_id=job.id, + reason="max_attempts_reached", + ) + await session.commit() + if dropped is not None: + logger.warning( + "scheduler.queue_item_dropped_max_attempts", + extra={ + "event": "scheduler.queue_item_dropped_max_attempts", + "queue_item_id": job.id, + "user_id": job.user_id, + "scholar_profile_id": job.scholar_profile_id, + "attempt_count": job.attempt_count, + "max_attempts": self._continuation_max_attempts, + }, + ) + return True + + async def _mark_queue_job_retrying( + self, + job: queue_service.ContinuationQueueJob, + ) -> bool: + session_factory = get_session_factory() + async with session_factory() as session: + queue_item = await queue_service.mark_retrying(session, job_id=job.id) + await session.commit() + if queue_item is None: + return False + if queue_item.status == QueueItemStatus.DROPPED.value: + return False + return True + + async def _queue_job_has_available_scholar( + self, + job: queue_service.ContinuationQueueJob, + ) -> bool: + session_factory = get_session_factory() + async with session_factory() as session: + scholar_result = await session.execute( + select(ScholarProfile.id).where( + ScholarProfile.user_id == job.user_id, + ScholarProfile.id == job.scholar_profile_id, + ScholarProfile.is_enabled.is_(True), + ) + ) + scholar_id = scholar_result.scalar_one_or_none() + if scholar_id is not None: + return True + dropped = await queue_service.mark_dropped( + session, + job_id=job.id, + reason="scholar_unavailable", + ) + await session.commit() + if dropped is not None: + logger.info( + "scheduler.queue_item_dropped_scholar_unavailable", + extra={ + "event": "scheduler.queue_item_dropped_scholar_unavailable", + "queue_item_id": job.id, + "user_id": job.user_id, + "scholar_profile_id": job.scholar_profile_id, + }, + ) + return False + + async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None: + session_factory = get_session_factory() + async with session_factory() as recovery_session: + await queue_service.reschedule_job( + recovery_session, + job_id=job.id, + delay_seconds=max(self._tick_seconds, 15), + reason="user_run_lock_active", + error="run_already_in_progress", + ) + await recovery_session.commit() + logger.info( + "scheduler.queue_item_deferred_lock", + extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id}, + ) + + async def _reschedule_queue_job_safety_cooldown( + self, + job: queue_service.ContinuationQueueJob, + exc: RunBlockedBySafetyPolicyError, + ) -> None: + cooldown_remaining_seconds = max( + self._tick_seconds, + int(exc.safety_state.get("cooldown_remaining_seconds") or 0), + ) + session_factory = get_session_factory() + async with session_factory() as recovery_session: + await queue_service.reschedule_job( + recovery_session, + job_id=job.id, + delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds), + reason="scrape_safety_cooldown", + error=str(exc.message), + ) + await recovery_session.commit() + logger.info( + "scheduler.queue_item_deferred_safety_cooldown", + extra={ + "event": "scheduler.queue_item_deferred_safety_cooldown", + "queue_item_id": job.id, + "user_id": job.user_id, + "reason": exc.safety_state.get("cooldown_reason"), + "cooldown_remaining_seconds": cooldown_remaining_seconds, + "metric_name": "scheduler_queue_item_deferred_safety_cooldown_total", + "metric_value": 1, + }, + ) + + async def _reschedule_queue_job_after_exception( + self, + job: queue_service.ContinuationQueueJob, + *, + exc: Exception, + ) -> None: + session_factory = get_session_factory() + async with session_factory() as recovery_session: + queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id) + if queue_item is None: + await recovery_session.commit() + return + if int(queue_item.attempt_count) >= self._continuation_max_attempts: + await queue_service.mark_dropped( + recovery_session, + job_id=job.id, + reason="scheduler_exception_max_attempts", + error=str(exc), + ) + await recovery_session.commit() + logger.warning( + "scheduler.queue_item_dropped_after_exception", + extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count}, + ) + return + delay_seconds = queue_service.compute_backoff_seconds( + base_seconds=self._continuation_base_delay_seconds, + attempt_count=int(queue_item.attempt_count), + max_seconds=self._continuation_max_delay_seconds, + ) + await queue_service.reschedule_job( + recovery_session, + job_id=job.id, + delay_seconds=delay_seconds, + reason="scheduler_exception", + error=str(exc), + ) + await recovery_session.commit() + logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id}) + + async def _run_ingestion_for_queue_job( + self, + job: queue_service.ContinuationQueueJob, + ): + session_factory = get_session_factory() + async with session_factory() as session: + request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id) + ingestion = ScholarIngestionService(source=self._source) + try: + return await ingestion.run_for_user( + session, + user_id=job.user_id, + trigger_type=RunTriggerType.SCHEDULED, + request_delay_seconds=request_delay_seconds, + network_error_retries=self._network_error_retries, + retry_backoff_seconds=self._retry_backoff_seconds, + rate_limit_retries=settings.ingestion_rate_limit_retries, + rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds, + max_pages_per_scholar=self._max_pages_per_scholar, + page_size=self._page_size, + scholar_profile_ids={job.scholar_profile_id}, + start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart}, + auto_queue_continuations=self._continuation_queue_enabled, + queue_delay_seconds=self._continuation_base_delay_seconds, + alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, + alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, + alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, + ) + except RunAlreadyInProgressError: + await session.rollback() + await self._reschedule_queue_job_lock_active(job) + except RunBlockedBySafetyPolicyError as exc: + await session.rollback() + await self._reschedule_queue_job_safety_cooldown(job, exc) + except Exception as exc: + await session.rollback() + await self._reschedule_queue_job_after_exception(job, exc=exc) + return None + + @staticmethod + def _log_queue_item_resolved( + *, + event_name: str, + job: queue_service.ContinuationQueueJob, + run_summary, + attempt_count: int | None = None, + delay_seconds: int | None = None, + ) -> None: + payload = { + "event": event_name, + "queue_item_id": job.id, + "user_id": job.user_id, + "run_id": run_summary.crawl_run_id, + "status": run_summary.status.value, + } + if attempt_count is not None: + payload["attempt_count"] = attempt_count + if delay_seconds is not None: + payload["delay_seconds"] = delay_seconds + logger.info(event_name, extra=payload) + + async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None: + session_factory = get_session_factory() + async with session_factory() as session: + if int(run_summary.failed_count) <= 0: + queue_item = await queue_service.reset_attempt_count(session, job_id=job.id) + await session.commit() + if queue_item is None: + self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary) + return + self._log_queue_item_resolved( + event_name="scheduler.queue_item_progressed", + job=job, + run_summary=run_summary, + attempt_count=int(queue_item.attempt_count), + ) + return + queue_item = await queue_service.increment_attempt_count(session, job_id=job.id) + if queue_item is None: + await session.commit() + self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary) + return + if int(queue_item.attempt_count) >= self._continuation_max_attempts: + await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run") + await session.commit() + logger.warning( + "scheduler.queue_item_dropped_max_attempts_after_run", + extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value}, + ) + return + delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds) + await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error) + await session.commit() + self._log_queue_item_resolved( + event_name="scheduler.queue_item_rescheduled_failed", + job=job, + run_summary=run_summary, + attempt_count=int(queue_item.attempt_count), + delay_seconds=delay_seconds, + ) + + async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None: + if await self._drop_queue_job_if_max_attempts(job): + return + if not await self._mark_queue_job_retrying(job): + return + if not await self._queue_job_has_available_scholar(job): + return + run_summary = await self._run_ingestion_for_queue_job(job) + if run_summary is None: + return + await self._finalize_queue_job_after_run(job, run_summary) + + async def _drain_pdf_queue(self) -> None: + from app.services.domains.publications.pdf_queue import drain_ready_jobs + + session_factory = get_session_factory() + async with session_factory() as session: + try: + processed = await drain_ready_jobs( + session, + limit=settings.scheduler_pdf_queue_batch_size, + max_attempts=settings.pdf_auto_retry_max_attempts, + ) + if processed > 0: + logger.info("scheduler.pdf_queue_drain_completed", extra={ + "event": "scheduler.pdf_queue_drain_completed", + "processed_count": processed, + }) + except Exception: + logger.exception("scheduler.pdf_queue_drain_failed", extra={ + "event": "scheduler.pdf_queue_drain_failed", + }) + + async def _load_request_delay_for_user( + self, + db_session: AsyncSession, + *, + user_id: int, + ) -> int: + result = await db_session.execute( + select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id) + ) + delay = result.scalar_one_or_none() + return _effective_request_delay_seconds(delay) diff --git a/build/lib/app/services/domains/ingestion/types.py b/build/lib/app/services/domains/ingestion/types.py new file mode 100644 index 0000000..09d87f1 --- /dev/null +++ b/build/lib/app/services/domains/ingestion/types.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from app.db.models import RunStatus +from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate +from app.services.domains.scholar.source import FetchResult + + +@dataclass(frozen=True) +class RunExecutionSummary: + crawl_run_id: int + status: RunStatus + scholar_count: int + succeeded_count: int + failed_count: int + partial_count: int + new_publication_count: int + + +@dataclass(frozen=True) +class PagedParseResult: + fetch_result: FetchResult + parsed_page: ParsedProfilePage + first_page_fetch_result: FetchResult + first_page_parsed_page: ParsedProfilePage + first_page_fingerprint_sha256: str | None + publications: list[PublicationCandidate] + attempt_log: list[dict[str, Any]] + page_logs: list[dict[str, Any]] + pages_fetched: int + pages_attempted: int + has_more_remaining: bool + pagination_truncated_reason: str | None + continuation_cstart: int | None + skipped_no_change: bool + discovered_publication_count: int + + +@dataclass +class RunProgress: + succeeded_count: int = 0 + failed_count: int = 0 + partial_count: int = 0 + scholar_results: list[dict[str, Any]] = field(default_factory=list) + + +@dataclass(frozen=True) +class ScholarProcessingOutcome: + result_entry: dict[str, Any] + succeeded_count_delta: int + failed_count_delta: int + partial_count_delta: int + discovered_publication_count: int + + +@dataclass(frozen=True) +class RunFailureSummary: + failed_state_counts: dict[str, int] + failed_reason_counts: dict[str, int] + scrape_failure_counts: dict[str, int] + retries_scheduled_count: int + scholars_with_retries_count: int + retry_exhausted_count: int + + +@dataclass(frozen=True) +class RunAlertSummary: + blocked_failure_count: int + network_failure_count: int + blocked_failure_threshold: int + network_failure_threshold: int + retry_scheduled_threshold: int + alert_flags: dict[str, bool] + + +@dataclass +class PagedLoopState: + fetch_result: FetchResult + parsed_page: ParsedProfilePage + attempt_log: list[dict[str, Any]] + page_logs: list[dict[str, Any]] + publications: list[PublicationCandidate] + pages_fetched: int + pages_attempted: int + current_cstart: int + next_cstart: int + has_more_remaining: bool = False + pagination_truncated_reason: str | None = None + continuation_cstart: int | None = None + discovered_publication_count: int = 0 + + +class RunAlreadyInProgressError(RuntimeError): + """Raised when a run lock for a user is already held by another process.""" + + +class RunBlockedBySafetyPolicyError(RuntimeError): + def __init__( + self, + *, + code: str, + message: str, + safety_state: dict[str, Any], + ) -> None: + super().__init__(message) + self.code = code + self.message = message + self.safety_state = safety_state diff --git a/build/lib/app/services/domains/openalex/__init__.py b/build/lib/app/services/domains/openalex/__init__.py new file mode 100644 index 0000000..a51555a --- /dev/null +++ b/build/lib/app/services/domains/openalex/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +import logging + +logger = logging.getLogger(__name__) diff --git a/build/lib/app/services/domains/openalex/client.py b/build/lib/app/services/domains/openalex/client.py new file mode 100644 index 0000000..c62d876 --- /dev/null +++ b/build/lib/app/services/domains/openalex/client.py @@ -0,0 +1,135 @@ +import asyncio +import logging +from typing import Any, Mapping + +import httpx +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_exponential, +) + +from app.services.domains.openalex.types import OpenAlexWork + +logger = logging.getLogger(__name__) + +OPENALEX_BASE_URL = "https://api.openalex.org" + + +class OpenAlexClientError(Exception): + pass + + +class OpenAlexRateLimitError(OpenAlexClientError): + pass + + +class OpenAlexClient: + def __init__( + self, + api_key: str | None = None, + mailto: str | None = None, + timeout: float = 10.0, + ) -> None: + self.api_key = api_key + self.mailto = mailto + self.timeout = timeout + + @property + def _base_params(self) -> dict[str, str]: + params = {} + if self.mailto: + params["mailto"] = self.mailto + if self.api_key: + params["api_key"] = self.api_key + return params + + @retry( + retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException, OpenAlexRateLimitError)), + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=10), + reraise=True, + ) + async def get_work_by_doi(self, doi: str) -> OpenAlexWork | None: + """Fetch a single work by DOI directly.""" + clean_doi = doi.replace("https://doi.org/", "") + if not clean_doi: + return None + + url = f"{OPENALEX_BASE_URL}/works/{clean_doi}" + + headers = {} + if self.mailto: + headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})" + else: + headers["User-Agent"] = "scholar-scraper/1.0" + + async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client: + response = await client.get(url, params=self._base_params) + + if response.status_code == 404: + return None + if response.status_code == 429: + raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI") + if response.status_code >= 400: + logger.warning("OpenAlex API error: %s %s", response.status_code, response.text) + raise OpenAlexClientError(f"API Error {response.status_code}") + + data = response.json() + return OpenAlexWork.from_api_dict(data) + + @retry( + retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException, OpenAlexRateLimitError)), + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=10), + reraise=True, + ) + async def get_works_by_filter( + self, + filters: dict[str, str], + limit: int = 50, + ) -> list[OpenAlexWork]: + """ + Fetch works using the ?filter= query parameter. + Supports fetching multiple records by joining filters with | (OR logic). + """ + if not filters: + return [] + + # Example: {"doi": "10.foo|10.bar", "title.search": "query"} + filter_str = ",".join(f"{k}:{v}" for k, v in filters.items()) + + params = self._base_params.copy() + params["filter"] = filter_str + params["per-page"] = str(limit) + + url = f"{OPENALEX_BASE_URL}/works" + + headers = {} + if self.mailto: + headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})" + else: + headers["User-Agent"] = "scholar-scraper/1.0" + + async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client: + response = await client.get(url, params=params) + + if response.status_code == 429: + raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list") + if response.status_code >= 400: + logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text) + raise OpenAlexClientError(f"API Error {response.status_code}") + + data = response.json() + results = data.get("results") or [] + + parsed_works = [] + for raw_work in results: + try: + parsed_works.append(OpenAlexWork.from_api_dict(raw_work)) + except Exception as e: + logger.warning("Failed to parse OpenAlex raw dict: %s", e) + continue + + return parsed_works diff --git a/build/lib/app/services/domains/openalex/matching.py b/build/lib/app/services/domains/openalex/matching.py new file mode 100644 index 0000000..cfa511c --- /dev/null +++ b/build/lib/app/services/domains/openalex/matching.py @@ -0,0 +1,108 @@ +import logging +import re + +from rapidfuzz import fuzz + +from app.services.domains.openalex.types import OpenAlexWork + +logger = logging.getLogger(__name__) + +# A minimum similarity score out of 100 for a title to be considered a match candidate. +TITLE_MATCH_THRESHOLD = 90.0 +# The margin within the top score where a secondary tiebreaker (author/year) is necessary. +TIEBREAKER_MARGIN = 5.0 + + +def _clean_string(s: str | None) -> str: + if not s: + return "" + # Strip non-alphanumeric (keep spaces), lowercase, and collapse whitespace + cleaned = re.sub(r"[^a-z0-9\s]", " ", s.lower()) + return " ".join(cleaned.split()) + + +def _author_overlap_score(target_authors: str | None, candidate_authors: list[str]) -> bool: + if not target_authors or not candidate_authors: + return False + + target_clean = _clean_string(target_authors) + if not target_clean: + return False + + for candidate in candidate_authors: + cand_clean = _clean_string(candidate) + if cand_clean and (cand_clean in target_clean or target_clean in cand_clean): + return True + # Alternatively check rapidfuzz token_set_ratio + if cand_clean and fuzz.token_set_ratio(target_clean, cand_clean) > 80: + return True + + return False + + +def find_best_match( + target_title: str, + target_year: int | None, + target_authors: str | None, + candidates: list[OpenAlexWork], +) -> OpenAlexWork | None: + """ + Finds the best matching OpenAlexWork from a list of candidates, prioritizing title similarity (>90%) + with year and author overlap as tiebreakers for close candidates. + """ + if not target_title or not candidates: + return None + + clean_target = _clean_string(target_title) + if not clean_target: + return None + + scored_candidates: list[tuple[float, OpenAlexWork]] = [] + + for cand in candidates: + if not cand.title: + continue + + clean_cand = _clean_string(cand.title) + + # Primary sort: string similarity ratio + score = fuzz.ratio(clean_target, clean_cand) + + if score >= TITLE_MATCH_THRESHOLD: + scored_candidates.append((score, cand)) + + if not scored_candidates: + return None + + # Sort descending by score + scored_candidates.sort(key=lambda x: x[0], reverse=True) + + best_score = scored_candidates[0][0] + + # Extract all candidates within the tiebreaker margin + top_scored_candidates = [ + (score, cand) for score, cand in scored_candidates + if best_score - score <= TIEBREAKER_MARGIN + ] + + if len(top_scored_candidates) == 1: + return top_scored_candidates[0][1] + + # We have a tie or near-tie. Use year and author overlap to break the tie. + # Score candidates: +1 for year match (within 1 year), +1 for author overlap + tiebreaker_scores: list[tuple[int, float, OpenAlexWork]] = [] + + for original_score, cand in top_scored_candidates: + tb_score = 0 + if target_year is not None and cand.publication_year is not None: + if abs(target_year - cand.publication_year) <= 1: + tb_score += 1 + + candidate_author_names = [a.display_name for a in cand.authors if a.display_name] + if _author_overlap_score(target_authors, candidate_author_names): + tb_score += 1 + + tiebreaker_scores.append((tb_score, original_score, cand)) + + tiebreaker_scores.sort(key=lambda x: (x[0], x[1]), reverse=True) + return tiebreaker_scores[0][2] diff --git a/build/lib/app/services/domains/openalex/types.py b/build/lib/app/services/domains/openalex/types.py new file mode 100644 index 0000000..0d86754 --- /dev/null +++ b/build/lib/app/services/domains/openalex/types.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Mapping + + +@dataclass(frozen=True) +class OpenAlexAuthor: + openalex_id: str | None + display_name: str | None + + +@dataclass(frozen=True) +class OpenAlexWork: + openalex_id: str + doi: str | None + pmid: str | None + pmcid: str | None + title: str | None + publication_year: int | None + cited_by_count: int + is_oa: bool + oa_url: str | None + authors: list[OpenAlexAuthor] = field(default_factory=list) + raw_data: Mapping[str, Any] = field(default_factory=dict, repr=False) + + @classmethod + def from_api_dict(cls, data: Mapping[str, Any]) -> OpenAlexWork: + ids = data.get("ids") or {} + + # Extract DOI without the https://doi.org/ prefix + doi = ids.get("doi") + if doi and doi.startswith("https://doi.org/"): + doi = doi[16:] + + # Extract PMID without the url prefix + pmid = ids.get("pmid") + if pmid and pmid.startswith("https://pubmed.ncbi.nlm.nih.gov/"): + pmid = pmid[32:] + + # Extract PMCID without the url prefix + pmcid = ids.get("pmcid") + if pmcid and pmcid.startswith("https://www.ncbi.nlm.nih.gov/pmc/articles/"): + pmcid = pmcid[42:] + + open_access = data.get("open_access") or {} + + authors = [] + for authorship in data.get("authorships") or []: + author_data = authorship.get("author") or {} + authors.append( + OpenAlexAuthor( + openalex_id=author_data.get("id"), + display_name=author_data.get("display_name"), + ) + ) + + return cls( + openalex_id=data.get("id", ""), + doi=doi, + pmid=pmid, + pmcid=pmcid, + title=data.get("title"), + publication_year=data.get("publication_year"), + cited_by_count=data.get("cited_by_count", 0), + is_oa=bool(open_access.get("is_oa")), + oa_url=open_access.get("oa_url"), + authors=authors, + raw_data=dict(data), + ) diff --git a/build/lib/app/services/domains/portability/__init__.py b/build/lib/app/services/domains/portability/__init__.py new file mode 100644 index 0000000..28aae8b --- /dev/null +++ b/build/lib/app/services/domains/portability/__init__.py @@ -0,0 +1 @@ +from app.services.domains.portability.application import * diff --git a/build/lib/app/services/domains/portability/application.py b/build/lib/app/services/domains/portability/application.py new file mode 100644 index 0000000..7e0b56e --- /dev/null +++ b/build/lib/app/services/domains/portability/application.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Publication +from app.services.domains.portability.constants import ( + EXPORT_SCHEMA_VERSION, + MAX_IMPORT_PUBLICATIONS, + MAX_IMPORT_SCHOLARS, +) +from app.services.domains.portability.exporting import export_user_data +from app.services.domains.portability.normalize import _validate_import_sizes +from app.services.domains.portability.publication_import import ( + _build_imported_publication_input, + _initialize_import_counters, + _upsert_imported_publication, +) +from app.services.domains.portability.scholar_import import _upsert_imported_scholars +from app.services.domains.portability.types import ImportExportError, ImportedPublicationInput + + +async def import_user_data( + db_session: AsyncSession, + *, + user_id: int, + scholars: list[dict[str, Any]], + publications: list[dict[str, Any]], +) -> dict[str, int]: + _validate_import_sizes(scholars=scholars, publications=publications) + scholar_map, counters = await _upsert_imported_scholars( + db_session, + user_id=user_id, + scholars=scholars, + ) + cluster_cache: dict[str, Publication | None] = {} + fingerprint_cache: dict[str, Publication | None] = {} + _initialize_import_counters(counters) + for item in publications: + parsed_item = _build_imported_publication_input( + item=item, + scholar_map=scholar_map, + ) + if parsed_item is None: + counters["skipped_records"] += 1 + continue + await _upsert_imported_publication( + db_session, + payload=parsed_item, + cluster_cache=cluster_cache, + fingerprint_cache=fingerprint_cache, + counters=counters, + ) + await db_session.commit() + return counters + + +__all__ = [ + "EXPORT_SCHEMA_VERSION", + "MAX_IMPORT_SCHOLARS", + "MAX_IMPORT_PUBLICATIONS", + "ImportExportError", + "ImportedPublicationInput", + "export_user_data", + "import_user_data", +] diff --git a/build/lib/app/services/domains/portability/constants.py b/build/lib/app/services/domains/portability/constants.py new file mode 100644 index 0000000..6815347 --- /dev/null +++ b/build/lib/app/services/domains/portability/constants.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +import re + +EXPORT_SCHEMA_VERSION = 1 +MAX_IMPORT_SCHOLARS = 10_000 +MAX_IMPORT_PUBLICATIONS = 100_000 +WORD_RE = re.compile(r"[a-z0-9]+") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") diff --git a/build/lib/app/services/domains/portability/exporting.py b/build/lib/app/services/domains/portability/exporting.py new file mode 100644 index 0000000..5c89006 --- /dev/null +++ b/build/lib/app/services/domains/portability/exporting.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Publication, ScholarProfile, ScholarPublication +from app.services.domains.portability.constants import EXPORT_SCHEMA_VERSION + + +def _exported_at_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]: + return { + "scholar_id": profile.scholar_id, + "display_name": profile.display_name, + "is_enabled": bool(profile.is_enabled), + "profile_image_override_url": profile.profile_image_override_url, + } + + +def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]: + ( + scholar_id, + cluster_id, + fingerprint_sha256, + title_raw, + year, + citation_count, + author_text, + venue_text, + pub_url, + pdf_url, + is_read, + ) = row + return { + "scholar_id": scholar_id, + "cluster_id": cluster_id, + "fingerprint_sha256": fingerprint_sha256, + "title": title_raw, + "year": year, + "citation_count": int(citation_count or 0), + "author_text": author_text, + "venue_text": venue_text, + "pub_url": pub_url, + "pdf_url": pdf_url, + "is_read": bool(is_read), + } + + +async def export_user_data( + db_session: AsyncSession, + *, + user_id: int, +) -> dict[str, Any]: + scholars_result = await db_session.execute( + select(ScholarProfile) + .where(ScholarProfile.user_id == user_id) + .order_by(ScholarProfile.id.asc()) + ) + publication_result = await db_session.execute( + select( + ScholarProfile.scholar_id, + Publication.cluster_id, + Publication.fingerprint_sha256, + Publication.title_raw, + Publication.year, + Publication.citation_count, + Publication.author_text, + Publication.venue_text, + Publication.pub_url, + Publication.pdf_url, + ScholarPublication.is_read, + ) + .join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id) + .join(Publication, Publication.id == ScholarPublication.publication_id) + .where(ScholarProfile.user_id == user_id) + .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) + ) + scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()] + publications = [_serialize_export_publication(row) for row in publication_result.all()] + return { + "schema_version": EXPORT_SCHEMA_VERSION, + "exported_at": _exported_at_iso(), + "scholars": scholars, + "publications": publications, + } diff --git a/build/lib/app/services/domains/portability/normalize.py b/build/lib/app/services/domains/portability/normalize.py new file mode 100644 index 0000000..60dd619 --- /dev/null +++ b/build/lib/app/services/domains/portability/normalize.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import hashlib +from typing import Any + +from app.services.domains.ingestion.application import normalize_title +from app.services.domains.portability.constants import ( + MAX_IMPORT_PUBLICATIONS, + MAX_IMPORT_SCHOLARS, + SHA256_RE, + WORD_RE, +) +from app.services.domains.portability.types import ImportExportError + + +def _normalize_optional_text(value: Any) -> str | None: + if value is None: + return None + normalized = str(value).strip() + return normalized or None + + +def _normalize_optional_year(value: Any) -> int | None: + if value is None: + return None + try: + year = int(value) + except (TypeError, ValueError): + return None + if year < 1500 or year > 3000: + return None + return year + + +def _normalize_citation_count(value: Any) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + return 0 + return max(0, parsed) + + +def _first_author_last_name(authors_text: str | None) -> str: + if not authors_text: + return "" + first_author = authors_text.split(",", maxsplit=1)[0].strip().lower() + words = WORD_RE.findall(first_author) + if not words: + return "" + return words[-1] + + +def _first_venue_word(venue_text: str | None) -> str: + if not venue_text: + return "" + words = WORD_RE.findall(venue_text.lower()) + if not words: + return "" + return words[0] + + +def _build_fingerprint( + *, + title: str, + year: int | None, + author_text: str | None, + venue_text: str | None, +) -> str: + canonical = "|".join( + [ + normalize_title(title), + str(year) if year is not None else "", + _first_author_last_name(author_text), + _first_venue_word(venue_text), + ] + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _resolve_fingerprint( + *, + title: str, + year: int | None, + author_text: str | None, + venue_text: str | None, + provided_fingerprint: Any, +) -> str: + normalized = _normalize_optional_text(provided_fingerprint) + if normalized and SHA256_RE.fullmatch(normalized.lower()): + return normalized.lower() + return _build_fingerprint( + title=title, + year=year, + author_text=author_text, + venue_text=venue_text, + ) + + +def _validate_import_sizes( + *, + scholars: list[dict[str, Any]], + publications: list[dict[str, Any]], +) -> None: + if len(scholars) > MAX_IMPORT_SCHOLARS: + raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).") + if len(publications) > MAX_IMPORT_PUBLICATIONS: + raise ImportExportError( + f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})." + ) diff --git a/build/lib/app/services/domains/portability/publication_import.py b/build/lib/app/services/domains/portability/publication_import.py new file mode 100644 index 0000000..7654586 --- /dev/null +++ b/build/lib/app/services/domains/portability/publication_import.py @@ -0,0 +1,370 @@ +from __future__ import annotations + +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Publication, ScholarProfile, ScholarPublication +from app.services.domains.ingestion.application import build_publication_url, normalize_title +from app.services.domains.doi.normalize import normalize_doi +from app.services.domains.portability.normalize import ( + _normalize_citation_count, + _normalize_optional_text, + _normalize_optional_year, + _resolve_fingerprint, +) +from app.services.domains.portability.types import ImportedPublicationInput + + +async def _find_publication_by_cluster( + db_session: AsyncSession, + *, + cluster_id: str, +) -> Publication | None: + result = await db_session.execute( + select(Publication).where(Publication.cluster_id == cluster_id) + ) + return result.scalar_one_or_none() + + +async def _find_publication_by_fingerprint( + db_session: AsyncSession, + *, + fingerprint_sha256: str, +) -> Publication | None: + result = await db_session.execute( + select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256) + ) + return result.scalar_one_or_none() + + +async def _find_linked_publication_by_title( + db_session: AsyncSession, + *, + scholar_profile_id: int, + title: str, +) -> Publication | None: + normalized_title = normalize_title(title) + result = await db_session.execute( + select(Publication) + .join( + ScholarPublication, + ScholarPublication.publication_id == Publication.id, + ) + .where( + ScholarPublication.scholar_profile_id == scholar_profile_id, + Publication.title_normalized == normalized_title, + ) + .order_by(Publication.id.asc()) + .limit(1) + ) + return result.scalar_one_or_none() + + +def _apply_imported_publication_values( + *, + publication: Publication, + title: str, + year: int | None, + citation_count: int, + author_text: str | None, + venue_text: str | None, + pub_url: str | None, + pdf_url: str | None, + cluster_id: str | None, +) -> bool: + updated = False + if cluster_id and publication.cluster_id != cluster_id: + publication.cluster_id = cluster_id + updated = True + if publication.title_raw != title: + publication.title_raw = title + publication.title_normalized = normalize_title(title) + updated = True + if publication.year != year: + publication.year = year + updated = True + if int(publication.citation_count or 0) != citation_count: + publication.citation_count = citation_count + updated = True + if publication.author_text != author_text: + publication.author_text = author_text + updated = True + if publication.venue_text != venue_text: + publication.venue_text = venue_text + updated = True + if pub_url and publication.pub_url != pub_url: + publication.pub_url = pub_url + updated = True + if pdf_url and publication.pdf_url != pdf_url: + publication.pdf_url = pdf_url + updated = True + return updated + + +def _new_publication( + *, + cluster_id: str | None, + fingerprint_sha256: str, + title: str, + year: int | None, + citation_count: int, + author_text: str | None, + venue_text: str | None, + pub_url: str | None, + pdf_url: str | None, +) -> Publication: + return Publication( + cluster_id=cluster_id, + fingerprint_sha256=fingerprint_sha256, + title_raw=title, + title_normalized=normalize_title(title), + year=year, + citation_count=citation_count, + author_text=author_text, + venue_text=venue_text, + pub_url=pub_url, + pdf_url=pdf_url, + ) + + +async def _resolve_publication_for_import( + db_session: AsyncSession, + *, + scholar_profile_id: int, + title: str, + cluster_id: str | None, + fingerprint_sha256: str, + cluster_cache: dict[str, Publication | None], + fingerprint_cache: dict[str, Publication | None], +) -> Publication | None: + if cluster_id: + if cluster_id not in cluster_cache: + cluster_cache[cluster_id] = await _find_publication_by_cluster( + db_session, + cluster_id=cluster_id, + ) + if cluster_cache[cluster_id] is not None: + return cluster_cache[cluster_id] + if fingerprint_sha256 not in fingerprint_cache: + fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint( + db_session, + fingerprint_sha256=fingerprint_sha256, + ) + if fingerprint_cache[fingerprint_sha256] is not None: + return fingerprint_cache[fingerprint_sha256] + return await _find_linked_publication_by_title( + db_session, + scholar_profile_id=scholar_profile_id, + title=title, + ) + + +async def _upsert_scholar_publication_link( + db_session: AsyncSession, + *, + scholar_profile_id: int, + publication_id: int, + is_read: bool, +) -> tuple[bool, bool]: + result = await db_session.execute( + select(ScholarPublication).where( + ScholarPublication.scholar_profile_id == scholar_profile_id, + ScholarPublication.publication_id == publication_id, + ) + ) + link = result.scalar_one_or_none() + if link is None: + db_session.add( + ScholarPublication( + scholar_profile_id=scholar_profile_id, + publication_id=publication_id, + is_read=bool(is_read), + ) + ) + return True, False + if bool(link.is_read) == bool(is_read): + return False, False + link.is_read = bool(is_read) + return False, True + + +def _initialize_import_counters(counters: dict[str, int]) -> None: + counters.update( + { + "publications_created": 0, + "publications_updated": 0, + "links_created": 0, + "links_updated": 0, + } + ) + + +def _build_imported_publication_input( + *, + item: dict[str, Any], + scholar_map: dict[str, ScholarProfile], +) -> ImportedPublicationInput | None: + scholar_id = _normalize_optional_text(item.get("scholar_id")) + title = _normalize_optional_text(item.get("title")) + if not scholar_id or not title: + return None + + profile = scholar_map.get(scholar_id) + if profile is None: + return None + + year = _normalize_optional_year(item.get("year")) + author_text = _normalize_optional_text(item.get("author_text")) + venue_text = _normalize_optional_text(item.get("venue_text")) + return ImportedPublicationInput( + profile=profile, + title=title, + year=year, + citation_count=_normalize_citation_count(item.get("citation_count")), + author_text=author_text, + venue_text=venue_text, + cluster_id=_normalize_optional_text(item.get("cluster_id")), + pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))), + doi=normalize_doi(_normalize_optional_text(item.get("doi"))), + pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))), + fingerprint=_resolve_fingerprint( + title=title, + year=year, + author_text=author_text, + venue_text=venue_text, + provided_fingerprint=item.get("fingerprint_sha256"), + ), + is_read=bool(item.get("is_read", False)), + ) + + +def _update_link_counters( + *, + counters: dict[str, int], + link_created: bool, + link_updated: bool, +) -> None: + if link_created: + counters["links_created"] += 1 + if link_updated: + counters["links_updated"] += 1 + + +def _cache_resolved_publication( + *, + publication: Publication, + cluster_id: str | None, + fingerprint_sha256: str, + cluster_cache: dict[str, Publication | None], + fingerprint_cache: dict[str, Publication | None], +) -> None: + if cluster_id: + cluster_cache[cluster_id] = publication + fingerprint_cache[fingerprint_sha256] = publication + + +async def _create_import_publication( + db_session: AsyncSession, + *, + payload: ImportedPublicationInput, +) -> Publication: + publication = _new_publication( + cluster_id=payload.cluster_id, + fingerprint_sha256=payload.fingerprint, + title=payload.title, + year=payload.year, + citation_count=payload.citation_count, + author_text=payload.author_text, + venue_text=payload.venue_text, + pub_url=payload.pub_url, + pdf_url=payload.pdf_url, + ) + db_session.add(publication) + await db_session.flush() + return publication + + +def _update_import_publication( + *, + publication: Publication, + payload: ImportedPublicationInput, +) -> bool: + return _apply_imported_publication_values( + publication=publication, + title=payload.title, + year=payload.year, + citation_count=payload.citation_count, + author_text=payload.author_text, + venue_text=payload.venue_text, + pub_url=payload.pub_url, + pdf_url=payload.pdf_url, + cluster_id=payload.cluster_id, + ) + + +async def _upsert_publication_entity( + db_session: AsyncSession, + *, + payload: ImportedPublicationInput, + cluster_cache: dict[str, Publication | None], + fingerprint_cache: dict[str, Publication | None], +) -> tuple[Publication, bool, bool]: + publication = await _resolve_publication_for_import( + db_session, + scholar_profile_id=int(payload.profile.id), + title=payload.title, + cluster_id=payload.cluster_id, + fingerprint_sha256=payload.fingerprint, + cluster_cache=cluster_cache, + fingerprint_cache=fingerprint_cache, + ) + created = False + updated = False + if publication is None: + publication = await _create_import_publication(db_session, payload=payload) + created = True + else: + updated = _update_import_publication(publication=publication, payload=payload) + + _cache_resolved_publication( + publication=publication, + cluster_id=payload.cluster_id, + fingerprint_sha256=payload.fingerprint, + cluster_cache=cluster_cache, + fingerprint_cache=fingerprint_cache, + ) + return publication, created, updated + + +async def _upsert_imported_publication( + db_session: AsyncSession, + *, + payload: ImportedPublicationInput, + cluster_cache: dict[str, Publication | None], + fingerprint_cache: dict[str, Publication | None], + counters: dict[str, int], +) -> None: + publication, created, updated = await _upsert_publication_entity( + db_session, + payload=payload, + cluster_cache=cluster_cache, + fingerprint_cache=fingerprint_cache, + ) + if created: + counters["publications_created"] += 1 + if updated: + counters["publications_updated"] += 1 + + link_created, link_updated = await _upsert_scholar_publication_link( + db_session, + scholar_profile_id=int(payload.profile.id), + publication_id=int(publication.id), + is_read=payload.is_read, + ) + _update_link_counters( + counters=counters, + link_created=link_created, + link_updated=link_updated, + ) diff --git a/build/lib/app/services/domains/portability/scholar_import.py b/build/lib/app/services/domains/portability/scholar_import.py new file mode 100644 index 0000000..74e6ec1 --- /dev/null +++ b/build/lib/app/services/domains/portability/scholar_import.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ScholarProfile +from app.services.domains.portability.normalize import _normalize_optional_text +from app.services.domains.scholars import application as scholar_service + + +async def _load_user_scholar_map( + db_session: AsyncSession, + *, + user_id: int, +) -> dict[str, ScholarProfile]: + result = await db_session.execute( + select(ScholarProfile).where(ScholarProfile.user_id == user_id) + ) + profiles = list(result.scalars().all()) + return {profile.scholar_id: profile for profile in profiles} + + +def _apply_imported_scholar_values( + *, + profile: ScholarProfile, + display_name: str | None, + profile_image_override_url: str | None, + is_enabled: bool, +) -> bool: + updated = False + if display_name and profile.display_name != display_name: + profile.display_name = display_name + updated = True + if profile.profile_image_override_url != profile_image_override_url: + profile.profile_image_override_url = profile_image_override_url + updated = True + if bool(profile.is_enabled) != bool(is_enabled): + profile.is_enabled = bool(is_enabled) + updated = True + return updated + + +def _new_scholar_profile( + *, + user_id: int, + scholar_id: str, + display_name: str | None, + profile_image_override_url: str | None, + is_enabled: bool, +) -> ScholarProfile: + return ScholarProfile( + user_id=user_id, + scholar_id=scholar_id, + display_name=display_name, + profile_image_override_url=profile_image_override_url, + is_enabled=bool(is_enabled), + ) + + +async def _upsert_imported_scholars( + db_session: AsyncSession, + *, + user_id: int, + scholars: list[dict[str, Any]], +) -> tuple[dict[str, ScholarProfile], dict[str, int]]: + scholar_map = await _load_user_scholar_map(db_session, user_id=user_id) + counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0} + for item in scholars: + try: + scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"])) + display_name = scholar_service.normalize_display_name( + str(item.get("display_name") or "") + ) + override_url = scholar_service.normalize_profile_image_url( + _normalize_optional_text(item.get("profile_image_override_url")) + ) + except (KeyError, scholar_service.ScholarServiceError): + counters["skipped_records"] += 1 + continue + + is_enabled = bool(item.get("is_enabled", True)) + existing = scholar_map.get(scholar_id) + if existing is None: + profile = _new_scholar_profile( + user_id=user_id, + scholar_id=scholar_id, + display_name=display_name, + profile_image_override_url=override_url, + is_enabled=is_enabled, + ) + db_session.add(profile) + scholar_map[scholar_id] = profile + counters["scholars_created"] += 1 + continue + + if _apply_imported_scholar_values( + profile=existing, + display_name=display_name, + profile_image_override_url=override_url, + is_enabled=is_enabled, + ): + counters["scholars_updated"] += 1 + + await db_session.flush() + return scholar_map, counters diff --git a/build/lib/app/services/domains/portability/types.py b/build/lib/app/services/domains/portability/types.py new file mode 100644 index 0000000..2a7f6e8 --- /dev/null +++ b/build/lib/app/services/domains/portability/types.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from app.db.models import ScholarProfile + + +class ImportExportError(ValueError): + """Raised when import/export payload constraints are violated.""" + + +@dataclass(frozen=True) +class ImportedPublicationInput: + profile: ScholarProfile + title: str + year: int | None + citation_count: int + author_text: str | None + venue_text: str | None + cluster_id: str | None + pub_url: str | None + doi: str | None + pdf_url: str | None + fingerprint: str + is_read: bool diff --git a/build/lib/app/services/domains/publication_identifiers/__init__.py b/build/lib/app/services/domains/publication_identifiers/__init__.py new file mode 100644 index 0000000..3a35c66 --- /dev/null +++ b/build/lib/app/services/domains/publication_identifiers/__init__.py @@ -0,0 +1,17 @@ +from app.services.domains.publication_identifiers.application import ( + DisplayIdentifier, + derive_display_identifier_from_values, + overlay_pdf_queue_items_with_display_identifiers, + overlay_publication_items_with_display_identifiers, + sync_identifiers_for_publication_fields, + sync_identifiers_for_publication_resolution, +) + +__all__ = [ + "DisplayIdentifier", + "derive_display_identifier_from_values", + "overlay_pdf_queue_items_with_display_identifiers", + "overlay_publication_items_with_display_identifiers", + "sync_identifiers_for_publication_fields", + "sync_identifiers_for_publication_resolution", +] diff --git a/build/lib/app/services/domains/publication_identifiers/application.py b/build/lib/app/services/domains/publication_identifiers/application.py new file mode 100644 index 0000000..ca57d57 --- /dev/null +++ b/build/lib/app/services/domains/publication_identifiers/application.py @@ -0,0 +1,433 @@ +from __future__ import annotations + +from dataclasses import replace +from typing import TYPE_CHECKING + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Publication, PublicationIdentifier +from app.services.domains.doi.normalize import normalize_doi +from app.services.domains.publication_identifiers.normalize import ( + normalize_arxiv_id, + normalize_pmcid, + normalize_pmid, +) +from app.services.domains.publication_identifiers.types import ( + DisplayIdentifier, + IdentifierCandidate, + IdentifierKind, +) + +if TYPE_CHECKING: + from app.services.domains.publications.pdf_queue import PdfQueueListItem + from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem + +CONFIDENCE_HIGH = 0.98 +CONFIDENCE_MEDIUM = 0.9 +CONFIDENCE_LOW = 0.6 +CONFIDENCE_FALLBACK = 0.4 +PRIORITY_DOI = 400 +PRIORITY_ARXIV = 300 +PRIORITY_PMCID = 200 +PRIORITY_PMID = 100 + + +def derive_display_identifier_from_values( + *, + doi: str | None, + pub_url: str | None = None, + pdf_url: str | None = None, +) -> DisplayIdentifier | None: + candidates = _fallback_candidates_from_values(doi=doi, pub_url=pub_url, pdf_url=pdf_url) + return _best_display_identifier(candidates) + + +def _fallback_candidates_from_values( + *, + doi: str | None, + pub_url: str | None, + pdf_url: str | None, +) -> list[IdentifierCandidate]: + values = [value for value in [pub_url, pdf_url] if value] + candidates = [] + if doi: + normalized_doi = normalize_doi(doi) + if normalized_doi: + candidates.append(_candidate(IdentifierKind.DOI, doi, normalized_doi, "legacy_doi", CONFIDENCE_HIGH, pub_url)) + candidates.extend(_url_identifier_candidates(values=values, source="legacy_urls")) + return _dedup_candidates(candidates) + + +def _url_identifier_candidates(*, values: list[str], source: str) -> list[IdentifierCandidate]: + candidates: list[IdentifierCandidate] = [] + for value in values: + candidates.extend(_url_candidates_for_value(value=value, source=source)) + return candidates + + +def _url_candidates_for_value(*, value: str, source: str) -> list[IdentifierCandidate]: + candidates: list[IdentifierCandidate] = [] + arxiv = normalize_arxiv_id(value) + if arxiv: + candidates.append(_candidate(IdentifierKind.ARXIV, value, arxiv, source, CONFIDENCE_MEDIUM, value)) + pmcid = normalize_pmcid(value) + if pmcid: + candidates.append(_candidate(IdentifierKind.PMCID, value, pmcid, source, CONFIDENCE_LOW, value)) + pmid = normalize_pmid(value) + if pmid: + candidates.append(_candidate(IdentifierKind.PMID, value, pmid, source, CONFIDENCE_FALLBACK, value)) + return candidates + + +def _candidate( + kind: IdentifierKind, + value_raw: str, + value_normalized: str, + source: str, + confidence_score: float, + evidence_url: str | None, +) -> IdentifierCandidate: + return IdentifierCandidate( + kind=kind, + value_raw=value_raw, + value_normalized=value_normalized, + source=source, + confidence_score=float(confidence_score), + evidence_url=evidence_url, + ) + + +def _dedup_candidates(candidates: list[IdentifierCandidate]) -> list[IdentifierCandidate]: + deduped: dict[tuple[str, str], IdentifierCandidate] = {} + for candidate in candidates: + key = (candidate.kind.value, candidate.value_normalized) + current = deduped.get(key) + if current is None or candidate.confidence_score > current.confidence_score: + deduped[key] = candidate + return list(deduped.values()) + + +async def sync_identifiers_for_publication_fields( + db_session: AsyncSession, + *, + publication: Publication, +) -> None: + candidates = _publication_field_candidates(publication) + await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=candidates) + + +async def discover_and_sync_identifiers_for_publication( + db_session: AsyncSession, + *, + publication: Publication, + scholar_label: str, +) -> None: + await sync_identifiers_for_publication_fields(db_session, publication=publication) + + existing_doi = await _existing_identifier_by_kind( + db_session, + publication_id=int(publication.id), + kind=IdentifierKind.DOI.value, + ) + if existing_doi is not None: + return + + from app.services.domains.crossref import application as crossref_service + from app.services.domains.publications.types import UnreadPublicationItem + + item = UnreadPublicationItem( + publication_id=int(publication.id), + scholar_profile_id=0, + scholar_label=scholar_label, + title=str(publication.title_raw or ""), + year=publication.year, + citation_count=publication.citation_count, + venue_text=publication.venue_text, + pub_url=publication.pub_url, + pdf_url=publication.pdf_url, + ) + + discovered_doi = await crossref_service.discover_doi_for_publication(item=item) + if discovered_doi: + normalized_doi = normalize_doi(discovered_doi) + if normalized_doi: + candidate = _candidate( + IdentifierKind.DOI, + discovered_doi, + normalized_doi, + "crossref_api", + CONFIDENCE_MEDIUM, + None, + ) + await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate) + + existing_arxiv = await _existing_identifier_by_kind( + db_session, + publication_id=int(publication.id), + kind=IdentifierKind.ARXIV.value, + ) + if existing_arxiv is None: + from app.services.domains.arxiv import application as arxiv_service + discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item) + if discovered_arxiv: + normalized_arxiv = normalize_arxiv_id(discovered_arxiv) + if normalized_arxiv: + candidate = _candidate( + IdentifierKind.ARXIV, + discovered_arxiv, + normalized_arxiv, + "arxiv_api", + CONFIDENCE_MEDIUM, + None, + ) + await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate) + + +def _publication_field_candidates(publication: Publication) -> list[IdentifierCandidate]: + return _fallback_candidates_from_values( + doi=None, + pub_url=publication.pub_url, + pdf_url=publication.pdf_url, + ) + + +async def sync_identifiers_for_publication_resolution( + db_session: AsyncSession, + *, + publication: Publication, + source: str | None, +) -> None: + candidates = _publication_field_candidates(publication) + rewritten = [_candidate_with_source(candidate, source=source) for candidate in candidates] + await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=rewritten) + + +def _candidate_with_source(candidate: IdentifierCandidate, *, source: str | None) -> IdentifierCandidate: + if not source: + return candidate + return IdentifierCandidate( + kind=candidate.kind, + value_raw=candidate.value_raw, + value_normalized=candidate.value_normalized, + source=source, + confidence_score=candidate.confidence_score, + evidence_url=candidate.evidence_url, + ) + + +async def _upsert_publication_candidates( + db_session: AsyncSession, + *, + publication_id: int, + candidates: list[IdentifierCandidate], +) -> None: + for candidate in _dedup_candidates(candidates): + await _upsert_publication_candidate(db_session, publication_id=publication_id, candidate=candidate) + + +async def _upsert_publication_candidate( + db_session: AsyncSession, + *, + publication_id: int, + candidate: IdentifierCandidate, +) -> None: + existing = await _existing_identifier( + db_session, + publication_id=publication_id, + kind=candidate.kind.value, + value_normalized=candidate.value_normalized, + ) + if existing is None: + db_session.add(_new_identifier_row(publication_id=publication_id, candidate=candidate)) + return + _merge_identifier_row(existing, candidate=candidate) + + +async def _existing_identifier( + db_session: AsyncSession, + *, + publication_id: int, + kind: str, + value_normalized: str, +) -> PublicationIdentifier | None: + result = await db_session.execute( + select(PublicationIdentifier).where( + PublicationIdentifier.publication_id == publication_id, + PublicationIdentifier.kind == kind, + PublicationIdentifier.value_normalized == value_normalized, + ) + ) + return result.scalar_one_or_none() + + +async def _existing_identifier_by_kind( + db_session: AsyncSession, + *, + publication_id: int, + kind: str, +) -> PublicationIdentifier | None: + result = await db_session.execute( + select(PublicationIdentifier).where( + PublicationIdentifier.publication_id == publication_id, + PublicationIdentifier.kind == kind, + ).order_by(PublicationIdentifier.confidence_score.desc()).limit(1) + ) + return result.scalar_one_or_none() + + +def _new_identifier_row( + *, + publication_id: int, + candidate: IdentifierCandidate, +) -> PublicationIdentifier: + return PublicationIdentifier( + publication_id=publication_id, + kind=candidate.kind.value, + value_raw=candidate.value_raw, + value_normalized=candidate.value_normalized, + source=candidate.source, + confidence_score=candidate.confidence_score, + evidence_url=candidate.evidence_url, + ) + + +def _merge_identifier_row(existing: PublicationIdentifier, *, candidate: IdentifierCandidate) -> None: + if candidate.confidence_score >= float(existing.confidence_score): + existing.value_raw = candidate.value_raw + existing.source = candidate.source + existing.confidence_score = candidate.confidence_score + if candidate.evidence_url: + existing.evidence_url = candidate.evidence_url + + +async def overlay_publication_items_with_display_identifiers( + db_session: AsyncSession, + *, + items: list[PublicationListItem], +) -> list[PublicationListItem]: + if not items: + return [] + mapping = await _display_identifier_map(db_session, publication_ids=[item.publication_id for item in items]) + return [_overlay_publication_item(item, mapping.get(item.publication_id)) for item in items] + + +def _overlay_publication_item( + item: PublicationListItem, + display_identifier: DisplayIdentifier | None, +) -> PublicationListItem: + fallback = display_identifier or derive_display_identifier_from_values(doi=None, pub_url=item.pub_url, pdf_url=item.pdf_url) + return replace(item, display_identifier=fallback) + + +async def overlay_pdf_queue_items_with_display_identifiers( + db_session: AsyncSession, + *, + items: list[PdfQueueListItem], +) -> list[PdfQueueListItem]: + if not items: + return [] + mapping = await _display_identifier_map(db_session, publication_ids=[item.publication_id for item in items]) + return [_overlay_queue_item(item, mapping.get(item.publication_id)) for item in items] + + +def _overlay_queue_item( + item: PdfQueueListItem, + display_identifier: DisplayIdentifier | None, +) -> PdfQueueListItem: + fallback = display_identifier or derive_display_identifier_from_values(doi=None, pdf_url=item.pdf_url) + return replace(item, display_identifier=fallback) + + +async def _display_identifier_map( + db_session: AsyncSession, + *, + publication_ids: list[int], +) -> dict[int, DisplayIdentifier]: + normalized_ids = sorted({int(value) for value in publication_ids if int(value) > 0}) + if not normalized_ids: + return {} + result = await db_session.execute( + select(PublicationIdentifier).where(PublicationIdentifier.publication_id.in_(normalized_ids)) + ) + rows = list(result.scalars().all()) + return _best_display_identifier_map(rows) + + +def _best_display_identifier_map(rows: list[PublicationIdentifier]) -> dict[int, DisplayIdentifier]: + grouped: dict[int, list[IdentifierCandidate]] = {} + for row in rows: + grouped.setdefault(int(row.publication_id), []).append(_candidate_from_row(row)) + return { + publication_id: display + for publication_id, display in ( + (publication_id, _best_display_identifier(candidates)) + for publication_id, candidates in grouped.items() + ) + if display is not None + } + + +def _candidate_from_row(row: PublicationIdentifier) -> IdentifierCandidate: + return IdentifierCandidate( + kind=IdentifierKind(str(row.kind)), + value_raw=str(row.value_raw), + value_normalized=str(row.value_normalized), + source=str(row.source), + confidence_score=float(row.confidence_score), + evidence_url=row.evidence_url, + ) + + +def _best_display_identifier(candidates: list[IdentifierCandidate]) -> DisplayIdentifier | None: + if not candidates: + return None + ordered = sorted(candidates, key=_display_sort_key, reverse=True) + return _display_identifier_from_candidate(ordered[0]) + + +def _display_sort_key(candidate: IdentifierCandidate) -> tuple[int, float]: + return (_kind_priority(candidate.kind), float(candidate.confidence_score)) + + +def _kind_priority(kind: IdentifierKind) -> int: + if kind == IdentifierKind.DOI: + return PRIORITY_DOI + if kind == IdentifierKind.ARXIV: + return PRIORITY_ARXIV + if kind == IdentifierKind.PMCID: + return PRIORITY_PMCID + return PRIORITY_PMID + + +def _display_identifier_from_candidate(candidate: IdentifierCandidate) -> DisplayIdentifier: + value = candidate.value_normalized + return DisplayIdentifier( + kind=candidate.kind.value, + value=value, + label=_display_label(candidate.kind, value), + url=_identifier_url(candidate.kind, value), + confidence_score=float(candidate.confidence_score), + ) + + +def _display_label(kind: IdentifierKind, value: str) -> str: + if kind == IdentifierKind.DOI: + return f"DOI: {value}" + if kind == IdentifierKind.ARXIV: + return f"arXiv: {value}" + if kind == IdentifierKind.PMCID: + return f"PMCID: {value}" + return f"PMID: {value}" + + +def _identifier_url(kind: IdentifierKind, value: str) -> str | None: + if kind == IdentifierKind.DOI: + return f"https://doi.org/{value}" + if kind == IdentifierKind.ARXIV: + return f"https://arxiv.org/abs/{value}" + if kind == IdentifierKind.PMCID: + return f"https://pmc.ncbi.nlm.nih.gov/articles/{value}/" + if kind == IdentifierKind.PMID: + return f"https://pubmed.ncbi.nlm.nih.gov/{value}/" + return None diff --git a/build/lib/app/services/domains/publication_identifiers/normalize.py b/build/lib/app/services/domains/publication_identifiers/normalize.py new file mode 100644 index 0000000..9f7484a --- /dev/null +++ b/build/lib/app/services/domains/publication_identifiers/normalize.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import re +from urllib.parse import urlparse + +from app.services.domains.doi.normalize import normalize_doi +from app.services.domains.publication_identifiers.types import IdentifierKind + +ARXIV_ABS_RE = re.compile(r"\barxiv:\s*([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?\b", re.I) +ARXIV_PATH_RE = re.compile(r"^/(?:abs|pdf|html|ps|format)/([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?(?:\.pdf)?/?$", re.I) +PMCID_RE = re.compile(r"\b(PMC\d+)\b", re.I) +PUBMED_PATH_RE = re.compile(r"^/(\d+)/?$") + + +def normalize_identifier(kind: IdentifierKind, value: str | None) -> str | None: + if kind == IdentifierKind.DOI: + return normalize_doi(value) + if kind == IdentifierKind.ARXIV: + return normalize_arxiv_id(value) + if kind == IdentifierKind.PMCID: + return normalize_pmcid(value) + if kind == IdentifierKind.PMID: + return normalize_pmid(value) + return None + + +def normalize_arxiv_id(value: str | None) -> str | None: + text = (value or "").strip() + if not text: + return None + parsed = urlparse(text) + if parsed.scheme in {"http", "https"} and "arxiv.org" in parsed.netloc.lower(): + return _arxiv_from_path(parsed.path) + match = ARXIV_ABS_RE.search(text) + if not match: + return None + version = (match.group(2) or "").lower() + return f"{match.group(1).lower()}{version}" + + +def _arxiv_from_path(path: str) -> str | None: + match = ARXIV_PATH_RE.match(path or "") + if not match: + return None + version = (match.group(2) or "").lower() + return f"{match.group(1).lower()}{version}" + + +def normalize_pmcid(value: str | None) -> str | None: + text = (value or "").strip() + if not text: + return None + parsed = urlparse(text) + if parsed.scheme in {"http", "https"} and "ncbi.nlm.nih.gov" in parsed.netloc.lower(): + return _first_match(PMCID_RE, parsed.path) + return _first_match(PMCID_RE, text) + + +def normalize_pmid(value: str | None) -> str | None: + text = (value or "").strip() + if not text: + return None + parsed = urlparse(text) + if parsed.scheme in {"http", "https"} and "pubmed.ncbi.nlm.nih.gov" in parsed.netloc.lower(): + match = PUBMED_PATH_RE.match(parsed.path or "") + if not match: + return None + return match.group(1) + return None + + +def _first_match(pattern: re.Pattern[str], value: str) -> str | None: + match = pattern.search(value) + if not match: + return None + return match.group(1).upper() diff --git a/build/lib/app/services/domains/publication_identifiers/types.py b/build/lib/app/services/domains/publication_identifiers/types.py new file mode 100644 index 0000000..34b20d0 --- /dev/null +++ b/build/lib/app/services/domains/publication_identifiers/types.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + + +class IdentifierKind(StrEnum): + DOI = "doi" + ARXIV = "arxiv" + PMCID = "pmcid" + PMID = "pmid" + + +@dataclass(frozen=True) +class DisplayIdentifier: + kind: str + value: str + label: str + url: str | None + confidence_score: float + + +@dataclass(frozen=True) +class IdentifierCandidate: + kind: IdentifierKind + value_raw: str + value_normalized: str + source: str + confidence_score: float + evidence_url: str | None diff --git a/build/lib/app/services/domains/publications/__init__.py b/build/lib/app/services/domains/publications/__init__.py new file mode 100644 index 0000000..371b02c --- /dev/null +++ b/build/lib/app/services/domains/publications/__init__.py @@ -0,0 +1 @@ +from app.services.domains.publications.application import * diff --git a/build/lib/app/services/domains/publications/application.py b/build/lib/app/services/domains/publications/application.py new file mode 100644 index 0000000..b1161fb --- /dev/null +++ b/build/lib/app/services/domains/publications/application.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from app.services.domains.publications.counts import ( + count_for_user, + count_favorite_for_user, + count_latest_for_user, + count_unread_for_user, +) +from app.services.domains.publications.listing import ( + list_for_user, + retry_pdf_for_user, + list_unread_for_user, +) +from app.services.domains.publications.enrichment import ( + hydrate_pdf_enrichment_state, + schedule_missing_pdf_enrichment_for_user, + schedule_retry_pdf_enrichment_for_row, +) +from app.services.domains.publications.modes import ( + MODE_ALL, + MODE_LATEST, + MODE_NEW, + MODE_UNREAD, + resolve_publication_view_mode, +) +from app.services.domains.publications.queries import ( + get_latest_completed_run_id_for_user, + get_publication_item_for_user, + publications_query, +) +from app.services.domains.publications.read_state import ( + mark_all_unread_as_read_for_user, + mark_selected_as_read_for_user, + set_publication_favorite_for_user, +) +from app.services.domains.publications.pdf_queue import ( + count_pdf_queue_items, + enqueue_all_missing_pdf_jobs, + enqueue_retry_pdf_job_for_publication_id, + list_pdf_queue_page, + list_pdf_queue_items, +) +from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem + +__all__ = [ + "MODE_ALL", + "MODE_UNREAD", + "MODE_LATEST", + "MODE_NEW", + "PublicationListItem", + "UnreadPublicationItem", + "resolve_publication_view_mode", + "get_latest_completed_run_id_for_user", + "publications_query", + "get_publication_item_for_user", + "list_for_user", + "list_unread_for_user", + "retry_pdf_for_user", + "hydrate_pdf_enrichment_state", + "schedule_retry_pdf_enrichment_for_row", + "list_pdf_queue_items", + "list_pdf_queue_page", + "count_pdf_queue_items", + "enqueue_all_missing_pdf_jobs", + "enqueue_retry_pdf_job_for_publication_id", + "schedule_missing_pdf_enrichment_for_user", + "count_for_user", + "count_favorite_for_user", + "count_unread_for_user", + "count_latest_for_user", + "mark_all_unread_as_read_for_user", + "mark_selected_as_read_for_user", + "set_publication_favorite_for_user", +] diff --git a/build/lib/app/services/domains/publications/counts.py b/build/lib/app/services/domains/publications/counts.py new file mode 100644 index 0000000..8f65172 --- /dev/null +++ b/build/lib/app/services/domains/publications/counts.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ScholarProfile, ScholarPublication +from app.services.domains.publications.modes import ( + MODE_ALL, + MODE_LATEST, + MODE_UNREAD, + resolve_publication_view_mode, +) +from app.services.domains.publications.queries import get_latest_completed_run_id_for_user + + +async def count_for_user( + db_session: AsyncSession, + *, + user_id: int, + mode: str = MODE_ALL, + scholar_profile_id: int | None = None, + favorite_only: bool = False, +) -> int: + resolved_mode = resolve_publication_view_mode(mode) + latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) + stmt = ( + select(func.count()) + .select_from(ScholarPublication) + .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) + .where(ScholarProfile.user_id == user_id) + ) + if scholar_profile_id is not None: + stmt = stmt.where(ScholarProfile.id == scholar_profile_id) + if favorite_only: + stmt = stmt.where(ScholarPublication.is_favorite.is_(True)) + if resolved_mode == MODE_UNREAD: + stmt = stmt.where(ScholarPublication.is_read.is_(False)) + if resolved_mode == MODE_LATEST: + if latest_run_id is None: + return 0 + stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id) + result = await db_session.execute(stmt) + return int(result.scalar_one() or 0) + + +async def count_unread_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int | None = None, + favorite_only: bool = False, +) -> int: + return await count_for_user( + db_session, + user_id=user_id, + mode=MODE_UNREAD, + scholar_profile_id=scholar_profile_id, + favorite_only=favorite_only, + ) + + +async def count_latest_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int | None = None, + favorite_only: bool = False, +) -> int: + return await count_for_user( + db_session, + user_id=user_id, + mode=MODE_LATEST, + scholar_profile_id=scholar_profile_id, + favorite_only=favorite_only, + ) + + +async def count_favorite_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int | None = None, +) -> int: + return await count_for_user( + db_session, + user_id=user_id, + mode=MODE_ALL, + scholar_profile_id=scholar_profile_id, + favorite_only=True, + ) diff --git a/build/lib/app/services/domains/publications/enrichment.py b/build/lib/app/services/domains/publications/enrichment.py new file mode 100644 index 0000000..381107a --- /dev/null +++ b/build/lib/app/services/domains/publications/enrichment.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import logging + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.services.domains.publications.pdf_queue import ( + enqueue_missing_pdf_jobs, + enqueue_retry_pdf_job, + overlay_pdf_job_state, +) +from app.services.domains.publications.types import PublicationListItem + +logger = logging.getLogger(__name__) + + +async def schedule_missing_pdf_enrichment_for_user( + db_session: AsyncSession, + *, + user_id: int, + request_email: str | None, + items: list[PublicationListItem], + max_items: int, +) -> int: + queued_ids = await enqueue_missing_pdf_jobs( + db_session, + user_id=user_id, + request_email=request_email, + rows=items, + max_items=max_items, + ) + logger.info( + "publications.enrichment.scheduled", + extra={ + "event": "publications.enrichment.scheduled", + "user_id": user_id, + "publication_count": len(queued_ids), + }, + ) + return len(queued_ids) + + +async def schedule_retry_pdf_enrichment_for_row( + db_session: AsyncSession, + *, + user_id: int, + request_email: str | None, + item: PublicationListItem, +) -> bool: + queued = await enqueue_retry_pdf_job( + db_session, + user_id=user_id, + request_email=request_email, + row=item, + ) + logger.info( + "publications.enrichment.retry_scheduled", + extra={ + "event": "publications.enrichment.retry_scheduled", + "user_id": user_id, + "publication_id": item.publication_id, + "queued": queued, + }, + ) + return queued + + +async def hydrate_pdf_enrichment_state( + db_session: AsyncSession, + *, + items: list[PublicationListItem], +) -> list[PublicationListItem]: + return await overlay_pdf_job_state(db_session, rows=items) diff --git a/build/lib/app/services/domains/publications/listing.py b/build/lib/app/services/domains/publications/listing.py new file mode 100644 index 0000000..0d09d4c --- /dev/null +++ b/build/lib/app/services/domains/publications/listing.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.services.domains.publication_identifiers import application as identifier_service +from app.services.domains.publications.modes import ( + MODE_ALL, + MODE_UNREAD, + resolve_publication_view_mode, +) +from app.services.domains.publications.queries import ( + get_latest_completed_run_id_for_user, + get_publication_item_for_user, + publication_list_item_from_row, + publications_query, + unread_item_from_row, +) +from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem + + +async def list_for_user( + db_session: AsyncSession, + *, + user_id: int, + mode: str = MODE_ALL, + scholar_profile_id: int | None = None, + favorite_only: bool = False, + limit: int = 300, + offset: int = 0, +) -> list[PublicationListItem]: + resolved_mode = resolve_publication_view_mode(mode) + latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) + result = await db_session.execute( + publications_query( + user_id=user_id, + mode=resolved_mode, + latest_run_id=latest_run_id, + scholar_profile_id=scholar_profile_id, + favorite_only=favorite_only, + limit=limit, + offset=offset, + ) + ) + rows = [ + publication_list_item_from_row(row, latest_run_id=latest_run_id) + for row in result.all() + ] + return await identifier_service.overlay_publication_items_with_display_identifiers( + db_session, + items=rows, + ) + + +async def retry_pdf_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, + publication_id: int, +) -> PublicationListItem | None: + item = await get_publication_item_for_user( + db_session, + user_id=user_id, + scholar_profile_id=scholar_profile_id, + publication_id=publication_id, + ) + if item is None: + return None + hydrated = await identifier_service.overlay_publication_items_with_display_identifiers( + db_session, + items=[item], + ) + return hydrated[0] if hydrated else item + + +async def list_unread_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 100, +) -> list[UnreadPublicationItem]: + result = await db_session.execute( + publications_query( + user_id=user_id, + mode=MODE_UNREAD, + latest_run_id=None, + scholar_profile_id=None, + favorite_only=False, + limit=limit, + offset=0, + ) + ) + return [unread_item_from_row(row) for row in result.all()] diff --git a/build/lib/app/services/domains/publications/modes.py b/build/lib/app/services/domains/publications/modes.py new file mode 100644 index 0000000..2975169 --- /dev/null +++ b/build/lib/app/services/domains/publications/modes.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +MODE_ALL = "all" +MODE_UNREAD = "unread" +MODE_LATEST = "latest" +MODE_NEW = "new" # compatibility alias for MODE_LATEST + + +def resolve_publication_view_mode(value: str | None) -> str: + if value == MODE_UNREAD: + return MODE_UNREAD + if value in {MODE_LATEST, MODE_NEW}: + return MODE_LATEST + return MODE_ALL diff --git a/build/lib/app/services/domains/publications/pdf_queue.py b/build/lib/app/services/domains/publications/pdf_queue.py new file mode 100644 index 0000000..18ac8d9 --- /dev/null +++ b/build/lib/app/services/domains/publications/pdf_queue.py @@ -0,0 +1,915 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from datetime import datetime, timezone +import logging + +from sqlalchemy import Select, func, literal, or_, select, union_all +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + Publication, + PublicationPdfJob, + PublicationPdfJobEvent, + ScholarProfile, + ScholarPublication, + User, +) +from app.db.session import get_session_factory +from app.services.domains.publication_identifiers import application as identifier_service +from app.services.domains.publication_identifiers.types import DisplayIdentifier +from app.services.domains.publications.pdf_resolution_pipeline import ( + resolve_publication_pdf_outcome_for_row, +) +from app.services.domains.publications.types import PublicationListItem +from app.services.domains.unpaywall.application import ( + FAILURE_RESOLUTION_EXCEPTION, + OaResolutionOutcome, +) +from app.settings import settings + +PDF_STATUS_UNTRACKED = "untracked" +PDF_STATUS_QUEUED = "queued" +PDF_STATUS_RUNNING = "running" +PDF_STATUS_RESOLVED = "resolved" +PDF_STATUS_FAILED = "failed" + +PDF_EVENT_QUEUED = "queued" +PDF_EVENT_ATTEMPT_STARTED = "attempt_started" +PDF_EVENT_RESOLVED = "resolved" +PDF_EVENT_FAILED = "failed" + +logger = logging.getLogger(__name__) +_scheduled_tasks: set[asyncio.Task[None]] = set() + + +@dataclass(frozen=True) +class PdfQueueListItem: + publication_id: int + title: str + pdf_url: str | None + status: str + attempt_count: int + last_failure_reason: str | None + last_failure_detail: str | None + last_source: str | None + requested_by_user_id: int | None + requested_by_email: str | None + queued_at: datetime | None + last_attempt_at: datetime | None + resolved_at: datetime | None + updated_at: datetime + display_identifier: DisplayIdentifier | None = None + + +@dataclass(frozen=True) +class PdfRequeueResult: + publication_exists: bool + queued: bool + + +@dataclass(frozen=True) +class PdfBulkQueueResult: + requested_count: int + queued_count: int + + +@dataclass(frozen=True) +class PdfQueuePage: + items: list[PdfQueueListItem] + total_count: int + limit: int + offset: int + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def _publication_ids(rows: list[PublicationListItem]) -> list[int]: + return sorted({row.publication_id for row in rows}) + + +def _status_from_job(row: PublicationListItem, job: PublicationPdfJob | None) -> str: + if row.pdf_url: + return PDF_STATUS_RESOLVED + if job is None: + return PDF_STATUS_UNTRACKED + return job.status + + +def _item_from_row_and_job( + row: PublicationListItem, + job: PublicationPdfJob | None, +) -> PublicationListItem: + return PublicationListItem( + publication_id=row.publication_id, + scholar_profile_id=row.scholar_profile_id, + scholar_label=row.scholar_label, + title=row.title, + year=row.year, + citation_count=row.citation_count, + venue_text=row.venue_text, + pub_url=row.pub_url, + pdf_url=row.pdf_url, + is_read=row.is_read, + is_favorite=row.is_favorite, + first_seen_at=row.first_seen_at, + is_new_in_latest_run=row.is_new_in_latest_run, + pdf_status=_status_from_job(row, job), + pdf_attempt_count=int(job.attempt_count) if job is not None else 0, + pdf_failure_reason=job.last_failure_reason if job is not None else None, + pdf_failure_detail=job.last_failure_detail if job is not None else None, + display_identifier=row.display_identifier, + ) + + +def _queueable_rows( + rows: list[PublicationListItem], + *, + max_items: int, +) -> list[PublicationListItem]: + bounded = max(0, int(max_items)) + if bounded == 0: + return [] + candidates = [row for row in rows if not row.pdf_url] + return candidates[:bounded] + + +def _bounded_limit(limit: int, *, max_value: int = 500) -> int: + return max(1, min(int(limit), max_value)) + + +def _bounded_offset(offset: int) -> int: + return max(int(offset), 0) + + +def _auto_retry_interval_seconds() -> int: + return max(int(settings.pdf_auto_retry_interval_seconds), 1) + + +def _auto_retry_first_interval_seconds() -> int: + return max(int(settings.pdf_auto_retry_first_interval_seconds), 1) + + +def _auto_retry_max_attempts() -> int: + return max(int(settings.pdf_auto_retry_max_attempts), 1) + + +def _retry_interval_seconds_for_attempt_count(attempt_count: int) -> int: + if int(attempt_count) <= 1: + return _auto_retry_first_interval_seconds() + return _auto_retry_interval_seconds() + + +def _cooldown_active( + *, + last_attempt_at: datetime | None, + attempt_count: int, +) -> bool: + if last_attempt_at is None: + return False + elapsed = (_utcnow() - last_attempt_at).total_seconds() + return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count)) + + +def _can_enqueue_job( + job: PublicationPdfJob | None, + *, + force_retry: bool, +) -> bool: + if job is None: + return True + if job.status in {PDF_STATUS_QUEUED, PDF_STATUS_RUNNING}: + return False + if force_retry: + return job.status in {PDF_STATUS_FAILED, PDF_STATUS_RESOLVED, PDF_STATUS_UNTRACKED} + if job.status == PDF_STATUS_RESOLVED: + return False + if int(job.attempt_count) >= _auto_retry_max_attempts(): + return False + if _cooldown_active( + last_attempt_at=job.last_attempt_at, + attempt_count=int(job.attempt_count), + ): + return False + return True + + +def _event_row( + *, + publication_id: int, + user_id: int | None, + event_type: str, + status: str | None, + source: str | None = None, + failure_reason: str | None = None, + message: str | None = None, +) -> PublicationPdfJobEvent: + return PublicationPdfJobEvent( + publication_id=publication_id, + user_id=user_id, + event_type=event_type, + status=status, + source=source, + failure_reason=failure_reason, + message=message, + ) + + +def _queued_job( + *, + publication_id: int, + user_id: int, +) -> PublicationPdfJob: + now = _utcnow() + return PublicationPdfJob( + publication_id=publication_id, + status=PDF_STATUS_QUEUED, + queued_at=now, + last_requested_by_user_id=user_id, + ) + + +def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None: + now = _utcnow() + job.status = PDF_STATUS_QUEUED + job.queued_at = now + job.last_requested_by_user_id = user_id + job.last_failure_reason = None + job.last_failure_detail = None + job.last_source = None + + +def _state_map(jobs: list[PublicationPdfJob]) -> dict[int, PublicationPdfJob]: + return {int(job.publication_id): job for job in jobs} + + +async def _jobs_for_publication_ids( + db_session: AsyncSession, + *, + publication_ids: list[int], +) -> dict[int, PublicationPdfJob]: + if not publication_ids: + return {} + result = await db_session.execute( + select(PublicationPdfJob).where(PublicationPdfJob.publication_id.in_(publication_ids)) + ) + return _state_map(list(result.scalars())) + + +async def overlay_pdf_job_state( + db_session: AsyncSession, + *, + rows: list[PublicationListItem], +) -> list[PublicationListItem]: + if not rows: + return [] + jobs = await _jobs_for_publication_ids( + db_session, + publication_ids=_publication_ids(rows), + ) + return [_item_from_row_and_job(row, jobs.get(row.publication_id)) for row in rows] + + +async def _enqueue_rows( + db_session: AsyncSession, + *, + user_id: int, + rows: list[PublicationListItem], + force_retry: bool, +) -> list[PublicationListItem]: + if not rows: + return [] + queued: list[PublicationListItem] = [] + jobs = await _jobs_for_publication_ids( + db_session, + publication_ids=_publication_ids(rows), + ) + for row in rows: + job = jobs.get(row.publication_id) + if not _can_enqueue_job(job, force_retry=force_retry): + continue + if job is None: + job = _queued_job(publication_id=row.publication_id, user_id=user_id) + jobs[row.publication_id] = job + db_session.add(job) + else: + _mark_job_queued(job, user_id=user_id) + db_session.add( + _event_row( + publication_id=row.publication_id, + user_id=user_id, + event_type=PDF_EVENT_QUEUED, + status=PDF_STATUS_QUEUED, + ) + ) + queued.append(row) + if queued: + await db_session.commit() + return queued + + +def _register_task(task: asyncio.Task[None]) -> None: + _scheduled_tasks.add(task) + + +def _drop_finished_task(task: asyncio.Task[None]) -> None: + _scheduled_tasks.discard(task) + try: + task.result() + except Exception: + logger.exception( + "publications.pdf_queue.task_failed", + extra={"event": "publications.pdf_queue.task_failed"}, + ) + + +async def _mark_attempt_started( + *, + publication_id: int, + user_id: int, +) -> None: + session_factory = get_session_factory() + async with session_factory() as db_session: + job = await db_session.get(PublicationPdfJob, publication_id) + if job is None: + job = _queued_job(publication_id=publication_id, user_id=user_id) + db_session.add(job) + job.status = PDF_STATUS_RUNNING + job.last_attempt_at = _utcnow() + job.attempt_count = int(job.attempt_count) + 1 + db_session.add( + _event_row( + publication_id=publication_id, + user_id=user_id, + event_type=PDF_EVENT_ATTEMPT_STARTED, + status=PDF_STATUS_RUNNING, + ) + ) + await db_session.commit() + + +def _failed_outcome( + *, + row: PublicationListItem, +) -> OaResolutionOutcome: + return OaResolutionOutcome( + publication_id=row.publication_id, + doi=None, + pdf_url=None, + failure_reason=FAILURE_RESOLUTION_EXCEPTION, + source=None, + used_crossref=False, + ) + + +async def _fetch_outcome_for_row( + *, + row: PublicationListItem, + request_email: str | None, +) -> OaResolutionOutcome: + pipeline_result = await resolve_publication_pdf_outcome_for_row( + row=row, + request_email=request_email, + ) + outcome = pipeline_result.outcome + if outcome is not None: + return outcome + return _failed_outcome(row=row) + + +def _apply_publication_update( + publication: Publication, + *, + pdf_url: str | None, +) -> None: + if pdf_url and publication.pdf_url != pdf_url: + publication.pdf_url = pdf_url + + +def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None: + job.last_source = outcome.source + if outcome.pdf_url: + job.status = PDF_STATUS_RESOLVED + job.resolved_at = _utcnow() + job.last_failure_reason = None + job.last_failure_detail = None + return + job.status = PDF_STATUS_FAILED + job.last_failure_reason = outcome.failure_reason + job.last_failure_detail = outcome.failure_reason + + +def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]: + if outcome.pdf_url: + return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED + return PDF_EVENT_FAILED, PDF_STATUS_FAILED + + +async def _persist_outcome( + *, + publication_id: int, + user_id: int, + outcome: OaResolutionOutcome, +) -> None: + session_factory = get_session_factory() + async with session_factory() as db_session: + publication = await db_session.get(Publication, publication_id) + job = await db_session.get(PublicationPdfJob, publication_id) + if publication is None or job is None: + return + _apply_publication_update(publication, pdf_url=outcome.pdf_url) + await identifier_service.sync_identifiers_for_publication_resolution( + db_session, + publication=publication, + source=outcome.source, + ) + _apply_job_outcome(job, outcome=outcome) + event_type, status = _result_event(outcome) + db_session.add( + _event_row( + publication_id=publication_id, + user_id=user_id, + event_type=event_type, + status=status, + source=outcome.source, + failure_reason=outcome.failure_reason, + message=outcome.failure_reason, + ) + ) + await db_session.commit() + + +async def _resolve_publication_row( + *, + user_id: int, + request_email: str | None, + row: PublicationListItem, +) -> None: + await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id) + try: + outcome = await _fetch_outcome_for_row(row=row, request_email=request_email) + except Exception as exc: # pragma: no cover - defensive network boundary + logger.warning( + "publications.pdf_queue.resolve_failed", + extra={ + "event": "publications.pdf_queue.resolve_failed", + "publication_id": row.publication_id, + "error": str(exc), + }, + ) + outcome = _failed_outcome(row=row) + await _persist_outcome( + publication_id=row.publication_id, + user_id=user_id, + outcome=outcome, + ) + + +async def _run_resolution_task( + *, + user_id: int, + request_email: str | None, + rows: list[PublicationListItem], +) -> None: + for row in rows: + await _resolve_publication_row( + user_id=user_id, + request_email=request_email, + row=row, + ) + + +def _schedule_rows( + *, + user_id: int, + request_email: str | None, + rows: list[PublicationListItem], +) -> None: + if not rows: + return + task = asyncio.create_task( + _run_resolution_task( + user_id=user_id, + request_email=request_email, + rows=rows, + ) + ) + _register_task(task) + task.add_done_callback(_drop_finished_task) + + +async def enqueue_missing_pdf_jobs( + db_session: AsyncSession, + *, + user_id: int, + request_email: str | None, + rows: list[PublicationListItem], + max_items: int, +) -> list[int]: + queueable = _queueable_rows(rows, max_items=max_items) + queued_rows = await _enqueue_rows( + db_session, + user_id=user_id, + rows=queueable, + force_retry=False, + ) + _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows) + return [row.publication_id for row in queued_rows] + + +async def enqueue_retry_pdf_job( + db_session: AsyncSession, + *, + user_id: int, + request_email: str | None, + row: PublicationListItem, +) -> bool: + queued_rows = await _enqueue_rows( + db_session, + user_id=user_id, + rows=[row], + force_retry=True, + ) + _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows) + return bool(queued_rows) + + +def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str: + return str(display_name or scholar_id or "unknown") + + +def _retry_item_from_publication( + publication: Publication, + *, + link_row: tuple | None, +) -> PublicationListItem: + if link_row is None: + scholar_profile_id = 0 + scholar_label = "unknown" + is_read = True + first_seen_at = publication.created_at or _utcnow() + else: + scholar_profile_id = int(link_row[0]) + scholar_label = _retry_item_label(link_row[1], link_row[2]) + is_read = bool(link_row[3]) + first_seen_at = link_row[4] or publication.created_at or _utcnow() + return PublicationListItem( + publication_id=int(publication.id), + scholar_profile_id=scholar_profile_id, + scholar_label=scholar_label, + title=publication.title_raw, + year=publication.year, + citation_count=int(publication.citation_count or 0), + venue_text=publication.venue_text, + pub_url=publication.pub_url, + pdf_url=publication.pdf_url, + is_read=is_read, + first_seen_at=first_seen_at, + is_new_in_latest_run=False, + ) + + +async def _retry_item_for_publication_id( + db_session: AsyncSession, + *, + publication_id: int, +) -> PublicationListItem | None: + publication = await db_session.get(Publication, publication_id) + if publication is None: + return None + result = await db_session.execute( + select( + ScholarProfile.id, + ScholarProfile.display_name, + ScholarProfile.scholar_id, + ScholarPublication.is_read, + ScholarPublication.created_at, + ) + .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) + .where(ScholarPublication.publication_id == publication_id) + .order_by(ScholarPublication.created_at.asc()) + .limit(1) + ) + return _retry_item_from_publication(publication, link_row=result.one_or_none()) + + +async def enqueue_retry_pdf_job_for_publication_id( + db_session: AsyncSession, + *, + user_id: int, + request_email: str | None, + publication_id: int, +) -> PdfRequeueResult: + row = await _retry_item_for_publication_id( + db_session, + publication_id=publication_id, + ) + if row is None: + return PdfRequeueResult(publication_exists=False, queued=False) + queued = await enqueue_retry_pdf_job( + db_session, + user_id=user_id, + request_email=request_email, + row=row, + ) + return PdfRequeueResult(publication_exists=True, queued=queued) + + +def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem: + return PublicationListItem( + publication_id=int(publication.id), + scholar_profile_id=0, + scholar_label="admin", + title=publication.title_raw, + year=publication.year, + citation_count=int(publication.citation_count or 0), + venue_text=publication.venue_text, + pub_url=publication.pub_url, + pdf_url=publication.pdf_url, + is_read=True, + first_seen_at=publication.created_at or _utcnow(), + is_new_in_latest_run=False, + ) + + +async def _missing_pdf_candidates( + db_session: AsyncSession, + *, + limit: int, +) -> list[PublicationListItem]: + bounded_limit = max(1, min(int(limit), 5000)) + result = await db_session.execute( + select(Publication) + .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id) + .where(Publication.pdf_url.is_(None)) + .where( + or_( + PublicationPdfJob.publication_id.is_(None), + PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]), + ) + ) + .order_by(Publication.updated_at.desc(), Publication.id.desc()) + .limit(bounded_limit) + ) + return [ + _queue_candidate_from_publication(publication) + for publication in result.scalars() + ] + + +async def enqueue_all_missing_pdf_jobs( + db_session: AsyncSession, + *, + user_id: int, + request_email: str | None, + limit: int = 1000, +) -> PdfBulkQueueResult: + candidates = await _missing_pdf_candidates(db_session, limit=limit) + queued_rows = await _enqueue_rows( + db_session, + user_id=user_id, + rows=candidates, + force_retry=True, + ) + _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows) + return PdfBulkQueueResult( + requested_count=len(candidates), + queued_count=len(queued_rows), + ) + + +def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]: + stmt = ( + select( + PublicationPdfJob.publication_id, + Publication.title_raw, + Publication.pdf_url, + PublicationPdfJob.status, + PublicationPdfJob.attempt_count, + PublicationPdfJob.last_failure_reason, + PublicationPdfJob.last_failure_detail, + PublicationPdfJob.last_source, + PublicationPdfJob.last_requested_by_user_id, + User.email, + PublicationPdfJob.queued_at, + PublicationPdfJob.last_attempt_at, + PublicationPdfJob.resolved_at, + PublicationPdfJob.updated_at, + ) + .join(Publication, Publication.id == PublicationPdfJob.publication_id) + .outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id) + ) + if status: + stmt = stmt.where(PublicationPdfJob.status == status) + return stmt + + +def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]: + return ( + _tracked_queue_select_base(status=status) + .order_by(PublicationPdfJob.updated_at.desc()) + .limit(_bounded_limit(limit)) + .offset(_bounded_offset(offset)) + ) + + +def _untracked_queue_select_base() -> Select[tuple]: + return ( + select( + Publication.id, + Publication.title_raw, + Publication.pdf_url, + literal(PDF_STATUS_UNTRACKED), + literal(0), + literal(None), + literal(None), + literal(None), + literal(None), + literal(None), + literal(None), + literal(None), + literal(None), + Publication.updated_at, + ) + .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id) + .where(Publication.pdf_url.is_(None)) + .where(PublicationPdfJob.publication_id.is_(None)) + ) + + +def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]: + return ( + _untracked_queue_select_base() + .order_by(Publication.updated_at.desc(), Publication.id.desc()) + .limit(_bounded_limit(limit)) + .offset(_bounded_offset(offset)) + ) + + +def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]: + union_stmt = union_all( + _tracked_queue_select_base(status=None), + _untracked_queue_select_base(), + ).subquery() + return ( + select(union_stmt) + .order_by(union_stmt.c.updated_at.desc()) + .limit(_bounded_limit(limit)) + .offset(_bounded_offset(offset)) + ) + + +def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]: + stmt = select(func.count()).select_from(PublicationPdfJob) + if status: + stmt = stmt.where(PublicationPdfJob.status == status) + return stmt + + +def _untracked_queue_count_select() -> Select[tuple]: + return ( + select(func.count()) + .select_from(Publication) + .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id) + .where(Publication.pdf_url.is_(None)) + .where(PublicationPdfJob.publication_id.is_(None)) + ) + + +def _queue_item_from_row(row: tuple) -> PdfQueueListItem: + return PdfQueueListItem( + publication_id=int(row[0]), + title=str(row[1] or ""), + pdf_url=row[2], + status=str(row[3] or PDF_STATUS_UNTRACKED), + attempt_count=int(row[4] or 0), + last_failure_reason=row[5], + last_failure_detail=row[6], + last_source=row[7], + requested_by_user_id=int(row[8]) if row[8] is not None else None, + requested_by_email=row[9], + queued_at=row[10], + last_attempt_at=row[11], + resolved_at=row[12], + updated_at=row[13], + ) + + +async def _hydrated_queue_items( + db_session: AsyncSession, + *, + rows: list[tuple], +) -> list[PdfQueueListItem]: + items = [_queue_item_from_row(row) for row in rows] + return await identifier_service.overlay_pdf_queue_items_with_display_identifiers( + db_session, + items=items, + ) + + +async def list_pdf_queue_items( + db_session: AsyncSession, + *, + limit: int = 100, + offset: int = 0, + status: str | None = None, +) -> list[PdfQueueListItem]: + bounded_limit = _bounded_limit(limit) + bounded_offset = _bounded_offset(offset) + normalized_status = (status or "").strip().lower() or None + if normalized_status == PDF_STATUS_UNTRACKED: + result = await db_session.execute( + _untracked_queue_select( + limit=bounded_limit, + offset=bounded_offset, + ) + ) + return await _hydrated_queue_items(db_session, rows=list(result.all())) + if normalized_status is None: + result = await db_session.execute( + _all_queue_select( + limit=bounded_limit, + offset=bounded_offset, + ) + ) + return await _hydrated_queue_items(db_session, rows=list(result.all())) + result = await db_session.execute( + _tracked_queue_select( + limit=bounded_limit, + offset=bounded_offset, + status=normalized_status, + ) + ) + return await _hydrated_queue_items(db_session, rows=list(result.all())) + + +async def count_pdf_queue_items( + db_session: AsyncSession, + *, + status: str | None = None, +) -> int: + normalized_status = (status or "").strip().lower() or None + if normalized_status == PDF_STATUS_UNTRACKED: + result = await db_session.execute(_untracked_queue_count_select()) + return int(result.scalar_one() or 0) + tracked_result = await db_session.execute( + _tracked_queue_count_select(status=normalized_status) + ) + tracked_count = int(tracked_result.scalar_one() or 0) + if normalized_status is not None: + return tracked_count + untracked_result = await db_session.execute(_untracked_queue_count_select()) + untracked_count = int(untracked_result.scalar_one() or 0) + return tracked_count + untracked_count + + +async def list_pdf_queue_page( + db_session: AsyncSession, + *, + limit: int = 100, + offset: int = 0, + status: str | None = None, +) -> PdfQueuePage: + bounded_limit = _bounded_limit(limit) + bounded_offset = _bounded_offset(offset) + items = await list_pdf_queue_items( + db_session, + limit=bounded_limit, + offset=bounded_offset, + status=status, + ) + total_count = await count_pdf_queue_items( + db_session, + status=status, + ) + return PdfQueuePage( + items=items, + total_count=total_count, + limit=bounded_limit, + offset=bounded_offset, + ) + + +async def drain_ready_jobs( + db_session: AsyncSession, + *, + limit: int, + max_attempts: int, +) -> int: + result = await db_session.execute( + select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1) + ) + system_user_id = result.scalar_one_or_none() + if system_user_id is None: + return 0 + + bulk_result = await enqueue_all_missing_pdf_jobs( + db_session, + user_id=system_user_id, + request_email=settings.unpaywall_email, + limit=limit, + ) + return bulk_result.queued_count diff --git a/build/lib/app/services/domains/publications/pdf_resolution_pipeline.py b/build/lib/app/services/domains/publications/pdf_resolution_pipeline.py new file mode 100644 index 0000000..9e07c0b --- /dev/null +++ b/build/lib/app/services/domains/publications/pdf_resolution_pipeline.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from dataclasses import dataclass +import logging +from typing import Any + +from app.services.domains.publications.types import PublicationListItem +from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes +from app.settings import settings + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class PipelineOutcome: + outcome: OaResolutionOutcome | None + scholar_candidates: Any | None # Kept for backward compatibility with calling signatures + + +async def resolve_publication_pdf_outcome_for_row( + *, + row: PublicationListItem, + request_email: str | None, +) -> PipelineOutcome: + # 1. OpenAlex OA + openalex_outcome = await _openalex_outcome(row, request_email=request_email) + if openalex_outcome and openalex_outcome.pdf_url: + return PipelineOutcome(openalex_outcome, None) + + # 2. arXiv + arxiv_outcome = await _arxiv_outcome(row, request_email=request_email) + if arxiv_outcome and arxiv_outcome.pdf_url: + return PipelineOutcome(arxiv_outcome, None) + + # 3. Unpaywall (which falls back to Crossref) + oa_outcome = await _oa_outcome(row=row, request_email=request_email) + return PipelineOutcome(oa_outcome, None) + + +async def _openalex_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None: + from app.services.domains.openalex.client import OpenAlexClient + from app.services.domains.openalex.matching import find_best_match + + if not row.title: + return None + + import re + safe_title = re.sub(r"[^\w\s]", " ", row.title) + safe_title = " ".join(safe_title.split()) + if not safe_title: + return None + + client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=request_email or settings.crossref_api_mailto) + try: + openalex_works = await client.get_works_by_filter({"title.search": safe_title}, limit=5) + match = find_best_match( + target_title=row.title, + target_year=row.year, + target_authors=row.scholar_label, + candidates=openalex_works, + ) + if match and match.oa_url: + return OaResolutionOutcome( + publication_id=row.publication_id, + doi=match.doi, + pdf_url=match.oa_url, + failure_reason=None, + source="openalex", + used_crossref=False, + ) + except Exception as exc: + logger.warning( + "publications.pdf_resolution.openalex_failed", + extra={"event": "publications.pdf_resolution.openalex_failed", "error": str(exc)}, + ) + return None + + +async def _arxiv_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None: + from app.services.domains.arxiv.application import discover_arxiv_id_for_publication + + try: + arxiv_id = await discover_arxiv_id_for_publication(item=row, request_email=request_email) + if arxiv_id: + pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf" + return OaResolutionOutcome( + publication_id=row.publication_id, + doi=None, + pdf_url=pdf_url, + failure_reason=None, + source="arxiv", + used_crossref=False, + ) + except Exception as exc: + logger.warning( + "publications.pdf_resolution.arxiv_failed", + extra={"event": "publications.pdf_resolution.arxiv_failed", "error": str(exc)}, + ) + return None + + +async def _oa_outcome( + *, + row: PublicationListItem, + request_email: str | None, +) -> OaResolutionOutcome | None: + outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email) + return outcomes.get(row.publication_id) diff --git a/build/lib/app/services/domains/publications/queries.py b/build/lib/app/services/domains/publications/queries.py new file mode 100644 index 0000000..e7356f4 --- /dev/null +++ b/build/lib/app/services/domains/publications/queries.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +from sqlalchemy import Select, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import CrawlRun, Publication, RunStatus, ScholarProfile, ScholarPublication +from app.services.domains.publications.modes import MODE_LATEST, MODE_UNREAD +from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem + + +def _normalized_citation_count(value: object) -> int: + try: + return int(value or 0) + except (TypeError, ValueError): + return 0 + + +async def get_latest_completed_run_id_for_user( + db_session: AsyncSession, + *, + user_id: int, +) -> int | None: + result = await db_session.execute( + select(func.max(CrawlRun.id)).where( + CrawlRun.user_id == user_id, + CrawlRun.status != RunStatus.RUNNING, + ) + ) + latest_run_id = result.scalar_one_or_none() + return int(latest_run_id) if latest_run_id is not None else None + + +def publications_query( + *, + user_id: int, + mode: str, + latest_run_id: int | None, + scholar_profile_id: int | None, + favorite_only: bool, + limit: int, + offset: int = 0, +) -> Select[tuple]: + scholar_label = ScholarProfile.display_name + stmt = ( + select( + Publication.id, + ScholarProfile.id, + scholar_label, + ScholarProfile.scholar_id, + Publication.title_raw, + Publication.year, + Publication.citation_count, + Publication.venue_text, + Publication.pub_url, + Publication.pdf_url, + ScholarPublication.is_read, + ScholarPublication.is_favorite, + ScholarPublication.first_seen_run_id, + ScholarPublication.created_at, + ) + .join(ScholarPublication, ScholarPublication.publication_id == Publication.id) + .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) + .where(ScholarProfile.user_id == user_id) + .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) + .offset(max(int(offset), 0)) + .limit(limit) + ) + if scholar_profile_id is not None: + stmt = stmt.where(ScholarProfile.id == scholar_profile_id) + if favorite_only: + stmt = stmt.where(ScholarPublication.is_favorite.is_(True)) + if mode == MODE_UNREAD: + stmt = stmt.where(ScholarPublication.is_read.is_(False)) + if mode == MODE_LATEST: + if latest_run_id is None: + return stmt.where(False) + stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id) + return stmt + + +def publication_query_for_user( + *, + user_id: int, + scholar_profile_id: int, + publication_id: int, +) -> Select[tuple]: + return ( + select( + Publication.id, + ScholarProfile.id, + ScholarProfile.display_name, + ScholarProfile.scholar_id, + Publication.title_raw, + Publication.year, + Publication.citation_count, + Publication.venue_text, + Publication.pub_url, + Publication.pdf_url, + ScholarPublication.is_read, + ScholarPublication.is_favorite, + ScholarPublication.first_seen_run_id, + ScholarPublication.created_at, + ) + .join(ScholarPublication, ScholarPublication.publication_id == Publication.id) + .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) + .where( + ScholarProfile.user_id == user_id, + ScholarProfile.id == scholar_profile_id, + Publication.id == publication_id, + ) + .limit(1) + ) + + +async def get_publication_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, + publication_id: int, +) -> PublicationListItem | None: + latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) + result = await db_session.execute( + publication_query_for_user( + user_id=user_id, + scholar_profile_id=scholar_profile_id, + publication_id=publication_id, + ) + ) + row = result.one_or_none() + if row is None: + return None + return publication_list_item_from_row(row, latest_run_id=latest_run_id) + + +def publication_list_item_from_row( + row: tuple, + *, + latest_run_id: int | None, +) -> PublicationListItem: + ( + publication_id, + scholar_profile_id, + display_name, + scholar_id, + title_raw, + year, + citation_count, + venue_text, + pub_url, + pdf_url, + is_read, + is_favorite, + first_seen_run_id, + created_at, + ) = row + return PublicationListItem( + publication_id=int(publication_id), + scholar_profile_id=int(scholar_profile_id), + scholar_label=(display_name or scholar_id), + title=title_raw, + year=year, + citation_count=_normalized_citation_count(citation_count), + venue_text=venue_text, + pub_url=pub_url, + pdf_url=pdf_url, + is_read=bool(is_read), + is_favorite=bool(is_favorite), + first_seen_at=created_at, + is_new_in_latest_run=( + latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id + ), + ) + + +def unread_item_from_row(row: tuple) -> UnreadPublicationItem: + ( + publication_id, + scholar_profile_id, + display_name, + scholar_id, + title_raw, + year, + citation_count, + venue_text, + pub_url, + pdf_url, + _is_read, + _is_favorite, + _first_seen_run_id, + _created_at, + ) = row + return UnreadPublicationItem( + publication_id=int(publication_id), + scholar_profile_id=int(scholar_profile_id), + scholar_label=(display_name or scholar_id), + title=title_raw, + year=year, + citation_count=_normalized_citation_count(citation_count), + venue_text=venue_text, + pub_url=pub_url, + pdf_url=pdf_url, + ) diff --git a/build/lib/app/services/domains/publications/read_state.py b/build/lib/app/services/domains/publications/read_state.py new file mode 100644 index 0000000..41fc31a --- /dev/null +++ b/build/lib/app/services/domains/publications/read_state.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from sqlalchemy import select, tuple_, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ScholarProfile, ScholarPublication + + +def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[int, int]]: + pairs: set[tuple[int, int]] = set() + for scholar_profile_id, publication_id in selections: + normalized = (int(scholar_profile_id), int(publication_id)) + if normalized[0] <= 0 or normalized[1] <= 0: + continue + pairs.add(normalized) + return pairs + + +def _scoped_scholar_ids_query(*, user_id: int): + return ( + select(ScholarProfile.id) + .where(ScholarProfile.user_id == user_id) + .scalar_subquery() + ) + + +async def mark_all_unread_as_read_for_user( + db_session: AsyncSession, + *, + user_id: int, +) -> int: + scholar_ids = _scoped_scholar_ids_query(user_id=user_id) + stmt = ( + update(ScholarPublication) + .where( + ScholarPublication.scholar_profile_id.in_(scholar_ids), + ScholarPublication.is_read.is_(False), + ) + .values(is_read=True) + ) + result = await db_session.execute(stmt) + await db_session.commit() + return int(result.rowcount or 0) + + +async def mark_selected_as_read_for_user( + db_session: AsyncSession, + *, + user_id: int, + selections: list[tuple[int, int]], +) -> int: + normalized_pairs = _normalized_selection_pairs(selections) + if not normalized_pairs: + return 0 + + scholar_ids = _scoped_scholar_ids_query(user_id=user_id) + stmt = ( + update(ScholarPublication) + .where( + ScholarPublication.scholar_profile_id.in_(scholar_ids), + tuple_( + ScholarPublication.scholar_profile_id, + ScholarPublication.publication_id, + ).in_(list(normalized_pairs)), + ScholarPublication.is_read.is_(False), + ) + .values(is_read=True) + ) + result = await db_session.execute(stmt) + await db_session.commit() + return int(result.rowcount or 0) + + +async def set_publication_favorite_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, + publication_id: int, + is_favorite: bool, +) -> int: + scholar_ids = _scoped_scholar_ids_query(user_id=user_id) + stmt = ( + update(ScholarPublication) + .where( + ScholarPublication.scholar_profile_id.in_(scholar_ids), + ScholarPublication.scholar_profile_id == int(scholar_profile_id), + ScholarPublication.publication_id == int(publication_id), + ) + .values(is_favorite=bool(is_favorite)) + ) + result = await db_session.execute(stmt) + await db_session.commit() + return int(result.rowcount or 0) diff --git a/build/lib/app/services/domains/publications/types.py b/build/lib/app/services/domains/publications/types.py new file mode 100644 index 0000000..62d038b --- /dev/null +++ b/build/lib/app/services/domains/publications/types.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from app.services.domains.publication_identifiers.types import DisplayIdentifier + + +@dataclass(frozen=True) +class PublicationListItem: + publication_id: int + scholar_profile_id: int + scholar_label: str + title: str + year: int | None + citation_count: int + venue_text: str | None + pub_url: str | None + pdf_url: str | None + is_read: bool + first_seen_at: datetime + is_new_in_latest_run: bool + is_favorite: bool = False + pdf_status: str = "untracked" + pdf_attempt_count: int = 0 + pdf_failure_reason: str | None = None + pdf_failure_detail: str | None = None + display_identifier: DisplayIdentifier | None = None + + +@dataclass(frozen=True) +class UnreadPublicationItem: + publication_id: int + scholar_profile_id: int + scholar_label: str + title: str + year: int | None + citation_count: int + venue_text: str | None + pub_url: str | None + pdf_url: str | None diff --git a/build/lib/app/services/domains/runs/__init__.py b/build/lib/app/services/domains/runs/__init__.py new file mode 100644 index 0000000..848f6e0 --- /dev/null +++ b/build/lib/app/services/domains/runs/__init__.py @@ -0,0 +1 @@ +from app.services.domains.runs.application import * diff --git a/build/lib/app/services/domains/runs/application.py b/build/lib/app/services/domains/runs/application.py new file mode 100644 index 0000000..0019446 --- /dev/null +++ b/build/lib/app/services/domains/runs/application.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from app.services.domains.runs.queue_service import ( + clear_queue_item_for_user, + drop_queue_item_for_user, + get_queue_item_for_user, + list_queue_items_for_user, + queue_status_counts_for_user, + retry_queue_item_for_user, +) +from app.services.domains.runs.runs_service import ( + get_manual_run_by_idempotency_key, + get_run_for_user, + list_recent_runs_for_user, + list_runs_for_user, +) +from app.services.domains.runs.summary import extract_run_summary +from app.services.domains.runs.types import ( + QUEUE_STATUS_DROPPED, + QUEUE_STATUS_QUEUED, + QUEUE_STATUS_RETRYING, + QueueClearResult, + QueueListItem, + QueueTransitionError, +) + +__all__ = [ + "QUEUE_STATUS_QUEUED", + "QUEUE_STATUS_RETRYING", + "QUEUE_STATUS_DROPPED", + "QueueListItem", + "QueueClearResult", + "QueueTransitionError", + "extract_run_summary", + "list_recent_runs_for_user", + "list_runs_for_user", + "get_run_for_user", + "get_manual_run_by_idempotency_key", + "list_queue_items_for_user", + "get_queue_item_for_user", + "retry_queue_item_for_user", + "drop_queue_item_for_user", + "clear_queue_item_for_user", + "queue_status_counts_for_user", +] diff --git a/build/lib/app/services/domains/runs/events.py b/build/lib/app/services/domains/runs/events.py new file mode 100644 index 0000000..ef28069 --- /dev/null +++ b/build/lib/app/services/domains/runs/events.py @@ -0,0 +1,59 @@ +import asyncio +import json +import logging +from typing import Any, AsyncGenerator, Dict, Set + +logger = logging.getLogger(__name__) + +class RunEventPublisher: + def __init__(self) -> None: + # Maps run_id to a set of subscriber queues + self._subscribers: Dict[int, Set[asyncio.Queue]] = {} + + def subscribe(self, run_id: int) -> asyncio.Queue: + if run_id not in self._subscribers: + self._subscribers[run_id] = set() + queue: asyncio.Queue[Any] = asyncio.Queue() + self._subscribers[run_id].add(queue) + logger.debug(f"New subscriber for run {run_id}. Total: {len(self._subscribers[run_id])}") + return queue + + def unsubscribe(self, run_id: int, queue: asyncio.Queue) -> None: + if run_id in self._subscribers: + self._subscribers[run_id].discard(queue) + if not self._subscribers[run_id]: + self._subscribers.pop(run_id, None) + + async def publish(self, run_id: int, event_type: str, data: dict[str, Any]) -> None: + if run_id not in self._subscribers: + return + + message = { + "type": event_type, + "data": data + } + + # Fan-out to all active subscribers for this run + for queue in list(self._subscribers[run_id]): + try: + queue.put_nowait(message) + except asyncio.QueueFull: + logger.warning(f"Subscriber queue full for run {run_id}, dropping message") + +run_events = RunEventPublisher() + +async def event_generator(run_id: int) -> AsyncGenerator[str, None]: + queue = run_events.subscribe(run_id) + try: + while True: + # Wait for a new event + message = await queue.get() + # Server-Sent Events format: "event: \ndata: \n\n" + event_type = message["type"] + data_str = json.dumps(message["data"]) + yield f"event: {event_type}\ndata: {data_str}\n\n" + except asyncio.CancelledError: + logger.debug(f"Client disconnected from SSE stream for run {run_id}") + raise + finally: + run_events.unsubscribe(run_id, queue) diff --git a/build/lib/app/services/domains/runs/queue_queries.py b/build/lib/app/services/domains/runs/queue_queries.py new file mode 100644 index 0000000..7d2e501 --- /dev/null +++ b/build/lib/app/services/domains/runs/queue_queries.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from sqlalchemy import and_, select + +from app.db.models import IngestionQueueItem, ScholarProfile +from app.services.domains.runs.types import QueueListItem + + +def queue_item_columns() -> tuple: + return ( + IngestionQueueItem.id, + IngestionQueueItem.scholar_profile_id, + ScholarProfile.display_name, + ScholarProfile.scholar_id, + IngestionQueueItem.status, + IngestionQueueItem.reason, + IngestionQueueItem.dropped_reason, + IngestionQueueItem.attempt_count, + IngestionQueueItem.resume_cstart, + IngestionQueueItem.next_attempt_dt, + IngestionQueueItem.updated_at, + IngestionQueueItem.last_error, + IngestionQueueItem.last_run_id, + ) + + +def queue_item_select(*, user_id: int): + return ( + select(*queue_item_columns()) + .join( + ScholarProfile, + and_( + ScholarProfile.id == IngestionQueueItem.scholar_profile_id, + ScholarProfile.user_id == IngestionQueueItem.user_id, + ), + ) + .where(IngestionQueueItem.user_id == user_id) + ) + + +def queue_list_item_from_row(row: tuple) -> QueueListItem: + ( + item_id, + scholar_profile_id, + display_name, + scholar_id, + status, + reason, + dropped_reason, + attempt_count, + resume_cstart, + next_attempt_dt, + updated_at, + last_error, + last_run_id, + ) = row + return QueueListItem( + id=int(item_id), + scholar_profile_id=int(scholar_profile_id), + scholar_label=(display_name or scholar_id), + status=str(status), + reason=str(reason), + dropped_reason=dropped_reason, + attempt_count=int(attempt_count or 0), + resume_cstart=int(resume_cstart or 0), + next_attempt_dt=next_attempt_dt, + updated_at=updated_at, + last_error=last_error, + last_run_id=int(last_run_id) if last_run_id is not None else None, + ) diff --git a/build/lib/app/services/domains/runs/queue_service.py b/build/lib/app/services/domains/runs/queue_service.py new file mode 100644 index 0000000..d69ce21 --- /dev/null +++ b/build/lib/app/services/domains/runs/queue_service.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +from sqlalchemy import case, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import IngestionQueueItem +from app.services.domains.ingestion import queue as queue_mutations +from app.services.domains.runs.queue_queries import queue_item_select, queue_list_item_from_row +from app.services.domains.runs.types import ( + QUEUE_STATUS_DROPPED, + QUEUE_STATUS_QUEUED, + QUEUE_STATUS_RETRYING, + QueueClearResult, + QueueListItem, + QueueTransitionError, +) + + +async def list_queue_items_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 200, +) -> list[QueueListItem]: + result = await db_session.execute( + queue_item_select(user_id=user_id) + .order_by( + case((IngestionQueueItem.status == QUEUE_STATUS_DROPPED, 1), else_=0).asc(), + IngestionQueueItem.next_attempt_dt.asc(), + IngestionQueueItem.id.asc(), + ) + .limit(limit) + ) + return [queue_list_item_from_row(row) for row in result.all()] + + +async def get_queue_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + queue_item_id: int, +) -> QueueListItem | None: + result = await db_session.execute( + queue_item_select(user_id=user_id) + .where(IngestionQueueItem.id == queue_item_id) + .limit(1) + ) + row = result.one_or_none() + if row is None: + return None + return queue_list_item_from_row(row) + + +async def retry_queue_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + queue_item_id: int, +) -> QueueListItem | None: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.id == queue_item_id, + IngestionQueueItem.user_id == user_id, + ) + ) + item = result.scalar_one_or_none() + if item is None: + return None + if item.status == QUEUE_STATUS_QUEUED: + raise QueueTransitionError( + code="queue_item_already_queued", + message="Queue item is already queued.", + current_status=item.status, + ) + if item.status == QUEUE_STATUS_RETRYING: + raise QueueTransitionError( + code="queue_item_retrying", + message="Queue item is currently retrying.", + current_status=item.status, + ) + + await queue_mutations.mark_queued_now( + db_session, + job_id=item.id, + reason="manual_retry", + reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED), + ) + await db_session.commit() + return await get_queue_item_for_user( + db_session, + user_id=user_id, + queue_item_id=queue_item_id, + ) + + +async def drop_queue_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + queue_item_id: int, +) -> QueueListItem | None: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.id == queue_item_id, + IngestionQueueItem.user_id == user_id, + ) + ) + item = result.scalar_one_or_none() + if item is None: + return None + if item.status == QUEUE_STATUS_DROPPED: + raise QueueTransitionError( + code="queue_item_already_dropped", + message="Queue item is already dropped.", + current_status=item.status, + ) + + await queue_mutations.mark_dropped( + db_session, + job_id=int(item.id), + reason="manual_drop", + ) + await db_session.commit() + return await get_queue_item_for_user( + db_session, + user_id=user_id, + queue_item_id=queue_item_id, + ) + + +async def clear_queue_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + queue_item_id: int, +) -> QueueClearResult | None: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.id == queue_item_id, + IngestionQueueItem.user_id == user_id, + ) + ) + item = result.scalar_one_or_none() + if item is None: + return None + if item.status != QUEUE_STATUS_DROPPED: + raise QueueTransitionError( + code="queue_item_not_dropped", + message="Queue item can only be cleared after it is dropped.", + current_status=item.status, + ) + + item_id = int(item.id) + previous_status = str(item.status) + deleted = await queue_mutations.delete_job_by_id(db_session, job_id=item_id) + await db_session.commit() + if not deleted: + return None + return QueueClearResult(queue_item_id=item_id, previous_status=previous_status) + + +async def queue_status_counts_for_user( + db_session: AsyncSession, + *, + user_id: int, +) -> dict[str, int]: + result = await db_session.execute( + select( + IngestionQueueItem.status, + func.count(IngestionQueueItem.id), + ) + .where(IngestionQueueItem.user_id == user_id) + .group_by(IngestionQueueItem.status) + ) + counts: dict[str, int] = { + QUEUE_STATUS_QUEUED: 0, + QUEUE_STATUS_RETRYING: 0, + QUEUE_STATUS_DROPPED: 0, + } + for status, count in result.all(): + counts[str(status)] = int(count or 0) + return counts diff --git a/build/lib/app/services/domains/runs/runs_service.py b/build/lib/app/services/domains/runs/runs_service.py new file mode 100644 index 0000000..1cb9aa2 --- /dev/null +++ b/build/lib/app/services/domains/runs/runs_service.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from sqlalchemy import or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import CrawlRun, RunStatus, RunTriggerType + + +async def list_recent_runs_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 20, +) -> list[CrawlRun]: + result = await db_session.execute( + select(CrawlRun) + .where(CrawlRun.user_id == user_id) + .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) + .limit(limit) + ) + return list(result.scalars().all()) + + +async def list_runs_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 100, + failed_only: bool = False, +) -> list[CrawlRun]: + stmt = ( + select(CrawlRun) + .where(CrawlRun.user_id == user_id) + .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) + .limit(limit) + ) + if failed_only: + stmt = stmt.where( + CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE]) + ) + result = await db_session.execute(stmt) + return list(result.scalars().all()) + + +async def get_run_for_user( + db_session: AsyncSession, + *, + user_id: int, + run_id: int, +) -> CrawlRun | None: + result = await db_session.execute( + select(CrawlRun).where( + CrawlRun.user_id == user_id, + CrawlRun.id == run_id, + ) + ) + return result.scalar_one_or_none() + + +async def get_manual_run_by_idempotency_key( + db_session: AsyncSession, + *, + user_id: int, + idempotency_key: str, +) -> CrawlRun | None: + result = await db_session.execute( + select(CrawlRun) + .where( + CrawlRun.user_id == user_id, + CrawlRun.trigger_type == RunTriggerType.MANUAL, + or_( + CrawlRun.idempotency_key == idempotency_key, + CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key, + ), + ) + .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) + .limit(1) + ) + return result.scalar_one_or_none() diff --git a/build/lib/app/services/domains/runs/summary.py b/build/lib/app/services/domains/runs/summary.py new file mode 100644 index 0000000..9555558 --- /dev/null +++ b/build/lib/app/services/domains/runs/summary.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from typing import Any + + +def _safe_int(value: object, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _summary_dict(error_log: object) -> dict[str, Any]: + if not isinstance(error_log, dict): + return {} + summary = error_log.get("summary") + if not isinstance(summary, dict): + return {} + return summary + + +def _summary_int_dict(summary: dict[str, Any], key: str) -> dict[str, int]: + value = summary.get(key) + if not isinstance(value, dict): + return {} + return { + str(item_key): _safe_int(item_value, 0) + for item_key, item_value in value.items() + if isinstance(item_key, str) + } + + +def _summary_bool_dict(summary: dict[str, Any], key: str) -> dict[str, bool]: + value = summary.get(key) + if not isinstance(value, dict): + return {} + return { + str(item_key): bool(item_value) + for item_key, item_value in value.items() + if isinstance(item_key, str) + } + + +def _retry_counts(summary: dict[str, Any]) -> dict[str, int]: + retry_counts = summary.get("retry_counts") + if not isinstance(retry_counts, dict): + retry_counts = {} + return { + "retries_scheduled_count": _safe_int( + retry_counts.get("retries_scheduled_count", 0), + 0, + ), + "scholars_with_retries_count": _safe_int( + retry_counts.get("scholars_with_retries_count", 0), + 0, + ), + "retry_exhausted_count": _safe_int( + retry_counts.get("retry_exhausted_count", 0), + 0, + ), + } + + +def extract_run_summary(error_log: object) -> dict[str, Any]: + summary = _summary_dict(error_log) + return { + "succeeded_count": _safe_int(summary.get("succeeded_count", 0)), + "failed_count": _safe_int(summary.get("failed_count", 0)), + "partial_count": _safe_int(summary.get("partial_count", 0)), + "failed_state_counts": _summary_int_dict(summary, "failed_state_counts"), + "failed_reason_counts": _summary_int_dict(summary, "failed_reason_counts"), + "scrape_failure_counts": _summary_int_dict(summary, "scrape_failure_counts"), + "retry_counts": _retry_counts(summary), + "alert_thresholds": _summary_int_dict(summary, "alert_thresholds"), + "alert_flags": _summary_bool_dict(summary, "alert_flags"), + } diff --git a/build/lib/app/services/domains/runs/types.py b/build/lib/app/services/domains/runs/types.py new file mode 100644 index 0000000..2524269 --- /dev/null +++ b/build/lib/app/services/domains/runs/types.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +QUEUE_STATUS_QUEUED = "queued" +QUEUE_STATUS_RETRYING = "retrying" +QUEUE_STATUS_DROPPED = "dropped" + + +@dataclass(frozen=True) +class QueueListItem: + id: int + scholar_profile_id: int + scholar_label: str + status: str + reason: str + dropped_reason: str | None + attempt_count: int + resume_cstart: int + next_attempt_dt: datetime | None + updated_at: datetime + last_error: str | None + last_run_id: int | None + + +@dataclass(frozen=True) +class QueueClearResult: + queue_item_id: int + previous_status: str + + +class QueueTransitionError(RuntimeError): + def __init__(self, *, code: str, message: str, current_status: str) -> None: + super().__init__(message) + self.code = code + self.message = message + self.current_status = current_status diff --git a/build/lib/app/services/domains/scholar/__init__.py b/build/lib/app/services/domains/scholar/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/build/lib/app/services/domains/scholar/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/build/lib/app/services/domains/scholar/author_rows.py b/build/lib/app/services/domains/scholar/author_rows.py new file mode 100644 index 0000000..8acd783 --- /dev/null +++ b/build/lib/app/services/domains/scholar/author_rows.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import re +from html.parser import HTMLParser +from typing import Any +from urllib.parse import parse_qs, urlparse + +from app.services.domains.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS +from app.services.domains.scholar.parser_types import ScholarSearchCandidate +from app.services.domains.scholar.parser_utils import ( + attr_class, + attr_href, + attr_src, + build_absolute_scholar_url, + normalize_space, +) + + +def parse_scholar_id_from_href(href: str | None) -> str | None: + if not href: + return None + parsed = urlparse(href) + query = parse_qs(parsed.query) + user_values = query.get("user") + if not user_values: + return None + candidate = user_values[0].strip() + return candidate or None + + +def _extract_verified_email_domain(value: str | None) -> str | None: + if not value: + return None + match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I) + if not match: + return None + domain = normalize_space(match.group(1)) + return domain or None + + +class ScholarAuthorSearchParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.candidates: list[ScholarSearchCandidate] = [] + self._candidate: dict[str, Any] | None = None + + def _begin_candidate(self) -> None: + self._candidate = { + "depth": 1, + "name_href": None, + "name_parts": [], + "aff_depth": 0, + "aff_parts": [], + "name_depth": 0, + "eml_depth": 0, + "eml_parts": [], + "cby_depth": 0, + "cby_parts": [], + "interest_depth": 0, + "interest_parts": [], + "interests": [], + "image_src": None, + } + + def _increment_capture_depths(self) -> None: + if self._candidate is None: + return + for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"): + if self._candidate[key] > 0: + self._candidate[key] += 1 + + def _finalize_candidate(self) -> None: + if self._candidate is None: + return + + name = normalize_space("".join(self._candidate["name_parts"])) + scholar_id = parse_scholar_id_from_href(self._candidate["name_href"]) + if not name or not scholar_id: + return + + affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None + email_domain = _extract_verified_email_domain( + normalize_space("".join(self._candidate["eml_parts"])) or None + ) + cited_by_text = normalize_space("".join(self._candidate["cby_parts"])) + cited_by_match = re.search(r"\d+", cited_by_text) + cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None + + seen_interests: set[str] = set() + interests: list[str] = [] + for interest in self._candidate["interests"]: + normalized = normalize_space(interest) + if not normalized or normalized in seen_interests: + continue + seen_interests.add(normalized) + interests.append(normalized) + + profile_url = build_absolute_scholar_url(self._candidate["name_href"]) + if not profile_url: + profile_url = ( + "https://scholar.google.com/citations" + f"?hl=en&user={scholar_id}" + ) + + self.candidates.append( + ScholarSearchCandidate( + scholar_id=scholar_id, + display_name=name, + affiliation=affiliation, + email_domain=email_domain, + cited_by_count=cited_by_count, + interests=interests, + profile_url=profile_url, + profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]), + ) + ) + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + classes = attr_class(attrs) + + if self._candidate is None: + if tag == "div" and "gsc_1usr" in classes: + self._begin_candidate() + return + + self._candidate["depth"] += 1 + self._increment_capture_depths() + + if tag == "a" and "gs_ai_name" in classes: + self._candidate["name_depth"] = 1 + self._candidate["name_href"] = attr_href(attrs) + return + + if tag == "div" and "gs_ai_aff" in classes: + self._candidate["aff_depth"] = 1 + return + + if tag == "div" and "gs_ai_eml" in classes: + self._candidate["eml_depth"] = 1 + return + + if tag == "div" and "gs_ai_cby" in classes: + self._candidate["cby_depth"] = 1 + return + + if tag == "a" and "gs_ai_one_int" in classes: + self._candidate["interest_depth"] = 1 + self._candidate["interest_parts"] = [] + return + + if tag == "img" and self._candidate["image_src"] is None: + self._candidate["image_src"] = attr_src(attrs) + + def handle_data(self, data: str) -> None: + if self._candidate is None: + return + if self._candidate["name_depth"] > 0: + self._candidate["name_parts"].append(data) + if self._candidate["aff_depth"] > 0: + self._candidate["aff_parts"].append(data) + if self._candidate["eml_depth"] > 0: + self._candidate["eml_parts"].append(data) + if self._candidate["cby_depth"] > 0: + self._candidate["cby_parts"].append(data) + if self._candidate["interest_depth"] > 0: + self._candidate["interest_parts"].append(data) + + def _decrement_capture_depth(self, key: str) -> bool: + if self._candidate is None: + return False + if self._candidate[key] <= 0: + return False + self._candidate[key] -= 1 + return self._candidate[key] == 0 + + def handle_endtag(self, _tag: str) -> None: + if self._candidate is None: + return + + interest_closed = self._decrement_capture_depth("interest_depth") + self._decrement_capture_depth("name_depth") + self._decrement_capture_depth("aff_depth") + self._decrement_capture_depth("eml_depth") + self._decrement_capture_depth("cby_depth") + + if interest_closed: + interest_text = normalize_space("".join(self._candidate["interest_parts"])) + if interest_text: + self._candidate["interests"].append(interest_text) + self._candidate["interest_parts"] = [] + + self._candidate["depth"] -= 1 + if self._candidate["depth"] > 0: + return + + self._finalize_candidate() + self._candidate = None + + +def count_author_search_markers(html: str) -> dict[str, int]: + lowered = html.lower() + return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS} diff --git a/build/lib/app/services/domains/scholar/parser.py b/build/lib/app/services/domains/scholar/parser.py new file mode 100644 index 0000000..060e825 --- /dev/null +++ b/build/lib/app/services/domains/scholar/parser.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +from app.services.domains.scholar.author_rows import ( + ScholarAuthorSearchParser, + count_author_search_markers, + parse_scholar_id_from_href, +) +from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE +from app.services.domains.scholar.parser_types import ( + ParseState, + ParsedAuthorSearchPage, + ParsedProfilePage, + PublicationCandidate, + ScholarDomInvariantError, + ScholarMalformedDataError, + ScholarParserError, + ScholarSearchCandidate, +) +from app.services.domains.scholar.parser_utils import ( + strip_tags, +) +from app.services.domains.scholar.profile_rows import ( + ScholarRowParser, + count_markers, + extract_articles_range, + extract_profile_image_url, + extract_profile_name, + extract_rows, + has_operation_error_banner, + has_show_more_button, + parse_citation_count, + parse_cluster_id_from_href, + parse_publications, + parse_year, +) +from app.services.domains.scholar.source import FetchResult +from app.services.domains.scholar.state_detection import ( + classify_block_or_captcha_reason, + classify_network_error_reason, + detect_author_search_state, + detect_state, +) + + +def _raise_dom_error(code: str, message: str) -> None: + raise ScholarDomInvariantError(code=code, message=message) + + +def _assert_profile_dom_invariants( + *, + fetch_result: FetchResult, + marker_counts: dict[str, int], + publications: list[PublicationCandidate], + warnings: list[str], + has_show_more_button_flag: bool, + articles_range: str | None, +) -> None: + if fetch_result.status_code is None: + return + final_url = (fetch_result.final_url or "").lower() + if "accounts.google.com" in final_url or "sorry/index" in final_url: + return + if any(code.startswith("layout_") for code in warnings): + reason = next(code for code in warnings if code.startswith("layout_")) + _raise_dom_error(reason, f"Detected layout warning: {reason}") + if has_show_more_button_flag and not articles_range: + _raise_dom_error( + "layout_show_more_without_articles_range", + "Show-more control exists without an articles range marker.", + ) + if marker_counts.get("gsc_a_tr", 0) > 0 and marker_counts.get("gsc_a_at", 0) <= 0: + _raise_dom_error( + "layout_missing_publication_title_anchor", + "Publication rows were present but title anchors were absent.", + ) + if not publications: + has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0 + has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0 + if not has_profile_markers and not has_table_markers: + _raise_dom_error("layout_markers_missing", "Expected scholar profile markers were absent.") + for publication in publications: + if not publication.title.strip(): + raise ScholarMalformedDataError( + code="malformed_publication_title", + message="Encountered a publication candidate with an empty title.", + ) + if publication.citation_count is not None and int(publication.citation_count) < 0: + raise ScholarMalformedDataError( + code="malformed_publication_negative_citations", + message="Encountered a publication candidate with negative citations.", + ) + + +def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage: + publications, warnings = parse_publications(fetch_result.body) + marker_counts = count_markers(fetch_result.body) + visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower() + + show_more = has_show_more_button(fetch_result.body) + operation_error_banner = has_operation_error_banner(fetch_result.body) + articles_range = extract_articles_range(fetch_result.body) + + if show_more: + warnings.append("possible_partial_page_show_more_present") + if operation_error_banner: + warnings.append("operation_error_banner_present") + + warnings = sorted(set(warnings)) + _assert_profile_dom_invariants( + fetch_result=fetch_result, + marker_counts=marker_counts, + publications=publications, + warnings=warnings, + has_show_more_button_flag=show_more, + articles_range=articles_range, + ) + + state, state_reason = detect_state( + fetch_result, + publications, + marker_counts, + warnings=warnings, + has_show_more_button_flag=show_more, + articles_range=articles_range, + visible_text=visible_text, + ) + + return ParsedProfilePage( + state=state, + state_reason=state_reason, + profile_name=extract_profile_name(fetch_result.body), + profile_image_url=extract_profile_image_url(fetch_result.body), + publications=publications, + marker_counts=marker_counts, + warnings=warnings, + has_show_more_button=show_more, + has_operation_error_banner=operation_error_banner, + articles_range=articles_range, + ) + + +def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage: + parser = ScholarAuthorSearchParser() + parser.feed(fetch_result.body) + + marker_counts = count_author_search_markers(fetch_result.body) + visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower() + warnings: list[str] = [] + if not parser.candidates: + warnings.append("no_author_candidates_detected") + + state, state_reason = detect_author_search_state( + fetch_result, + parser.candidates, + marker_counts, + visible_text=visible_text, + ) + + return ParsedAuthorSearchPage( + state=state, + state_reason=state_reason, + candidates=parser.candidates, + marker_counts=marker_counts, + warnings=warnings, + ) diff --git a/build/lib/app/services/domains/scholar/parser_constants.py b/build/lib/app/services/domains/scholar/parser_constants.py new file mode 100644 index 0000000..c3c7a15 --- /dev/null +++ b/build/lib/app/services/domains/scholar/parser_constants.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import re + +BLOCKED_KEYWORDS = [ + "unusual traffic", + "sorry/index", + "not a robot", + "our systems have detected", + "automated queries", + "recaptcha", + "captcha", +] + +NO_RESULTS_KEYWORDS = [ + "didn't match any articles", + "did not match any articles", + "no articles", + "no documents", +] + +NO_AUTHOR_RESULTS_KEYWORDS = [ + "didn't match any user profiles", + "did not match any user profiles", + "didn't match any scholars", + "did not match any scholars", + "no user profiles", +] + +MARKER_KEYS = [ + "gsc_a_tr", + "gsc_a_at", + "gsc_a_ac", + "gsc_a_h", + "gsc_a_y", + "gs_gray", + "gsc_prf_in", + "gsc_rsb_st", +] + +AUTHOR_SEARCH_MARKER_KEYS = [ + "gsc_1usr", + "gs_ai_name", + "gs_ai_aff", + "gs_ai_eml", + "gs_ai_cby", + "gs_ai_one_int", +] + +NETWORK_DNS_ERROR_KEYWORDS = [ + "temporary failure in name resolution", + "name or service not known", + "nodename nor servname provided", + "getaddrinfo failed", +] + +NETWORK_TIMEOUT_KEYWORDS = [ + "timed out", + "timeout", +] + +NETWORK_TLS_ERROR_KEYWORDS = [ + "ssl", + "tls", + "certificate verify failed", +] + +TAG_RE = re.compile(r"<[^>]+>", re.S) +SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?", re.I | re.S) +SHOW_MORE_BUTTON_RE = re.compile( + r"]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>", + re.I | re.S, +) + +PROFILE_ROW_PARSER_DIRECT_MARKERS = ( + "gs_ggs", + "gs_ggsd", + "gs_ggsa", + "gs_or_ggsm", +) + +PROFILE_ROW_DIRECT_LABEL_TOKENS = ( + "pdf", + "[pdf]", + "full text", + "download", +) diff --git a/build/lib/app/services/domains/scholar/parser_types.py b/build/lib/app/services/domains/scholar/parser_types.py new file mode 100644 index 0000000..7b657d8 --- /dev/null +++ b/build/lib/app/services/domains/scholar/parser_types.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + + +class ParseState(StrEnum): + OK = "ok" + NO_RESULTS = "no_results" + BLOCKED_OR_CAPTCHA = "blocked_or_captcha" + LAYOUT_CHANGED = "layout_changed" + NETWORK_ERROR = "network_error" + + +class ScholarParserError(RuntimeError): + code: str + + def __init__(self, *, code: str, message: str) -> None: + super().__init__(message) + self.code = code + + +class ScholarDomInvariantError(ScholarParserError): + pass + + +class ScholarMalformedDataError(ScholarParserError): + pass + + +@dataclass(frozen=True) +class PublicationCandidate: + title: str + title_url: str | None + cluster_id: str | None + year: int | None + citation_count: int | None + authors_text: str | None + venue_text: str | None + pdf_url: str | None + + +@dataclass(frozen=True) +class ScholarSearchCandidate: + scholar_id: str + display_name: str + affiliation: str | None + email_domain: str | None + cited_by_count: int | None + interests: list[str] + profile_url: str + profile_image_url: str | None + + +@dataclass(frozen=True) +class ParsedProfilePage: + state: ParseState + state_reason: str + profile_name: str | None + profile_image_url: str | None + publications: list[PublicationCandidate] + marker_counts: dict[str, int] + warnings: list[str] + has_show_more_button: bool + has_operation_error_banner: bool + articles_range: str | None + + +@dataclass(frozen=True) +class ParsedAuthorSearchPage: + state: ParseState + state_reason: str + candidates: list[ScholarSearchCandidate] + marker_counts: dict[str, int] + warnings: list[str] diff --git a/build/lib/app/services/domains/scholar/parser_utils.py b/build/lib/app/services/domains/scholar/parser_utils.py new file mode 100644 index 0000000..1bf8687 --- /dev/null +++ b/build/lib/app/services/domains/scholar/parser_utils.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from html import unescape + +from app.services.domains.scholar.parser_constants import TAG_RE + + +def normalize_space(value: str) -> str: + return " ".join(unescape(value).split()) + + +def strip_tags(value: str) -> str: + return normalize_space(TAG_RE.sub(" ", value)) + + +def attr_class(attrs: list[tuple[str, str | None]]) -> str: + for name, raw_value in attrs: + if name.lower() == "class": + return raw_value or "" + return "" + + +def attr_href(attrs: list[tuple[str, str | None]]) -> str | None: + for name, raw_value in attrs: + if name.lower() == "href": + return raw_value + return None + + +def attr_src(attrs: list[tuple[str, str | None]]) -> str | None: + for name, raw_value in attrs: + if name.lower() == "src": + return raw_value + return None + + +def build_absolute_scholar_url(path_or_url: str | None) -> str | None: + if not path_or_url: + return None + from urllib.parse import urljoin + + return urljoin("https://scholar.google.com", path_or_url) diff --git a/build/lib/app/services/domains/scholar/profile_rows.py b/build/lib/app/services/domains/scholar/profile_rows.py new file mode 100644 index 0000000..6a63274 --- /dev/null +++ b/build/lib/app/services/domains/scholar/profile_rows.py @@ -0,0 +1,270 @@ +from __future__ import annotations + +import re +from html.parser import HTMLParser +from urllib.parse import parse_qs, urlparse + +from app.services.domains.scholar.parser_constants import ( + MARKER_KEYS, + SHOW_MORE_BUTTON_RE, +) +from app.services.domains.scholar.parser_types import PublicationCandidate +from app.services.domains.scholar.parser_utils import ( + attr_class, + attr_href, + attr_src, + build_absolute_scholar_url, + normalize_space, + strip_tags, +) + + +class ScholarRowParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.title_href: str | None = None + self.title_parts: list[str] = [] + self.citation_parts: list[str] = [] + self.year_parts: list[str] = [] + self.gray_texts: list[str] = [] + + self._title_depth = 0 + self._citation_depth = 0 + self._year_depth = 0 + self._gray_stack: list[dict[str, object]] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if self._title_depth > 0: + self._title_depth += 1 + if self._citation_depth > 0: + self._citation_depth += 1 + if self._year_depth > 0: + self._year_depth += 1 + if self._gray_stack: + self._gray_stack[-1]["depth"] += 1 + + classes = attr_class(attrs) + + if tag == "a" and "gsc_a_at" in classes: + self._title_depth = 1 + self.title_href = attr_href(attrs) + return + + if tag == "a" and "gsc_a_ac" in classes: + self._citation_depth = 1 + return + + if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes): + self._year_depth = 1 + return + + if tag == "div" and "gs_gray" in classes: + self._gray_stack.append({"depth": 1, "parts": []}) + return + + def handle_data(self, data: str) -> None: + if self._title_depth > 0: + self.title_parts.append(data) + if self._citation_depth > 0: + self.citation_parts.append(data) + if self._year_depth > 0: + self.year_parts.append(data) + if self._gray_stack: + self._gray_stack[-1]["parts"].append(data) + + def handle_endtag(self, tag: str) -> None: + if self._title_depth > 0: + self._title_depth -= 1 + if self._citation_depth > 0: + self._citation_depth -= 1 + if self._year_depth > 0: + self._year_depth -= 1 + if self._gray_stack: + self._gray_stack[-1]["depth"] -= 1 + if self._gray_stack[-1]["depth"] == 0: + text_value = normalize_space("".join(self._gray_stack[-1]["parts"])) + if text_value: + self.gray_texts.append(text_value) + self._gray_stack.pop() + + +def extract_rows(html: str) -> list[str]: + pattern = re.compile( + r"]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)", + re.I | re.S, + ) + return [match.group(1) for match in pattern.finditer(html)] + + +def parse_cluster_id_from_href(href: str | None) -> str | None: + if not href: + return None + parsed = urlparse(href) + query = parse_qs(parsed.query) + + citation_for_view = query.get("citation_for_view") + if citation_for_view: + token = citation_for_view[0].strip() + if token: + return f"cfv:{token}" + + cluster = query.get("cluster") + if cluster: + token = cluster[0].strip() + if token: + return f"cluster:{token}" + return None + + +def parse_year(parts: list[str]) -> int | None: + text = normalize_space(" ".join(parts)) + match = re.search(r"\b(19|20)\d{2}\b", text) + if not match: + return None + try: + return int(match.group(0)) + except ValueError: + return None + + +def parse_citation_count(parts: list[str]) -> int | None: + text = normalize_space(" ".join(parts)) + if not text: + return 0 + digits = re.sub(r"\D+", "", text) + if not digits: + return None + return int(digits) + + +def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]: + parser = ScholarRowParser() + parser.feed(row_html) + warnings: list[str] = [] + title = normalize_space("".join(parser.title_parts)) + if not title: + warnings.append("row_missing_title") + return None, warnings + if not parser.title_href: + warnings.append("row_missing_title_href") + + citation_text = normalize_space(" ".join(parser.citation_parts)) + citation_count = parse_citation_count(parser.citation_parts) + if citation_text and citation_count is None: + warnings.append("layout_row_citation_unparseable") + + year_text = normalize_space(" ".join(parser.year_parts)) + year = parse_year(parser.year_parts) + if year_text and year is None: + warnings.append("layout_row_year_unparseable") + + authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None + venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None + return ( + PublicationCandidate( + title=title, + title_url=parser.title_href, + cluster_id=parse_cluster_id_from_href(parser.title_href), + year=year, + citation_count=citation_count, + authors_text=authors_text, + venue_text=venue_text, + pdf_url=None, + ), + warnings, + ) + + +def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]: + rows = extract_rows(html) + warnings: list[str] = [] + publications: list[PublicationCandidate] = [] + + for row_html in rows: + publication, row_warnings = _parse_publication_row(row_html) + warnings.extend(row_warnings) + if publication is None: + continue + publications.append(publication) + + if not rows: + warnings.append("no_rows_detected") + if rows and not publications: + warnings.append("layout_all_rows_unparseable") + + return publications, sorted(set(warnings)) + + +def extract_profile_name(html: str) -> str | None: + pattern = re.compile( + r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)]+>", + re.I | re.S, + ) + match = pattern.search(html) + if not match: + return None + value = strip_tags(match.group(1)) + return value or None + + +def extract_profile_image_url(html: str) -> str | None: + og_image_pattern = re.compile( + r"]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>", + re.I | re.S, + ) + og_match = og_image_pattern.search(html) + if og_match: + value = normalize_space(og_match.group(1)) + absolute = build_absolute_scholar_url(value) + if absolute: + return absolute + + image_pattern = re.compile( + r"]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>", + re.I | re.S, + ) + image_match = image_pattern.search(html) + if not image_match: + return None + + value = normalize_space(image_match.group(1)) + return build_absolute_scholar_url(value) + + +def extract_articles_range(html: str) -> str | None: + pattern = re.compile( + r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)]+>", + re.I | re.S, + ) + match = pattern.search(html) + if not match: + return None + value = strip_tags(match.group(1)) + return value or None + + +def has_show_more_button(html: str) -> bool: + match = SHOW_MORE_BUTTON_RE.search(html) + if match is None: + return False + + button_tag = match.group(0).lower() + if "disabled" in button_tag: + return False + if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag: + return False + if "gs_dis" in button_tag: + return False + return True + + +def has_operation_error_banner(html: str) -> bool: + lowered = html.lower() + if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered: + return False + return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered + + +def count_markers(html: str) -> dict[str, int]: + lowered = html.lower() + return {key: lowered.count(key.lower()) for key in MARKER_KEYS} diff --git a/build/lib/app/services/domains/scholar/rate_limit.py b/build/lib/app/services/domains/scholar/rate_limit.py new file mode 100644 index 0000000..947c91f --- /dev/null +++ b/build/lib/app/services/domains/scholar/rate_limit.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import asyncio +import time + +_REQUEST_LOCK = asyncio.Lock() +_LAST_REQUEST_AT = 0.0 + + +async def wait_for_scholar_slot(*, min_interval_seconds: float) -> None: + global _LAST_REQUEST_AT + interval = max(float(min_interval_seconds), 0.0) + async with _REQUEST_LOCK: + elapsed = time.monotonic() - _LAST_REQUEST_AT + remaining = interval - elapsed + if remaining > 0: + await asyncio.sleep(remaining) + _LAST_REQUEST_AT = time.monotonic() diff --git a/build/lib/app/services/domains/scholar/source.py b/build/lib/app/services/domains/scholar/source.py new file mode 100644 index 0000000..92c6702 --- /dev/null +++ b/build/lib/app/services/domains/scholar/source.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +import asyncio +import logging +import random +from dataclasses import dataclass +from typing import Protocol +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations" +DEFAULT_PAGE_SIZE = 100 + +DEFAULT_USER_AGENTS = [ + ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" + ), + ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 " + "(KHTML, like Gecko) Version/18.1 Safari/605.1.15" + ), + ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) " + "Gecko/20100101 Firefox/131.0" + ), +] + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class FetchResult: + requested_url: str + status_code: int | None + final_url: str | None + body: str + error: str | None + + +class ScholarSource(Protocol): + async def fetch_profile_html(self, scholar_id: str) -> FetchResult: + ... + + async def fetch_profile_page_html( + self, + scholar_id: str, + *, + cstart: int, + pagesize: int, + ) -> FetchResult: + ... + + async def fetch_author_search_html( + self, + query: str, + *, + start: int, + ) -> FetchResult: + ... + + +class LiveScholarSource: + def __init__( + self, + *, + timeout_seconds: float = 25.0, + user_agents: list[str] | None = None, + ) -> None: + self._timeout_seconds = timeout_seconds + self._user_agents = user_agents or DEFAULT_USER_AGENTS + + async def fetch_profile_html(self, scholar_id: str) -> FetchResult: + return await self.fetch_profile_page_html( + scholar_id, + cstart=0, + pagesize=DEFAULT_PAGE_SIZE, + ) + + async def fetch_profile_page_html( + self, + scholar_id: str, + *, + cstart: int, + pagesize: int = DEFAULT_PAGE_SIZE, + ) -> FetchResult: + requested_url = _build_profile_url( + scholar_id=scholar_id, + cstart=cstart, + pagesize=pagesize, + ) + logger.debug( + "scholar_source.fetch_started", + extra={ + "event": "scholar_source.fetch_started", + "scholar_id": scholar_id, + "requested_url": requested_url, + "cstart": cstart, + "pagesize": pagesize, + }, + ) + return await asyncio.to_thread(self._fetch_sync, requested_url) + + async def fetch_author_search_html( + self, + query: str, + *, + start: int = 0, + ) -> FetchResult: + requested_url = _build_author_search_url( + query=query, + start=start, + ) + logger.debug( + "scholar_source.search_fetch_started", + extra={ + "event": "scholar_source.search_fetch_started", + "query": query, + "requested_url": requested_url, + "start": start, + }, + ) + return await asyncio.to_thread(self._fetch_sync, requested_url) + + async def fetch_publication_html(self, publication_url: str) -> FetchResult: + logger.debug( + "scholar_source.publication_fetch_started", + extra={ + "event": "scholar_source.publication_fetch_started", + "requested_url": publication_url, + }, + ) + return await asyncio.to_thread(self._fetch_sync, publication_url) + + def _build_request(self, requested_url: str) -> Request: + return Request( + requested_url, + headers={ + "User-Agent": random.choice(self._user_agents), + "Accept": "text/html,application/xhtml+xml", + "Accept-Language": "en-US,en;q=0.9", + "Connection": "close", + }, + ) + + @staticmethod + def _http_error_body(exc: HTTPError) -> str: + try: + return exc.read().decode("utf-8", errors="replace") + except Exception: + return "" + + @staticmethod + def _network_error_result(requested_url: str, exc: URLError) -> FetchResult: + logger.warning( + "scholar_source.fetch_network_error", + extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url}, + ) + return FetchResult( + requested_url=requested_url, + status_code=None, + final_url=None, + body="", + error=str(exc), + ) + + @staticmethod + def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult: + logger.warning( + "scholar_source.fetch_http_error", + extra={ + "event": "scholar_source.fetch_http_error", + "requested_url": requested_url, + "status_code": exc.code, + }, + ) + return FetchResult( + requested_url=requested_url, + status_code=exc.code, + final_url=exc.geturl(), + body=LiveScholarSource._http_error_body(exc), + error=str(exc), + ) + + @staticmethod + def _success_result(requested_url: str, response) -> FetchResult: + body = response.read().decode("utf-8", errors="replace") + status_code = getattr(response, "status", 200) + logger.debug( + "scholar_source.fetch_succeeded", + extra={ + "event": "scholar_source.fetch_succeeded", + "requested_url": requested_url, + "status_code": status_code, + }, + ) + return FetchResult( + requested_url=requested_url, + status_code=status_code, + final_url=response.geturl(), + body=body, + error=None, + ) + + def _fetch_sync(self, requested_url: str) -> FetchResult: + request = self._build_request(requested_url) + + try: + with urlopen(request, timeout=self._timeout_seconds) as response: + return self._success_result(requested_url, response) + except HTTPError as exc: + return self._http_error_result(requested_url, exc) + except URLError as exc: + return self._network_error_result(requested_url, exc) + + +def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str: + query: dict[str, int | str] = {"hl": "en", "user": scholar_id} + if cstart > 0: + query["cstart"] = int(cstart) + if pagesize > 0: + query["pagesize"] = int(pagesize) + return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}" + + +def _build_author_search_url(*, query: str, start: int) -> str: + params: dict[str, int | str] = { + "hl": "en", + "view_op": "search_authors", + "mauthors": query, + } + if start > 0: + params["astart"] = int(start) + return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}" diff --git a/build/lib/app/services/domains/scholar/state_detection.py b/build/lib/app/services/domains/scholar/state_detection.py new file mode 100644 index 0000000..cdc8780 --- /dev/null +++ b/build/lib/app/services/domains/scholar/state_detection.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +from app.services.domains.scholar.parser_constants import ( + BLOCKED_KEYWORDS, + NETWORK_DNS_ERROR_KEYWORDS, + NETWORK_TIMEOUT_KEYWORDS, + NETWORK_TLS_ERROR_KEYWORDS, + NO_AUTHOR_RESULTS_KEYWORDS, + NO_RESULTS_KEYWORDS, +) +from app.services.domains.scholar.parser_types import ParseState, PublicationCandidate, ScholarSearchCandidate +from app.services.domains.scholar.source import FetchResult + + +def classify_network_error_reason(fetch_error: str | None) -> str: + lowered = (fetch_error or "").lower() + if lowered: + if any(keyword in lowered for keyword in NETWORK_DNS_ERROR_KEYWORDS): + return "network_dns_resolution_failed" + if any(keyword in lowered for keyword in NETWORK_TIMEOUT_KEYWORDS): + return "network_timeout" + if any(keyword in lowered for keyword in NETWORK_TLS_ERROR_KEYWORDS): + return "network_tls_error" + if "connection reset" in lowered: + return "network_connection_reset" + if "connection refused" in lowered: + return "network_connection_refused" + if "network is unreachable" in lowered: + return "network_unreachable" + return "network_error_missing_status_code" + + +def classify_block_or_captcha_reason( + *, + status_code: int, + final_url: str, + body_lowered: str, +) -> str | None: + if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url): + return "blocked_accounts_redirect" + if status_code == 429: + return "blocked_http_429_rate_limited" + if status_code == 403: + if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url: + return "blocked_http_403_captcha_challenge" + return "blocked_http_403_forbidden" + if "sorry/index" in final_url or "sorry/index" in body_lowered: + return "blocked_google_sorry_challenge" + if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered: + return "blocked_unusual_traffic_detected" + if "automated queries" in body_lowered: + return "blocked_automated_queries_detected" + if "not a robot" in body_lowered: + return "blocked_not_a_robot_challenge" + if "recaptcha" in body_lowered: + return "blocked_recaptcha_challenge" + if "captcha" in body_lowered: + return "blocked_captcha_challenge" + if any(keyword in body_lowered for keyword in BLOCKED_KEYWORDS): + return "blocked_keyword_detected" + return None + + +def _warnings_contain(warnings: list[str], code: str) -> bool: + return any(item == code for item in warnings) + + +def _has_layout_row_failure(marker_counts: dict[str, int], warnings: list[str]) -> bool: + if _warnings_contain(warnings, "layout_all_rows_unparseable"): + return True + if marker_counts.get("gsc_a_tr", 0) <= 0: + return False + if _warnings_contain(warnings, "row_missing_title"): + return True + return marker_counts.get("gsc_a_at", 0) <= 0 + + +def _first_layout_warning(warnings: list[str]) -> str | None: + for warning in warnings: + if warning.startswith("layout_"): + return warning + return None + + +def detect_state( + fetch_result: FetchResult, + publications: list[PublicationCandidate], + marker_counts: dict[str, int], + *, + warnings: list[str], + has_show_more_button_flag: bool, + articles_range: str | None, + visible_text: str, +) -> tuple[ParseState, str]: + if fetch_result.status_code is None: + return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error) + + lowered = fetch_result.body.lower() + final = (fetch_result.final_url or "").lower() + status_code = int(fetch_result.status_code) + + block_reason = classify_block_or_captcha_reason( + status_code=status_code, + final_url=final, + body_lowered=lowered, + ) + if block_reason is not None: + return ParseState.BLOCKED_OR_CAPTCHA, block_reason + + if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS): + return ParseState.NO_RESULTS, "no_results_keyword_detected" + + layout_warning = _first_layout_warning(warnings) + if layout_warning is not None: + return ParseState.LAYOUT_CHANGED, layout_warning + + if _has_layout_row_failure(marker_counts, warnings): + return ParseState.LAYOUT_CHANGED, "layout_publication_rows_unparseable" + + if has_show_more_button_flag and not articles_range: + return ParseState.LAYOUT_CHANGED, "layout_show_more_without_articles_range" + + if not publications: + has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0 + has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0 + if not has_profile_markers and not has_table_markers: + return ParseState.LAYOUT_CHANGED, "layout_markers_missing" + return ParseState.OK, "no_rows_with_known_markers" + + return ParseState.OK, "publications_extracted" + + +def detect_author_search_state( + fetch_result: FetchResult, + candidates: list[ScholarSearchCandidate], + marker_counts: dict[str, int], + *, + visible_text: str, +) -> tuple[ParseState, str]: + if fetch_result.status_code is None: + return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error) + + lowered = fetch_result.body.lower() + final = (fetch_result.final_url or "").lower() + status_code = int(fetch_result.status_code) + + block_reason = classify_block_or_captcha_reason( + status_code=status_code, + final_url=final, + body_lowered=lowered, + ) + if block_reason is not None: + return ParseState.BLOCKED_OR_CAPTCHA, block_reason + + if not candidates and any(keyword in visible_text for keyword in NO_AUTHOR_RESULTS_KEYWORDS): + return ParseState.NO_RESULTS, "no_results_keyword_detected" + + if not candidates: + has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0 + if not has_search_markers: + return ParseState.NO_RESULTS, "no_search_candidates_detected" + return ParseState.LAYOUT_CHANGED, "layout_author_candidates_unparseable" + + return ParseState.OK, "author_candidates_extracted" diff --git a/build/lib/app/services/domains/scholars/__init__.py b/build/lib/app/services/domains/scholars/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/build/lib/app/services/domains/scholars/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/build/lib/app/services/domains/scholars/application.py b/build/lib/app/services/domains/scholars/application.py new file mode 100644 index 0000000..8da041f --- /dev/null +++ b/build/lib/app/services/domains/scholars/application.py @@ -0,0 +1,974 @@ +from __future__ import annotations + +import asyncio +from dataclasses import replace +from datetime import datetime +from datetime import timedelta +from datetime import timezone +import logging +import os +import random +from uuid import uuid4 + +from sqlalchemy import delete, func, select, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile +from app.services.domains.scholar.parser import ( + ParseState, + ParsedAuthorSearchPage, + ScholarParserError, + parse_author_search_page, + parse_profile_page, +) +from app.services.domains.scholar.source import ScholarSource +from app.services.domains.scholars.constants import ( + ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES, + AUTHOR_SEARCH_LOCK_KEY, + AUTHOR_SEARCH_LOCK_NAMESPACE, + AUTHOR_SEARCH_RUNTIME_STATE_KEY, + DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, + DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, + DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, + DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD, + DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, + DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, + MAX_AUTHOR_SEARCH_LIMIT, + DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, + DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, + SEARCH_CACHED_BLOCK_REASON, + SEARCH_COOLDOWN_REASON, + SEARCH_DISABLED_REASON, +) +from app.services.domains.scholars.exceptions import ScholarServiceError +from app.services.domains.scholars.search_hints import ( + _merge_warnings, + _policy_blocked_author_search_result, + _trim_author_search_result, + resolve_profile_image, + scrape_state_hint, +) +from app.services.domains.scholars.uploads import ( + _ensure_upload_root, + _resolve_upload_path, + _safe_remove_upload, + resolve_upload_file_path, +) +from app.services.domains.scholars.validators import ( + normalize_display_name, + normalize_profile_image_url, + validate_scholar_id, +) +logger = logging.getLogger(__name__) + + +async def _acquire_author_search_lock(db_session: AsyncSession) -> None: + await db_session.execute( + text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"), + { + "namespace": AUTHOR_SEARCH_LOCK_NAMESPACE, + "lock_key": AUTHOR_SEARCH_LOCK_KEY, + }, + ) + + +async def _load_runtime_state_for_update( + db_session: AsyncSession, +) -> AuthorSearchRuntimeState: + result = await db_session.execute( + select(AuthorSearchRuntimeState) + .where(AuthorSearchRuntimeState.state_key == AUTHOR_SEARCH_RUNTIME_STATE_KEY) + .with_for_update() + ) + state = result.scalar_one_or_none() + if state is not None: + return state + + state = AuthorSearchRuntimeState(state_key=AUTHOR_SEARCH_RUNTIME_STATE_KEY) + db_session.add(state) + await db_session.flush() + return state + + +def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict: + return { + "state": parsed.state.value, + "state_reason": parsed.state_reason, + "marker_counts": {str(key): int(value) for key, value in parsed.marker_counts.items()}, + "warnings": [str(value) for value in parsed.warnings], + "candidates": [ + { + "scholar_id": candidate.scholar_id, + "display_name": candidate.display_name, + "affiliation": candidate.affiliation, + "email_domain": candidate.email_domain, + "cited_by_count": candidate.cited_by_count, + "interests": [str(interest) for interest in candidate.interests], + "profile_url": candidate.profile_url, + "profile_image_url": candidate.profile_image_url, + } + for candidate in parsed.candidates + ], + } + + +def _payload_state(payload: dict[str, object]) -> ParseState | None: + state_raw = str(payload.get("state", "")).strip() + try: + return ParseState(state_raw) + except ValueError: + return None + + +def _payload_marker_counts(payload: dict[str, object]) -> dict[str, int]: + marker_counts_payload = payload.get("marker_counts") + if not isinstance(marker_counts_payload, dict): + return {} + parsed: dict[str, int] = {} + for key, value in marker_counts_payload.items(): + try: + parsed[str(key)] = int(value) + except (TypeError, ValueError): + continue + return parsed + + +def _payload_warnings(payload: dict[str, object]) -> list[str]: + warnings_payload = payload.get("warnings") + if not isinstance(warnings_payload, list): + return [] + return [str(value) for value in warnings_payload if isinstance(value, str)] + + +def _parse_optional_string(value: object) -> str | None: + if value is None: + return None + normalized = str(value).strip() + return normalized or None + + +def _parse_optional_int(value: object) -> int | None: + if isinstance(value, int): + return value + if isinstance(value, str) and value.strip(): + try: + return int(value) + except ValueError: + return None + return None + + +def _normalize_interests(value: object) -> list[str]: + if not isinstance(value, list): + return [] + return [str(item) for item in value if isinstance(item, str)] + + +def _deserialize_candidate_payload(value: object) -> dict[str, object] | None: + if not isinstance(value, dict): + return None + scholar_id = str(value.get("scholar_id", "")).strip() + display_name = str(value.get("display_name", "")).strip() + profile_url = str(value.get("profile_url", "")).strip() + if not scholar_id or not display_name or not profile_url: + return None + return { + "scholar_id": scholar_id, + "display_name": display_name, + "affiliation": _parse_optional_string(value.get("affiliation")), + "email_domain": _parse_optional_string(value.get("email_domain")), + "cited_by_count": _parse_optional_int(value.get("cited_by_count")), + "interests": _normalize_interests(value.get("interests")), + "profile_url": profile_url, + "profile_image_url": _parse_optional_string(value.get("profile_image_url")), + } + + +def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, object]]: + candidates_payload = payload.get("candidates") + if not isinstance(candidates_payload, list): + return [] + normalized: list[dict[str, object]] = [] + for value in candidates_payload: + candidate = _deserialize_candidate_payload(value) + if candidate is not None: + normalized.append(candidate) + return normalized + + +def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None: + if not isinstance(payload, dict): + return None + + state = _payload_state(payload) + if state is None: + return None + + marker_counts = _payload_marker_counts(payload) + warnings = _payload_warnings(payload) + from app.services.domains.scholar.parser import ScholarSearchCandidate + + normalized_candidates = _deserialize_candidates(payload) + + return ParsedAuthorSearchPage( + state=state, + state_reason=str(payload.get("state_reason", "")).strip() or "unknown", + candidates=[ + ScholarSearchCandidate( + scholar_id=item["scholar_id"], + display_name=item["display_name"], + affiliation=item["affiliation"], + email_domain=item["email_domain"], + cited_by_count=item["cited_by_count"], + interests=item["interests"], + profile_url=item["profile_url"], + profile_image_url=item["profile_image_url"], + ) + for item in normalized_candidates + ], + marker_counts=marker_counts, + warnings=warnings, + ) + + +async def _cache_get_author_search_result( + db_session: AsyncSession, + *, + query_key: str, + now_utc: datetime, +) -> ParsedAuthorSearchPage | None: + result = await db_session.execute( + select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key) + ) + entry = result.scalar_one_or_none() + if entry is None: + return None + expires_at = entry.expires_at + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=timezone.utc) + if expires_at <= now_utc: + await db_session.delete(entry) + return None + parsed = _deserialize_parsed_author_search_page(entry.payload) + if parsed is None: + await db_session.delete(entry) + return None + return parsed + + +async def _cache_set_author_search_result( + db_session: AsyncSession, + *, + query_key: str, + parsed: ParsedAuthorSearchPage, + ttl_seconds: float, + max_entries: int, + now_utc: datetime, +) -> None: + ttl = max(float(ttl_seconds), 0.0) + existing_result = await db_session.execute( + select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key) + ) + existing = existing_result.scalar_one_or_none() + + if ttl <= 0.0: + if existing is not None: + await db_session.delete(existing) + return + + expires_at = now_utc + timedelta(seconds=ttl) + payload = _serialize_parsed_author_search_page(parsed) + if existing is None: + db_session.add( + AuthorSearchCacheEntry( + query_key=query_key, + payload=payload, + expires_at=expires_at, + cached_at=now_utc, + updated_at=now_utc, + ) + ) + else: + existing.payload = payload + existing.expires_at = expires_at + existing.cached_at = now_utc + existing.updated_at = now_utc + + await _prune_author_search_cache(db_session, now_utc=now_utc, max_entries=max_entries) + + +async def _prune_author_search_cache( + db_session: AsyncSession, + *, + now_utc: datetime, + max_entries: int, +) -> None: + await db_session.execute( + delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.expires_at <= now_utc) + ) + bounded_max_entries = max(1, int(max_entries)) + count_result = await db_session.execute( + select(func.count()).select_from(AuthorSearchCacheEntry) + ) + entry_count = int(count_result.scalar_one() or 0) + overflow = max(0, entry_count - bounded_max_entries) + if overflow <= 0: + return + stale_keys_result = await db_session.execute( + select(AuthorSearchCacheEntry.query_key) + .order_by(AuthorSearchCacheEntry.cached_at.asc()) + .limit(overflow) + ) + stale_keys = [str(row[0]) for row in stale_keys_result.all()] + if stale_keys: + await db_session.execute( + delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key.in_(stale_keys)) + ) + + +def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool: + return parsed.state == ParseState.BLOCKED_OR_CAPTCHA + + +def _author_search_cooldown_remaining_seconds( + runtime_state: AuthorSearchRuntimeState, + now_utc: datetime, +) -> int: + cooldown_until = runtime_state.cooldown_until + if cooldown_until is None: + return 0 + if cooldown_until.tzinfo is None: + cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) + remaining_seconds = int((cooldown_until - now_utc).total_seconds()) + return max(0, remaining_seconds) + + +async def list_scholars_for_user( + db_session: AsyncSession, + *, + user_id: int, +) -> list[ScholarProfile]: + result = await db_session.execute( + select(ScholarProfile) + .where(ScholarProfile.user_id == user_id) + .order_by(ScholarProfile.created_at.desc(), ScholarProfile.id.desc()) + ) + return list(result.scalars().all()) + + +async def create_scholar_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_id: str, + display_name: str, + profile_image_url: str | None = None, +) -> ScholarProfile: + profile = ScholarProfile( + user_id=user_id, + scholar_id=validate_scholar_id(scholar_id), + display_name=normalize_display_name(display_name), + profile_image_url=normalize_profile_image_url(profile_image_url), + ) + db_session.add(profile) + try: + await db_session.commit() + except IntegrityError as exc: + await db_session.rollback() + raise ScholarServiceError("That scholar is already tracked for this account.") from exc + await db_session.refresh(profile) + return profile + + +async def get_user_scholar_by_id( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, +) -> ScholarProfile | None: + result = await db_session.execute( + select(ScholarProfile).where( + ScholarProfile.id == scholar_profile_id, + ScholarProfile.user_id == user_id, + ) + ) + return result.scalar_one_or_none() + + +async def toggle_scholar_enabled( + db_session: AsyncSession, + *, + profile: ScholarProfile, +) -> ScholarProfile: + profile.is_enabled = not profile.is_enabled + await db_session.commit() + await db_session.refresh(profile) + return profile + + +async def delete_scholar( + db_session: AsyncSession, + *, + profile: ScholarProfile, + upload_dir: str | None = None, +) -> None: + if upload_dir: + upload_root = _ensure_upload_root(upload_dir, create=True) + _safe_remove_upload(upload_root, profile.profile_image_upload_path) + + await db_session.delete(profile) + await db_session.commit() + + +def _normalize_author_search_inputs(query: str, limit: int) -> tuple[str, int, str]: + normalized_query = query.strip() + if len(normalized_query) < 2: + raise ScholarServiceError("Search query must be at least 2 characters.") + bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT)) + return normalized_query, bounded_limit, normalized_query.casefold() + + +def _disabled_search_result(*, normalized_query: str, bounded_limit: int) -> ParsedAuthorSearchPage: + logger.warning( + "scholar_search.disabled_by_configuration", + extra={"event": "scholar_search.disabled_by_configuration", "query": normalized_query}, + ) + return _policy_blocked_author_search_result( + reason=SEARCH_DISABLED_REASON, + warning_codes=["author_search_disabled_by_configuration"], + limit=bounded_limit, + ) + + +def _normalize_runtime_cooldown_state( + runtime_state: AuthorSearchRuntimeState, + *, + now_utc: datetime, +) -> bool: + if runtime_state.cooldown_until is None: + return False + cooldown_until = runtime_state.cooldown_until + updated = False + if cooldown_until.tzinfo is None: + cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) + runtime_state.cooldown_until = cooldown_until + updated = True + if now_utc < cooldown_until: + return updated + logger.info( + "scholar_search.cooldown_expired", + extra={"event": "scholar_search.cooldown_expired", "cooldown_until_utc": cooldown_until.isoformat()}, + ) + runtime_state.cooldown_until = None + runtime_state.cooldown_rejection_count = 0 + runtime_state.cooldown_alert_emitted = False + return True + + +def _cooldown_warning_codes( + *, + runtime_state: AuthorSearchRuntimeState, + cooldown_remaining_seconds: int, +) -> list[str]: + warning_codes = [ + "author_search_cooldown_active", + f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s", + ] + if bool(runtime_state.cooldown_alert_emitted): + warning_codes.append("author_search_cooldown_alert_threshold_exceeded") + return warning_codes + + +def _emit_cooldown_threshold_alert( + *, + runtime_state: AuthorSearchRuntimeState, + normalized_query: str, + cooldown_rejection_alert_threshold: int, +) -> bool: + runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1 + threshold = max(1, int(cooldown_rejection_alert_threshold)) + if int(runtime_state.cooldown_rejection_count) < threshold: + return True + if bool(runtime_state.cooldown_alert_emitted): + return True + logger.error( + "scholar_search.cooldown_rejection_threshold_exceeded", + extra={ + "event": "scholar_search.cooldown_rejection_threshold_exceeded", + "query": normalized_query, + "cooldown_rejection_count": int(runtime_state.cooldown_rejection_count), + "threshold": threshold, + "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, + }, + ) + runtime_state.cooldown_alert_emitted = True + return True + + +def _cooldown_block_result( + *, + runtime_state: AuthorSearchRuntimeState, + normalized_query: str, + bounded_limit: int, + cooldown_rejection_alert_threshold: int, + cooldown_remaining_seconds: int, +) -> ParsedAuthorSearchPage: + _emit_cooldown_threshold_alert( + runtime_state=runtime_state, + normalized_query=normalized_query, + cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, + ) + logger.warning( + "scholar_search.cooldown_active", + extra={ + "event": "scholar_search.cooldown_active", + "query": normalized_query, + "cooldown_remaining_seconds": cooldown_remaining_seconds, + "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, + }, + ) + return _policy_blocked_author_search_result( + reason=SEARCH_COOLDOWN_REASON, + warning_codes=_cooldown_warning_codes( + runtime_state=runtime_state, + cooldown_remaining_seconds=cooldown_remaining_seconds, + ), + limit=bounded_limit, + ) + + +async def _cache_hit_result( + db_session: AsyncSession, + *, + query_key: str, + now_utc: datetime, + normalized_query: str, + bounded_limit: int, +) -> ParsedAuthorSearchPage | None: + cached = await _cache_get_author_search_result( + db_session, + query_key=query_key, + now_utc=now_utc, + ) + if cached is None: + return None + logger.info( + "scholar_search.cache_hit", + extra={ + "event": "scholar_search.cache_hit", + "query": normalized_query, + "state": cached.state.value, + "state_reason": cached.state_reason, + }, + ) + state_reason_override = SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None + return _trim_author_search_result( + cached, + limit=bounded_limit, + extra_warnings=["author_search_served_from_cache"], + state_reason_override=state_reason_override, + ) + + +def _throttle_sleep_seconds( + *, + runtime_state: AuthorSearchRuntimeState, + now_utc: datetime, + min_interval_seconds: float, + interval_jitter_seconds: float, +) -> tuple[float, bool]: + updated = False + if runtime_state.last_live_request_at is None: + enforced_wait_seconds = 0.0 + else: + last_live_request_at = runtime_state.last_live_request_at + if last_live_request_at.tzinfo is None: + last_live_request_at = last_live_request_at.replace(tzinfo=timezone.utc) + runtime_state.last_live_request_at = last_live_request_at + updated = True + enforced_wait_seconds = ( + last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc + ).total_seconds() + jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0)) + return max(0.0, float(enforced_wait_seconds)) + jitter_seconds, updated + + +async def _wait_for_author_search_throttle( + *, + runtime_state: AuthorSearchRuntimeState, + normalized_query: str, + now_utc: datetime, + min_interval_seconds: float, + interval_jitter_seconds: float, +) -> bool: + sleep_seconds, updated = _throttle_sleep_seconds( + runtime_state=runtime_state, + now_utc=now_utc, + min_interval_seconds=min_interval_seconds, + interval_jitter_seconds=interval_jitter_seconds, + ) + if sleep_seconds <= 0.0: + return updated + logger.info( + "scholar_search.throttle_wait", + extra={"event": "scholar_search.throttle_wait", "query": normalized_query, "sleep_seconds": round(sleep_seconds, 3)}, + ) + await asyncio.sleep(sleep_seconds) + return True + + +async def _fetch_author_search_with_retries( + *, + source: ScholarSource, + normalized_query: str, + network_error_retries: int, + retry_backoff_seconds: float, +) -> tuple[ParsedAuthorSearchPage, int, list[str]]: + max_attempts = max(1, int(network_error_retries) + 1) + parsed: ParsedAuthorSearchPage | None = None + retry_warnings: list[str] = [] + retry_scheduled_count = 0 + for attempt_index in range(max_attempts): + fetch_result = await source.fetch_author_search_html(normalized_query, start=0) + try: + parsed = parse_author_search_page(fetch_result) + except ScholarParserError as exc: + parsed = ParsedAuthorSearchPage( + state=ParseState.LAYOUT_CHANGED, + state_reason=exc.code, + candidates=[], + marker_counts={}, + warnings=[exc.code], + ) + if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1: + break + retry_warnings.append("network_retry_scheduled_for_author_search") + retry_scheduled_count += 1 + retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index) + if retry_sleep_seconds > 0: + await asyncio.sleep(retry_sleep_seconds) + if parsed is None: + raise ScholarServiceError("Unable to complete scholar author search.") + return parsed, retry_scheduled_count, retry_warnings + + +def _with_retry_warnings( + parsed: ParsedAuthorSearchPage, + *, + retry_warnings: list[str], + retry_scheduled_count: int, + retry_alert_threshold: int, + normalized_query: str, +) -> ParsedAuthorSearchPage: + merged = replace(parsed, warnings=_merge_warnings(parsed.warnings, retry_warnings)) + threshold = max(1, int(retry_alert_threshold)) + if retry_scheduled_count < threshold: + return merged + logger.warning( + "scholar_search.retry_threshold_exceeded", + extra={ + "event": "scholar_search.retry_threshold_exceeded", + "query": normalized_query, + "retry_scheduled_count": retry_scheduled_count, + "threshold": threshold, + "final_state": merged.state.value, + "final_state_reason": merged.state_reason, + }, + ) + return replace( + merged, + warnings=_merge_warnings( + merged.warnings, + [f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"], + ), + ) + + +def _apply_block_circuit_breaker( + *, + runtime_state: AuthorSearchRuntimeState, + merged_parsed: ParsedAuthorSearchPage, + cooldown_block_threshold: int, + cooldown_seconds: int, + normalized_query: str, +) -> ParsedAuthorSearchPage: + if not _is_author_search_block_state(merged_parsed): + runtime_state.consecutive_blocked_count = 0 + return merged_parsed + runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1 + logger.warning( + "scholar_search.block_detected", + extra={ + "event": "scholar_search.block_detected", + "query": normalized_query, + "state_reason": merged_parsed.state_reason, + "consecutive_blocked_count": int(runtime_state.consecutive_blocked_count), + }, + ) + if int(runtime_state.consecutive_blocked_count) < max(1, int(cooldown_block_threshold)): + return merged_parsed + runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(seconds=max(60, int(cooldown_seconds))) + runtime_state.consecutive_blocked_count = 0 + runtime_state.cooldown_rejection_count = 0 + runtime_state.cooldown_alert_emitted = False + logger.error( + "scholar_search.cooldown_activated", + extra={ + "event": "scholar_search.cooldown_activated", + "query": normalized_query, + "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, + }, + ) + return replace( + merged_parsed, + warnings=_merge_warnings(merged_parsed.warnings, ["author_search_circuit_breaker_armed"]), + ) + + +def _resolve_author_search_cache_ttl_seconds( + *, + merged_parsed: ParsedAuthorSearchPage, + blocked_cache_ttl_seconds: int, + cache_ttl_seconds: int, +) -> int: + if _is_author_search_block_state(merged_parsed): + return min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds))) + return max(1, int(cache_ttl_seconds)) + + +async def _load_locked_runtime_state( + db_session: AsyncSession, +) -> AuthorSearchRuntimeState: + await _acquire_author_search_lock(db_session) + return await _load_runtime_state_for_update(db_session) + + +async def _cooldown_or_cache_result( + db_session: AsyncSession, + *, + runtime_state: AuthorSearchRuntimeState, + query_key: str, + normalized_query: str, + bounded_limit: int, + cooldown_rejection_alert_threshold: int, +) -> tuple[ParsedAuthorSearchPage | None, bool]: + runtime_state_updated = _normalize_runtime_cooldown_state( + runtime_state, + now_utc=datetime.now(timezone.utc), + ) + cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds( + runtime_state, + datetime.now(timezone.utc), + ) + if cooldown_remaining_seconds > 0: + return ( + _cooldown_block_result( + runtime_state=runtime_state, + normalized_query=normalized_query, + bounded_limit=bounded_limit, + cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, + cooldown_remaining_seconds=cooldown_remaining_seconds, + ), + True, + ) + cached_result = await _cache_hit_result( + db_session, + query_key=query_key, + now_utc=datetime.now(timezone.utc), + normalized_query=normalized_query, + bounded_limit=bounded_limit, + ) + return cached_result, runtime_state_updated + + +async def _perform_live_author_search(db_session: AsyncSession, *, source: ScholarSource, runtime_state: AuthorSearchRuntimeState, normalized_query: str, query_key: str, network_error_retries: int, retry_backoff_seconds: float, min_interval_seconds: float, interval_jitter_seconds: float, retry_alert_threshold: int, cooldown_block_threshold: int, cooldown_seconds: int, blocked_cache_ttl_seconds: int, cache_ttl_seconds: int, cache_max_entries: int) -> tuple[ParsedAuthorSearchPage, bool]: + runtime_state_updated = await _wait_for_author_search_throttle( + runtime_state=runtime_state, + normalized_query=normalized_query, + now_utc=datetime.now(timezone.utc), + min_interval_seconds=min_interval_seconds, + interval_jitter_seconds=interval_jitter_seconds, + ) + parsed, retry_count, retry_warnings = await _fetch_author_search_with_retries( + source=source, + normalized_query=normalized_query, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + runtime_state.last_live_request_at = datetime.now(timezone.utc) + merged = _with_retry_warnings( + parsed, + retry_warnings=retry_warnings, + retry_scheduled_count=retry_count, + retry_alert_threshold=retry_alert_threshold, + normalized_query=normalized_query, + ) + merged = _apply_block_circuit_breaker( + runtime_state=runtime_state, + merged_parsed=merged, + cooldown_block_threshold=cooldown_block_threshold, + cooldown_seconds=cooldown_seconds, + normalized_query=normalized_query, + ) + ttl_seconds = _resolve_author_search_cache_ttl_seconds( + merged_parsed=merged, + blocked_cache_ttl_seconds=blocked_cache_ttl_seconds, + cache_ttl_seconds=cache_ttl_seconds, + ) + await _cache_set_author_search_result( + db_session, + query_key=query_key, + parsed=merged, + ttl_seconds=float(ttl_seconds), + max_entries=cache_max_entries, + now_utc=datetime.now(timezone.utc), + ) + return merged, True + + +async def search_author_candidates(*, source: ScholarSource, db_session: AsyncSession, query: str, limit: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, search_enabled: bool = True, cache_ttl_seconds: int = 21_600, blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD) -> ParsedAuthorSearchPage: + normalized_query, bounded_limit, query_key = _normalize_author_search_inputs(query, limit) + if not search_enabled: + return _disabled_search_result( + normalized_query=normalized_query, + bounded_limit=bounded_limit, + ) + + runtime_state = await _load_locked_runtime_state(db_session) + early_result, runtime_state_updated = await _cooldown_or_cache_result( + db_session, + runtime_state=runtime_state, + query_key=query_key, + normalized_query=normalized_query, + bounded_limit=bounded_limit, + cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, + ) + if early_result is not None: + await db_session.commit() + return early_result + + merged_parsed, live_runtime_updated = await _perform_live_author_search( + db_session, + source=source, + runtime_state=runtime_state, + normalized_query=normalized_query, + query_key=query_key, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + min_interval_seconds=min_interval_seconds, + interval_jitter_seconds=interval_jitter_seconds, + retry_alert_threshold=retry_alert_threshold, + cooldown_block_threshold=cooldown_block_threshold, + cooldown_seconds=cooldown_seconds, + blocked_cache_ttl_seconds=blocked_cache_ttl_seconds, + cache_ttl_seconds=cache_ttl_seconds, + cache_max_entries=cache_max_entries, + ) + runtime_state_updated = runtime_state_updated or live_runtime_updated + if runtime_state_updated: + await db_session.commit() + return _trim_author_search_result(merged_parsed, limit=bounded_limit) + + +async def hydrate_profile_metadata( + db_session: AsyncSession, + *, + profile: ScholarProfile, + source: ScholarSource, +) -> ScholarProfile: + fetch_result = await source.fetch_profile_html(profile.scholar_id) + try: + parsed_page = parse_profile_page(fetch_result) + except ScholarParserError: + return profile + + if parsed_page.profile_name and not (profile.display_name or "").strip(): + profile.display_name = parsed_page.profile_name + if parsed_page.profile_image_url and not profile.profile_image_url: + profile.profile_image_url = parsed_page.profile_image_url + + await db_session.commit() + await db_session.refresh(profile) + return profile + + +async def set_profile_image_override_url( + db_session: AsyncSession, + *, + profile: ScholarProfile, + image_url: str | None, + upload_dir: str, +) -> ScholarProfile: + upload_root = _ensure_upload_root(upload_dir, create=True) + _safe_remove_upload(upload_root, profile.profile_image_upload_path) + + profile.profile_image_upload_path = None + profile.profile_image_override_url = normalize_profile_image_url(image_url) + + await db_session.commit() + await db_session.refresh(profile) + return profile + + +async def clear_profile_image_customization( + db_session: AsyncSession, + *, + profile: ScholarProfile, + upload_dir: str, +) -> ScholarProfile: + upload_root = _ensure_upload_root(upload_dir, create=True) + _safe_remove_upload(upload_root, profile.profile_image_upload_path) + + profile.profile_image_upload_path = None + profile.profile_image_override_url = None + + await db_session.commit() + await db_session.refresh(profile) + return profile + + +async def set_profile_image_upload( + db_session: AsyncSession, + *, + profile: ScholarProfile, + content_type: str | None, + image_bytes: bytes, + upload_dir: str, + max_upload_bytes: int, +) -> ScholarProfile: + normalized_content_type = (content_type or "").strip().lower() + extension = ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES.get(normalized_content_type) + if extension is None: + raise ScholarServiceError( + "Unsupported image type. Use JPEG, PNG, WEBP, or GIF." + ) + + if not image_bytes: + raise ScholarServiceError("Uploaded image file is empty.") + + if len(image_bytes) > max_upload_bytes: + raise ScholarServiceError( + f"Uploaded image exceeds {max_upload_bytes} bytes." + ) + + upload_root = _ensure_upload_root(upload_dir, create=True) + user_dir = upload_root / str(profile.user_id) + user_dir.mkdir(parents=True, exist_ok=True) + + filename = f"{profile.id}_{uuid4().hex}{extension}" + relative_path = os.path.join(str(profile.user_id), filename) + absolute_path = _resolve_upload_path(upload_root, relative_path) + absolute_path.write_bytes(image_bytes) + + old_path = profile.profile_image_upload_path + profile.profile_image_upload_path = relative_path + profile.profile_image_override_url = None + + await db_session.commit() + await db_session.refresh(profile) + + if old_path and old_path != relative_path: + _safe_remove_upload(upload_root, old_path) + + return profile diff --git a/build/lib/app/services/domains/scholars/constants.py b/build/lib/app/services/domains/scholars/constants.py new file mode 100644 index 0000000..7ee18ad --- /dev/null +++ b/build/lib/app/services/domains/scholars/constants.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import re + +SCHOLAR_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{12}$") +MAX_IMAGE_URL_LENGTH = 2048 +MAX_AUTHOR_SEARCH_LIMIT = 25 + +DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES = 512 +DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS = 300 +DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD = 1 +DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS = 1800 +DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS = 3.0 +DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0 +DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD = 2 +DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD = 3 + +AUTHOR_SEARCH_RUNTIME_STATE_KEY = "global" +AUTHOR_SEARCH_LOCK_NAMESPACE = 3901 +AUTHOR_SEARCH_LOCK_KEY = 1 + +ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = { + "image/jpeg": ".jpg", + "image/png": ".png", + "image/webp": ".webp", + "image/gif": ".gif", +} + +SEARCH_DISABLED_REASON = "search_disabled_by_configuration" +SEARCH_COOLDOWN_REASON = "search_temporarily_disabled_due_to_repeated_blocks" +SEARCH_CACHED_BLOCK_REASON = "search_temporarily_disabled_from_cached_blocked_response" + +STATE_REASON_HINTS: dict[str, str] = { + SEARCH_DISABLED_REASON: ( + "Scholar name search is currently disabled by service policy. " + "Add scholars by profile URL or Scholar ID." + ), + SEARCH_COOLDOWN_REASON: ( + "Scholar name search is temporarily paused after repeated block responses. " + "Use Scholar URL/ID adds until cooldown expires." + ), + SEARCH_CACHED_BLOCK_REASON: ( + "A recent blocked response was cached to reduce traffic. " + "Retry later or add by Scholar URL/ID." + ), + "network_dns_resolution_failed": ( + "DNS resolution failed while reaching scholar.google.com. " + "Verify container DNS/network and retry." + ), + "network_timeout": ( + "Request timed out before Google Scholar responded. " + "Increase delay/backoff and retry." + ), + "network_tls_error": ( + "TLS handshake/certificate validation failed. " + "Verify outbound TLS/network configuration." + ), + "blocked_http_429_rate_limited": ( + "Google Scholar rate-limited the request. " + "Slow request cadence and retry later." + ), + "blocked_unusual_traffic_detected": ( + "Google Scholar flagged traffic as unusual. " + "Increase delay/jitter and reduce concurrent scraping." + ), + "blocked_accounts_redirect": ( + "Request was redirected to Google Account sign-in. " + "Treat as access block and retry later." + ), +} diff --git a/build/lib/app/services/domains/scholars/exceptions.py b/build/lib/app/services/domains/scholars/exceptions.py new file mode 100644 index 0000000..f1383e3 --- /dev/null +++ b/build/lib/app/services/domains/scholars/exceptions.py @@ -0,0 +1,5 @@ +from __future__ import annotations + + +class ScholarServiceError(ValueError): + """Raised for expected scholar-management validation failures.""" diff --git a/build/lib/app/services/domains/scholars/search_hints.py b/build/lib/app/services/domains/scholars/search_hints.py new file mode 100644 index 0000000..f905c14 --- /dev/null +++ b/build/lib/app/services/domains/scholars/search_hints.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from app.db.models import ScholarProfile +from app.services.domains.scholar.parser import ParseState, ParsedAuthorSearchPage +from app.services.domains.scholars.constants import ( + MAX_AUTHOR_SEARCH_LIMIT, + STATE_REASON_HINTS, +) + + +def resolve_profile_image( + profile: ScholarProfile, + *, + uploaded_image_url: str | None, +) -> tuple[str | None, str]: + if profile.profile_image_upload_path and uploaded_image_url: + return uploaded_image_url, "upload" + if profile.profile_image_override_url: + return profile.profile_image_override_url, "override" + if profile.profile_image_url: + return profile.profile_image_url, "scraped" + return None, "none" + + +def scrape_state_hint(*, state: ParseState, state_reason: str) -> str | None: + if state not in {ParseState.NETWORK_ERROR, ParseState.BLOCKED_OR_CAPTCHA}: + return None + return STATE_REASON_HINTS.get(state_reason) + + +def _merge_warnings(base: list[str], extra: list[str]) -> list[str]: + if not extra: + return sorted(set(base)) + return sorted(set(base + extra)) + + +def _trim_author_search_result( + parsed: ParsedAuthorSearchPage, + *, + limit: int, + extra_warnings: list[str] | None = None, + state_reason_override: str | None = None, +) -> ParsedAuthorSearchPage: + bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT)) + return ParsedAuthorSearchPage( + state=parsed.state, + state_reason=state_reason_override or parsed.state_reason, + candidates=parsed.candidates[:bounded_limit], + marker_counts=parsed.marker_counts, + warnings=_merge_warnings(parsed.warnings, extra_warnings or []), + ) + + +def _policy_blocked_author_search_result( + *, + reason: str, + warning_codes: list[str], + limit: int, +) -> ParsedAuthorSearchPage: + _ = limit + return ParsedAuthorSearchPage( + state=ParseState.BLOCKED_OR_CAPTCHA, + state_reason=reason, + candidates=[], + marker_counts={}, + warnings=_merge_warnings([], warning_codes), + ) diff --git a/build/lib/app/services/domains/scholars/uploads.py b/build/lib/app/services/domains/scholars/uploads.py new file mode 100644 index 0000000..c33fd47 --- /dev/null +++ b/build/lib/app/services/domains/scholars/uploads.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from pathlib import Path + +from app.services.domains.scholars.exceptions import ScholarServiceError + + +def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path: + root = Path(upload_dir).expanduser().resolve() + if create: + root.mkdir(parents=True, exist_ok=True) + return root + + +def _resolve_upload_path(upload_root: Path, relative_path: str) -> Path: + candidate = (upload_root / relative_path).resolve() + if upload_root != candidate and upload_root not in candidate.parents: + raise ScholarServiceError("Invalid scholar image path.") + return candidate + + +def _safe_remove_upload(upload_root: Path, relative_path: str | None) -> None: + if not relative_path: + return + try: + file_path = _resolve_upload_path(upload_root, relative_path) + except ScholarServiceError: + return + + try: + if file_path.exists() and file_path.is_file(): + file_path.unlink() + except OSError: + return + + +def resolve_upload_file_path(*, upload_dir: str, relative_path: str) -> Path: + root = _ensure_upload_root(upload_dir, create=False) + return _resolve_upload_path(root, relative_path) diff --git a/build/lib/app/services/domains/scholars/validators.py b/build/lib/app/services/domains/scholars/validators.py new file mode 100644 index 0000000..1c79baa --- /dev/null +++ b/build/lib/app/services/domains/scholars/validators.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from urllib.parse import urlparse + +from app.services.domains.scholars.constants import MAX_IMAGE_URL_LENGTH, SCHOLAR_ID_PATTERN +from app.services.domains.scholars.exceptions import ScholarServiceError + + +def validate_scholar_id(value: str) -> str: + scholar_id = value.strip() + if not SCHOLAR_ID_PATTERN.fullmatch(scholar_id): + raise ScholarServiceError("Scholar ID must match [a-zA-Z0-9_-]{12}.") + return scholar_id + + +def normalize_display_name(value: str) -> str | None: + normalized = value.strip() + return normalized if normalized else None + + +def normalize_profile_image_url(value: str | None) -> str | None: + if value is None: + return None + + candidate = value.strip() + if not candidate: + return None + + if len(candidate) > MAX_IMAGE_URL_LENGTH: + raise ScholarServiceError( + f"Image URL must be {MAX_IMAGE_URL_LENGTH} characters or fewer." + ) + + parsed = urlparse(candidate) + if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc: + raise ScholarServiceError("Image URL must be an absolute http(s) URL.") + + return candidate diff --git a/build/lib/app/services/domains/settings/__init__.py b/build/lib/app/services/domains/settings/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/build/lib/app/services/domains/settings/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/build/lib/app/services/domains/settings/application.py b/build/lib/app/services/domains/settings/application.py new file mode 100644 index 0000000..7f435e8 --- /dev/null +++ b/build/lib/app/services/domains/settings/application.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import UserSetting + + +class UserSettingsServiceError(ValueError): + """Raised for expected settings-validation failures.""" + + +NAV_PAGE_DASHBOARD = "dashboard" +NAV_PAGE_SCHOLARS = "scholars" +NAV_PAGE_PUBLICATIONS = "publications" +NAV_PAGE_SETTINGS = "settings" +NAV_PAGE_STYLE_GUIDE = "style-guide" +NAV_PAGE_RUNS = "runs" +NAV_PAGE_USERS = "users" + +ALLOWED_NAV_PAGES = ( + NAV_PAGE_DASHBOARD, + NAV_PAGE_SCHOLARS, + NAV_PAGE_PUBLICATIONS, + NAV_PAGE_SETTINGS, + NAV_PAGE_STYLE_GUIDE, + NAV_PAGE_RUNS, + NAV_PAGE_USERS, +) +REQUIRED_NAV_PAGES = ( + NAV_PAGE_DASHBOARD, + NAV_PAGE_SCHOLARS, + NAV_PAGE_SETTINGS, +) +DEFAULT_NAV_VISIBLE_PAGES = list(ALLOWED_NAV_PAGES) +HARD_MIN_RUN_INTERVAL_MINUTES = 15 +HARD_MIN_REQUEST_DELAY_SECONDS = 2 + + +def resolve_run_interval_minimum(configured_minimum: int | None) -> int: + try: + parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_RUN_INTERVAL_MINUTES + except (TypeError, ValueError): + parsed = HARD_MIN_RUN_INTERVAL_MINUTES + return max(HARD_MIN_RUN_INTERVAL_MINUTES, parsed) + + +def resolve_request_delay_minimum(configured_minimum: int | None) -> int: + try: + parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_REQUEST_DELAY_SECONDS + except (TypeError, ValueError): + parsed = HARD_MIN_REQUEST_DELAY_SECONDS + return max(HARD_MIN_REQUEST_DELAY_SECONDS, parsed) + + +def parse_run_interval_minutes(value: str, *, minimum: int = HARD_MIN_RUN_INTERVAL_MINUTES) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise UserSettingsServiceError("Check interval must be a whole number.") from exc + effective_minimum = resolve_run_interval_minimum(minimum) + if parsed < effective_minimum: + raise UserSettingsServiceError( + f"Check interval must be at least {effective_minimum} minutes." + ) + return parsed + + +def parse_request_delay_seconds(value: str, *, minimum: int = HARD_MIN_REQUEST_DELAY_SECONDS) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise UserSettingsServiceError("Request delay must be a whole number.") from exc + effective_minimum = resolve_request_delay_minimum(minimum) + if parsed < effective_minimum: + raise UserSettingsServiceError( + f"Request delay must be at least {effective_minimum} seconds." + ) + return parsed + + +def parse_nav_visible_pages(value: object) -> list[str]: + if not isinstance(value, list): + raise UserSettingsServiceError("Navigation visibility must be a list of page ids.") + + deduped: list[str] = [] + seen: set[str] = set() + + for raw_page in value: + if not isinstance(raw_page, str): + raise UserSettingsServiceError("Navigation visibility entries must be strings.") + + page_id = raw_page.strip() + if page_id not in ALLOWED_NAV_PAGES: + raise UserSettingsServiceError(f"Unsupported navigation page id: {page_id}") + + if page_id in seen: + continue + + seen.add(page_id) + deduped.append(page_id) + + missing_required = [page for page in REQUIRED_NAV_PAGES if page not in seen] + if missing_required: + raise UserSettingsServiceError( + "Dashboard, Scholars, and Settings must remain visible." + ) + + return deduped + + +from app.settings import settings as app_settings + +async def get_or_create_settings( + db_session: AsyncSession, + *, + user_id: int, +) -> UserSetting: + result = await db_session.execute( + select(UserSetting).where(UserSetting.user_id == user_id) + ) + settings = result.scalar_one_or_none() + if settings is not None: + return settings + + settings = UserSetting( + user_id=user_id, + openalex_api_key=app_settings.openalex_api_key, + crossref_api_token=app_settings.crossref_api_token, + crossref_api_mailto=app_settings.crossref_api_mailto, + ) + db_session.add(settings) + await db_session.commit() + await db_session.refresh(settings) + return settings + + +async def update_settings( + db_session: AsyncSession, + *, + settings: UserSetting, + auto_run_enabled: bool, + run_interval_minutes: int, + request_delay_seconds: int, + nav_visible_pages: list[str], + openalex_api_key: str | None, + crossref_api_token: str | None, + crossref_api_mailto: str | None, +) -> UserSetting: + settings.auto_run_enabled = auto_run_enabled + settings.run_interval_minutes = run_interval_minutes + settings.request_delay_seconds = request_delay_seconds + settings.nav_visible_pages = nav_visible_pages + settings.openalex_api_key = openalex_api_key + settings.crossref_api_token = crossref_api_token + settings.crossref_api_mailto = crossref_api_mailto + await db_session.commit() + await db_session.refresh(settings) + return settings diff --git a/build/lib/app/services/domains/unpaywall/__init__.py b/build/lib/app/services/domains/unpaywall/__init__.py new file mode 100644 index 0000000..a1baa12 --- /dev/null +++ b/build/lib/app/services/domains/unpaywall/__init__.py @@ -0,0 +1,10 @@ +from __future__ import annotations + + +async def resolve_publication_pdf_urls(*args, **kwargs): + from app.services.domains.unpaywall.application import resolve_publication_pdf_urls as _impl + + return await _impl(*args, **kwargs) + + +__all__ = ["resolve_publication_pdf_urls"] diff --git a/build/lib/app/services/domains/unpaywall/application.py b/build/lib/app/services/domains/unpaywall/application.py new file mode 100644 index 0000000..13104e9 --- /dev/null +++ b/build/lib/app/services/domains/unpaywall/application.py @@ -0,0 +1,458 @@ +from __future__ import annotations + +from dataclasses import dataclass +import logging +import re +from typing import TYPE_CHECKING +from urllib.parse import unquote + +from app.services.domains.crossref.application import discover_doi_for_publication +from app.services.domains.doi.normalize import normalize_doi +from app.services.domains.unpaywall.pdf_discovery import ( + looks_like_pdf_url, + resolve_pdf_from_landing_page, +) +from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot +from app.settings import settings + +if TYPE_CHECKING: + from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem + +DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I) +DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I) +DOI_URL_RE = re.compile(r"(?:https?://)?(?:dx\.)?doi\.org/(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I) +UNPAYWALL_URL_TEMPLATE = "https://api.unpaywall.org/v2/{doi}" +FAILURE_MISSING_DOI = "missing_doi" +FAILURE_NO_RECORD = "no_unpaywall_record" +FAILURE_NO_PDF = "no_pdf_found" +FAILURE_RESOLUTION_EXCEPTION = "resolution_exception" +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class OaResolutionOutcome: + publication_id: int + doi: str | None + pdf_url: str | None + failure_reason: str | None + source: str | None + used_crossref: bool + + +def _extract_doi_candidate(text: str | None) -> str | None: + if not text: + return None + decoded = unquote(text) + match = DOI_PATTERN.search(decoded) + if not match: + return None + return match.group(0).rstrip(" .;,)") + + +def _extract_explicit_doi(text: str | None) -> str | None: + if not text: + return None + decoded = unquote(text) + url_match = DOI_URL_RE.search(decoded) + if url_match: + return normalize_doi(url_match.group(1)) + prefix_match = DOI_PREFIX_RE.search(decoded) + if prefix_match: + return normalize_doi(prefix_match.group(1)) + return None + + +def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str | None: + stored = None + if getattr(item, "display_identifier", None) and item.display_identifier.kind == "doi": + stored = normalize_doi(item.display_identifier.value) + + explicit_doi = ( + _extract_explicit_doi(item.pub_url) + or _extract_explicit_doi(item.venue_text) + ) + if explicit_doi: + return explicit_doi + pub_url_doi = _extract_doi_candidate(item.pub_url) + if pub_url_doi: + return normalize_doi(pub_url_doi) + return stored + + +def _payload_locations(payload: dict) -> list[dict]: + locations: list[dict] = [] + best = payload.get("best_oa_location") + if isinstance(best, dict): + locations.append(best) + oa_locations = payload.get("oa_locations") + if isinstance(oa_locations, list): + locations.extend(location for location in oa_locations if isinstance(location, dict)) + return locations + + +def _location_value(location: dict, key: str) -> str | None: + value = location.get(key) + if not isinstance(value, str): + return None + normalized = value.strip() + return normalized or None + + +def _payload_pdf_candidates(payload: dict) -> list[str]: + candidates: list[str] = [] + seen: set[str] = set() + for location in _payload_locations(payload): + candidate = _location_value(location, "url_for_pdf") + if candidate is None or candidate in seen: + continue + seen.add(candidate) + candidates.append(candidate) + return candidates + + +def _payload_landing_candidates(payload: dict) -> list[str]: + candidates: list[str] = [] + seen: set[str] = set() + for location in _payload_locations(payload): + candidate = _location_value(location, "url") + if candidate is None or candidate in seen: + continue + seen.add(candidate) + candidates.append(candidate) + return candidates + + +def _crawl_targets( + *, + payload: dict, + pdf_candidates: list[str], +) -> list[str]: + targets = _payload_landing_candidates(payload) + seen = set(targets) + for candidate in pdf_candidates: + if candidate in seen: + continue + targets.append(candidate) + seen.add(candidate) + doi = normalize_doi(str(payload.get("doi") or "")) + doi_landing_url = f"https://doi.org/{doi}" if doi else None + if doi_landing_url and doi_landing_url not in seen: + targets.append(doi_landing_url) + return targets + + +def _has_direct_payload_pdf(payload: dict) -> bool: + return any(looks_like_pdf_url(candidate) for candidate in _payload_pdf_candidates(payload)) + + +async def _resolved_pdf_url_from_payload( + payload: dict, + *, + client, +) -> str | None: + pdf_candidates = _payload_pdf_candidates(payload) + for candidate in pdf_candidates: + if looks_like_pdf_url(candidate): + return candidate + for page_url in _crawl_targets(payload=payload, pdf_candidates=pdf_candidates)[:3]: + discovered = await resolve_pdf_from_landing_page(client, page_url=page_url) + if discovered: + logger.info( + "unpaywall.pdf_discovered_from_landing", + extra={"event": "unpaywall.pdf_discovered_from_landing", "landing_url": page_url}, + ) + return discovered + return None + + +async def _fetch_unpaywall_payload_by_doi( + *, + client, + doi: str, + email: str, +) -> dict | None: + await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds) + headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{email})"} + response = await client.get( + UNPAYWALL_URL_TEMPLATE.format(doi=doi), + params={"email": email}, + headers=headers, + ) + if response.status_code != 200: + return None + payload = response.json() + if not isinstance(payload, dict): + return None + return payload + + +def _email_for_request(request_email: str | None) -> str | None: + email = (request_email or "").strip() or settings.unpaywall_email.strip() + return email or None + + +def _log_resolution_summary( + *, + publication_count: int, + doi_input_count: int, + search_attempt_count: int, + resolved_pdf_count: int, + email: str, +) -> None: + logger.info( + "unpaywall.resolve_completed", + extra={ + "event": "unpaywall.resolve_completed", + "publication_count": publication_count, + "doi_input_count": doi_input_count, + "search_attempt_count": search_attempt_count, + "resolved_pdf_count": resolved_pdf_count, + "email_domain": email.split("@", 1)[-1] if "@" in email else None, + }, + ) + + +async def _resolve_item_payload( + *, + client, + item: PublicationListItem, + email: str, + allow_crossref: bool, +) -> tuple[dict | None, bool, str | None]: + doi = _publication_doi(item) + payload: dict | None = None + if doi: + payload = await _fetch_unpaywall_payload_by_doi(client=client, doi=doi, email=email) + if payload is not None and _has_direct_payload_pdf(payload): + return payload, False, doi + if not allow_crossref or not settings.crossref_enabled: + return payload, False, doi + crossref_doi = await discover_doi_for_publication( + item=item, + max_rows=settings.crossref_max_rows, + email=email, + ) + if crossref_doi is None or crossref_doi == doi: + return payload, crossref_doi is not None, doi or crossref_doi + crossref_payload = await _fetch_unpaywall_payload_by_doi( + client=client, + doi=crossref_doi, + email=email, + ) + if crossref_payload is not None: + return crossref_payload, True, crossref_doi + return payload, True, crossref_doi + + +async def _doi_and_pdf_from_payload( + payload: dict, + *, + client, +) -> tuple[str | None, str | None]: + doi = normalize_doi(str(payload.get("doi") or "")) + return doi, await _resolved_pdf_url_from_payload(payload, client=client) + + +def _resolution_targets(items: list[PublicationListItem]) -> list[PublicationListItem]: + return [item for item in items if not item.pdf_url] + + +def _crossref_budget_value() -> int: + return max(int(settings.crossref_max_lookups_per_request), 0) + + +def _outcome_with_failure( + *, + item: PublicationListItem, + failure_reason: str, + used_crossref: bool, + doi_override: str | None = None, +) -> OaResolutionOutcome: + return OaResolutionOutcome( + publication_id=item.publication_id, + doi=normalize_doi(doi_override) if doi_override is not None else _publication_doi(item), + pdf_url=None, + failure_reason=failure_reason, + source=None, + used_crossref=used_crossref, + ) + + +def _missing_payload_failure_reason(item: PublicationListItem, *, used_crossref: bool) -> str: + if _publication_doi(item): + return FAILURE_NO_RECORD + if used_crossref: + return FAILURE_NO_RECORD + return FAILURE_MISSING_DOI + + +def _source_name(*, used_crossref: bool) -> str: + return "crossref+unpaywall" if used_crossref else "unpaywall" + + +def _outcome_from_payload( + *, + item: PublicationListItem, + doi: str | None, + pdf_url: str | None, + used_crossref: bool, +) -> OaResolutionOutcome: + return OaResolutionOutcome( + publication_id=item.publication_id, + doi=doi, + pdf_url=pdf_url, + failure_reason=None if pdf_url else FAILURE_NO_PDF, + source=_source_name(used_crossref=used_crossref), + used_crossref=used_crossref, + ) + + +def _resolved_pdf_count(outcomes: dict[int, OaResolutionOutcome]) -> int: + return sum(1 for outcome in outcomes.values() if outcome.pdf_url) + + +def _outcome_metadata(outcomes: dict[int, OaResolutionOutcome]) -> dict[int, tuple[str | None, str | None]]: + return { + publication_id: (outcome.doi, outcome.pdf_url) + for publication_id, outcome in outcomes.items() + } + + +async def _resolve_outcome_for_item( + *, + client, + item: PublicationListItem, + email: str, + allow_crossref: bool, +) -> OaResolutionOutcome: + payload, used_crossref, resolved_doi = await _resolve_item_payload( + client=client, + item=item, + email=email, + allow_crossref=allow_crossref, + ) + if not isinstance(payload, dict): + return _outcome_with_failure( + item=item, + failure_reason=_missing_payload_failure_reason(item, used_crossref=used_crossref), + used_crossref=used_crossref, + doi_override=resolved_doi, + ) + doi, pdf_url = await _doi_and_pdf_from_payload(payload, client=client) + return _outcome_from_payload( + item=item, + doi=doi, + pdf_url=pdf_url, + used_crossref=used_crossref, + ) + + +def _doi_input_count(items: list[PublicationListItem]) -> int: + return len([item for item in items if _publication_doi(item)]) + + +def _search_attempt_count(*, targets: list[PublicationListItem]) -> int: + return len([item for item in targets if not _publication_doi(item)]) + + +async def _safe_outcome_for_item( + *, + client, + item: PublicationListItem, + email: str, + allow_crossref: bool, +) -> OaResolutionOutcome: + try: + return await _resolve_outcome_for_item( + client=client, + item=item, + email=email, + allow_crossref=allow_crossref, + ) + except Exception as exc: # pragma: no cover - defensive network boundary + logger.warning( + "unpaywall.resolve_item_failed", + extra={ + "event": "unpaywall.resolve_item_failed", + "publication_id": item.publication_id, + "error": str(exc), + }, + ) + return _outcome_with_failure( + item=item, + failure_reason=FAILURE_RESOLUTION_EXCEPTION, + used_crossref=allow_crossref and settings.crossref_enabled, + ) + + +async def _resolve_outcomes_with_client( + *, + client, + targets: list[PublicationListItem], + email: str, +) -> dict[int, OaResolutionOutcome]: + outcomes: dict[int, OaResolutionOutcome] = {} + crossref_budget = _crossref_budget_value() + crossref_lookups = 0 + for item in targets: + allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget + outcome = await _safe_outcome_for_item( + client=client, + item=item, + email=email, + allow_crossref=allow_crossref, + ) + if outcome.used_crossref: + crossref_lookups += 1 + outcomes[item.publication_id] = outcome + return outcomes + + +async def resolve_publication_oa_metadata( + items: list[PublicationListItem], + *, + request_email: str | None = None, +) -> dict[int, tuple[str | None, str | None]]: + outcomes = await resolve_publication_oa_outcomes(items, request_email=request_email) + return _outcome_metadata(outcomes) + + +async def resolve_publication_oa_outcomes( + items: list[PublicationListItem], + *, + request_email: str | None = None, +) -> dict[int, OaResolutionOutcome]: + if not settings.unpaywall_enabled: + return {} + email = _email_for_request(request_email) + if email is None: + logger.debug("unpaywall.resolve_skipped_missing_email") + return {} + import httpx + + timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5) + targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)] + headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{email})"} + async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True, headers=headers) as client: + outcomes = await _resolve_outcomes_with_client( + client=client, + targets=targets, + email=email, + ) + _log_resolution_summary( + publication_count=len(items), + doi_input_count=_doi_input_count(items), + search_attempt_count=_search_attempt_count(targets=targets), + resolved_pdf_count=_resolved_pdf_count(outcomes), + email=email, + ) + return outcomes + + +async def resolve_publication_pdf_urls( + items: list[PublicationListItem], + *, + request_email: str | None = None, +) -> dict[int, str | None]: + resolved = await resolve_publication_oa_metadata(items, request_email=request_email) + return {publication_id: pdf for publication_id, (_doi, pdf) in resolved.items()} diff --git a/build/lib/app/services/domains/unpaywall/pdf_discovery.py b/build/lib/app/services/domains/unpaywall/pdf_discovery.py new file mode 100644 index 0000000..5f82a58 --- /dev/null +++ b/build/lib/app/services/domains/unpaywall/pdf_discovery.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from html.parser import HTMLParser +import re +from urllib.parse import urljoin, urlparse + +from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot +from app.settings import settings + +PDF_MIME = "application/pdf" +URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.I) + + +class _LandingPdfParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.base_href: str | None = None + self.candidates: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + attrs_map = {key.lower(): (value or "").strip() for key, value in attrs} + if tag == "base": + self.base_href = attrs_map.get("href") or self.base_href + return + if tag == "meta": + self._append_meta_candidate(attrs_map) + return + if tag == "link": + self._append_link_candidate(attrs_map) + return + if tag == "a": + self._append_anchor_candidate(attrs_map) + + def _append_meta_candidate(self, attrs_map: dict[str, str]) -> None: + meta_name = (attrs_map.get("name") or attrs_map.get("property") or "").lower() + if meta_name != "citation_pdf_url": + return + content = attrs_map.get("content") + if content: + self.candidates.append(content) + + def _append_link_candidate(self, attrs_map: dict[str, str]) -> None: + href = attrs_map.get("href") + link_type = (attrs_map.get("type") or "").lower() + if href and "pdf" in link_type: + self.candidates.append(href) + + def _append_anchor_candidate(self, attrs_map: dict[str, str]) -> None: + href = attrs_map.get("href") + if href: + self.candidates.append(href) + + +def looks_like_pdf_url(url: str | None) -> bool: + if not isinstance(url, str): + return False + value = url.strip() + if not value: + return False + parsed = urlparse(value) + path = (parsed.path or "").lower() + query = (parsed.query or "").lower() + return path.endswith(".pdf") or ".pdf" in query + + +def _normalized_candidate_urls(*, page_url: str, html: str) -> list[str]: + parser = _LandingPdfParser() + parser.feed(html) + parser.close() + base_url = urljoin(page_url, parser.base_href) if parser.base_href else page_url + raw_candidates = [*parser.candidates, *_text_url_candidates(html)] + deduped: list[str] = [] + seen: set[str] = set() + for raw in raw_candidates: + absolute = urljoin(base_url, raw.strip()) + parsed = urlparse(absolute) + if parsed.scheme not in {"http", "https"}: + continue + if absolute in seen: + continue + seen.add(absolute) + deduped.append(absolute) + return sorted(deduped, key=_candidate_sort_key) + + +def _text_url_candidates(html: str) -> list[str]: + candidates: list[str] = [] + for match in URL_RE.findall(html): + cleaned = match.rstrip(".,);]}>") + if "pdf" not in cleaned.lower(): + continue + candidates.append(cleaned) + return candidates + + +def _candidate_sort_key(candidate: str) -> tuple[int, str]: + lowered = candidate.lower() + if looks_like_pdf_url(candidate): + return (0, lowered) + if any(token in lowered for token in ("doi", "full", "article", "download")): + return (1, lowered) + return (2, lowered) + + +def _is_html_response(response) -> bool: + content_type = str(response.headers.get("content-type") or "").lower() + return "text/html" in content_type or "application/xhtml+xml" in content_type + + +async def _fetch_page_html(client, *, page_url: str) -> str | None: + await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds) + response = await client.get(page_url, follow_redirects=True) + if response.status_code != 200 or not _is_html_response(response): + return None + text = response.text or "" + return text[: max(int(settings.unpaywall_pdf_discovery_max_html_bytes), 0)] + + +async def _candidate_is_pdf(client, *, candidate_url: str) -> bool: + if looks_like_pdf_url(candidate_url): + return True + await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds) + response = await client.get(candidate_url, follow_redirects=True) + content_type = str(response.headers.get("content-type") or "").lower() + return response.status_code == 200 and PDF_MIME in content_type + + +def _candidate_limit() -> int: + return max(int(settings.unpaywall_pdf_discovery_max_candidates), 1) + + +async def _resolve_pdf_from_candidate_page(client, *, candidate_url: str) -> str | None: + html = await _fetch_page_html(client, page_url=candidate_url) + if not html: + return None + nested_candidates = _normalized_candidate_urls(page_url=candidate_url, html=html) + for nested in nested_candidates[: _candidate_limit()]: + if await _candidate_is_pdf(client, candidate_url=nested): + return nested + return None + + +async def resolve_pdf_from_landing_page(client, *, page_url: str) -> str | None: + if not settings.unpaywall_pdf_discovery_enabled: + return None + html = await _fetch_page_html(client, page_url=page_url) + if not html: + return None + candidates = _normalized_candidate_urls(page_url=page_url, html=html) + for candidate in candidates[: _candidate_limit()]: + if await _candidate_is_pdf(client, candidate_url=candidate): + return candidate + nested_pdf = await _resolve_pdf_from_candidate_page(client, candidate_url=candidate) + if nested_pdf: + return nested_pdf + return None diff --git a/build/lib/app/services/domains/unpaywall/rate_limit.py b/build/lib/app/services/domains/unpaywall/rate_limit.py new file mode 100644 index 0000000..76d8a3d --- /dev/null +++ b/build/lib/app/services/domains/unpaywall/rate_limit.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import asyncio +import time + +_REQUEST_LOCK = asyncio.Lock() +_LAST_REQUEST_AT = 0.0 + + +async def wait_for_unpaywall_slot(*, min_interval_seconds: float) -> None: + global _LAST_REQUEST_AT + interval = max(float(min_interval_seconds), 0.0) + async with _REQUEST_LOCK: + elapsed = time.monotonic() - _LAST_REQUEST_AT + remaining = interval - elapsed + if remaining > 0: + await asyncio.sleep(remaining) + _LAST_REQUEST_AT = time.monotonic() diff --git a/build/lib/app/services/domains/users/__init__.py b/build/lib/app/services/domains/users/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/build/lib/app/services/domains/users/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/build/lib/app/services/domains/users/application.py b/build/lib/app/services/domains/users/application.py new file mode 100644 index 0000000..7f019c4 --- /dev/null +++ b/build/lib/app/services/domains/users/application.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import re + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import User + +EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") + + +class UserServiceError(ValueError): + """Raised for expected user-management validation failures.""" + + +def normalize_email(value: str) -> str: + return value.strip().lower() + + +def validate_email(value: str) -> str: + email = normalize_email(value) + if not EMAIL_PATTERN.fullmatch(email): + raise UserServiceError("Enter a valid email address.") + return email + + +def validate_password(value: str) -> str: + password = value.strip() + if len(password) < 8: + raise UserServiceError("Password must be at least 8 characters.") + return password + + +async def get_user_by_id(db_session: AsyncSession, user_id: int) -> User | None: + result = await db_session.execute(select(User).where(User.id == user_id)) + return result.scalar_one_or_none() + + +async def get_user_by_email(db_session: AsyncSession, email: str) -> User | None: + result = await db_session.execute( + select(User).where(User.email == normalize_email(email)) + ) + return result.scalar_one_or_none() + + +async def list_users(db_session: AsyncSession) -> list[User]: + result = await db_session.execute(select(User).order_by(User.email.asc())) + return list(result.scalars().all()) + + +async def create_user( + db_session: AsyncSession, + *, + email: str, + password_hash: str, + is_admin: bool, +) -> User: + user = User( + email=validate_email(email), + password_hash=password_hash, + is_admin=is_admin, + is_active=True, + ) + db_session.add(user) + try: + await db_session.commit() + except IntegrityError as exc: + await db_session.rollback() + raise UserServiceError("A user with that email already exists.") from exc + await db_session.refresh(user) + return user + + +async def set_user_active( + db_session: AsyncSession, + *, + user: User, + is_active: bool, +) -> User: + user.is_active = is_active + await db_session.commit() + await db_session.refresh(user) + return user + + +async def set_user_password_hash( + db_session: AsyncSession, + *, + user: User, + password_hash: str, +) -> User: + user.password_hash = password_hash + await db_session.commit() + await db_session.refresh(user) + return user + diff --git a/build/lib/app/settings.py b/build/lib/app/settings.py new file mode 100644 index 0000000..f56ff51 --- /dev/null +++ b/build/lib/app/settings.py @@ -0,0 +1,267 @@ +from dataclasses import dataclass +import os + +DEFAULT_SECURITY_PERMISSIONS_POLICY = ( + "accelerometer=(), autoplay=(), camera=(), display-capture=(), " + "geolocation=(), gyroscope=(), microphone=(), payment=(), usb=()" +) +DEFAULT_SECURITY_CSP_POLICY = ( + "default-src 'self'; " + "base-uri 'self'; " + "form-action 'self'; " + "frame-ancestors 'none'; " + "img-src 'self' data: https:; " + "script-src 'self'; " + "style-src 'self' 'unsafe-inline'; " + "font-src 'self' data:; " + "connect-src 'self'; " + "object-src 'none'" +) +DEFAULT_SECURITY_CSP_DOCS_POLICY = ( + "default-src 'self'; " + "img-src 'self' data: https:; " + "script-src 'self' 'unsafe-inline'; " + "style-src 'self' 'unsafe-inline'; " + "font-src 'self' data:; " + "connect-src 'self' https:; " + "object-src 'none'; " + "frame-ancestors 'none'" +) + + +def _env_bool(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _env_int(name: str, default: int) -> int: + value = os.getenv(name) + if value is None: + return default + return int(value) + + +def _env_float(name: str, default: float) -> float: + value = os.getenv(name) + if value is None: + return default + return float(value) + + +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + if value is None: + return default + return value.strip() or default + + +@dataclass(frozen=True) +class Settings: + app_name: str = os.getenv("APP_NAME", "scholarr") + database_url: str = os.getenv( + "DATABASE_URL", + "postgresql+asyncpg://scholar:scholar@db:5432/scholar", + ) + database_pool_mode: str = _env_str("DATABASE_POOL_MODE", "auto") + database_pool_size: int = _env_int("DATABASE_POOL_SIZE", 5) + database_pool_max_overflow: int = _env_int("DATABASE_POOL_MAX_OVERFLOW", 10) + database_pool_timeout_seconds: int = _env_int("DATABASE_POOL_TIMEOUT_SECONDS", 30) + session_secret_key: str = os.getenv("SESSION_SECRET_KEY", "dev-insecure-session-key") + session_cookie_secure: bool = _env_bool("SESSION_COOKIE_SECURE", False) + security_headers_enabled: bool = _env_bool("SECURITY_HEADERS_ENABLED", True) + security_x_content_type_options: str = _env_str("SECURITY_X_CONTENT_TYPE_OPTIONS", "nosniff") + security_x_frame_options: str = _env_str("SECURITY_X_FRAME_OPTIONS", "DENY") + security_referrer_policy: str = _env_str("SECURITY_REFERRER_POLICY", "strict-origin-when-cross-origin") + security_permissions_policy: str = _env_str( + "SECURITY_PERMISSIONS_POLICY", + DEFAULT_SECURITY_PERMISSIONS_POLICY, + ) + security_cross_origin_opener_policy: str = _env_str( + "SECURITY_CROSS_ORIGIN_OPENER_POLICY", + "same-origin", + ) + security_cross_origin_resource_policy: str = _env_str( + "SECURITY_CROSS_ORIGIN_RESOURCE_POLICY", + "same-origin", + ) + security_csp_enabled: bool = _env_bool("SECURITY_CSP_ENABLED", True) + security_csp_policy: str = _env_str("SECURITY_CSP_POLICY", DEFAULT_SECURITY_CSP_POLICY) + security_csp_docs_policy: str = _env_str("SECURITY_CSP_DOCS_POLICY", DEFAULT_SECURITY_CSP_DOCS_POLICY) + security_csp_report_only: bool = _env_bool("SECURITY_CSP_REPORT_ONLY", False) + security_strict_transport_security_enabled: bool = _env_bool( + "SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED", + False, + ) + security_strict_transport_security_max_age: int = _env_int( + "SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE", + 31_536_000, + ) + security_strict_transport_security_include_subdomains: bool = _env_bool( + "SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS", + True, + ) + security_strict_transport_security_preload: bool = _env_bool( + "SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD", + False, + ) + login_rate_limit_attempts: int = _env_int("LOGIN_RATE_LIMIT_ATTEMPTS", 5) + login_rate_limit_window_seconds: int = _env_int( + "LOGIN_RATE_LIMIT_WINDOW_SECONDS", + 60, + ) + log_level: str = _env_str("LOG_LEVEL", "INFO") + log_format: str = _env_str("LOG_FORMAT", "console") + log_requests: bool = _env_bool("LOG_REQUESTS", True) + log_uvicorn_access: bool = _env_bool("LOG_UVICORN_ACCESS", False) + log_request_skip_paths: str = _env_str("LOG_REQUEST_SKIP_PATHS", "/healthz") + log_redact_fields: str = os.getenv("LOG_REDACT_FIELDS", "") + scheduler_enabled: bool = _env_bool("SCHEDULER_ENABLED", True) + scheduler_tick_seconds: int = _env_int("SCHEDULER_TICK_SECONDS", 60) + ingestion_automation_allowed: bool = _env_bool( + "INGESTION_AUTOMATION_ALLOWED", + True, + ) + ingestion_manual_run_allowed: bool = _env_bool( + "INGESTION_MANUAL_RUN_ALLOWED", + True, + ) + ingestion_min_run_interval_minutes: int = _env_int( + "INGESTION_MIN_RUN_INTERVAL_MINUTES", + 15, + ) + ingestion_min_request_delay_seconds: int = _env_int( + "INGESTION_MIN_REQUEST_DELAY_SECONDS", + 2, + ) + ingestion_network_error_retries: int = _env_int("INGESTION_NETWORK_ERROR_RETRIES", 1) + ingestion_retry_backoff_seconds: float = _env_float( + "INGESTION_RETRY_BACKOFF_SECONDS", + 1.0, + ) + ingestion_rate_limit_retries: int = _env_int("INGESTION_RATE_LIMIT_RETRIES", 3) + ingestion_rate_limit_backoff_seconds: float = _env_float( + "INGESTION_RATE_LIMIT_BACKOFF_SECONDS", + 30.0, + ) + ingestion_max_pages_per_scholar: int = _env_int( + "INGESTION_MAX_PAGES_PER_SCHOLAR", + 30, + ) + ingestion_page_size: int = _env_int("INGESTION_PAGE_SIZE", 100) + ingestion_alert_blocked_failure_threshold: int = _env_int( + "INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD", + 1, + ) + ingestion_alert_network_failure_threshold: int = _env_int( + "INGESTION_ALERT_NETWORK_FAILURE_THRESHOLD", + 2, + ) + ingestion_alert_retry_scheduled_threshold: int = _env_int( + "INGESTION_ALERT_RETRY_SCHEDULED_THRESHOLD", + 3, + ) + ingestion_safety_cooldown_blocked_seconds: int = _env_int( + "INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS", + 1800, + ) + ingestion_safety_cooldown_network_seconds: int = _env_int( + "INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS", + 900, + ) + ingestion_continuation_queue_enabled: bool = _env_bool( + "INGESTION_CONTINUATION_QUEUE_ENABLED", + True, + ) + ingestion_continuation_base_delay_seconds: int = _env_int( + "INGESTION_CONTINUATION_BASE_DELAY_SECONDS", + 120, + ) + ingestion_continuation_max_delay_seconds: int = _env_int( + "INGESTION_CONTINUATION_MAX_DELAY_SECONDS", + 3600, + ) + ingestion_continuation_max_attempts: int = _env_int( + "INGESTION_CONTINUATION_MAX_ATTEMPTS", + 6, + ) + scheduler_queue_batch_size: int = _env_int("SCHEDULER_QUEUE_BATCH_SIZE", 10) + scheduler_pdf_queue_batch_size: int = _env_int("SCHEDULER_PDF_QUEUE_BATCH_SIZE", 15) + frontend_enabled: bool = _env_bool("FRONTEND_ENABLED", True) + frontend_dist_dir: str = _env_str("FRONTEND_DIST_DIR", "/app/frontend/dist") + scholar_image_upload_dir: str = _env_str( + "SCHOLAR_IMAGE_UPLOAD_DIR", + "/tmp/scholarr_uploads/scholar_images", + ) + scholar_image_upload_max_bytes: int = _env_int( + "SCHOLAR_IMAGE_UPLOAD_MAX_BYTES", + 2_000_000, + ) + scholar_name_search_enabled: bool = _env_bool("SCHOLAR_NAME_SEARCH_ENABLED", True) + scholar_name_search_cache_ttl_seconds: int = _env_int( + "SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS", + 21_600, + ) + scholar_name_search_blocked_cache_ttl_seconds: int = _env_int( + "SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS", + 300, + ) + scholar_name_search_cache_max_entries: int = _env_int( + "SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES", + 512, + ) + scholar_name_search_min_interval_seconds: float = _env_float( + "SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS", + 8.0, + ) + scholar_name_search_interval_jitter_seconds: float = _env_float( + "SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS", + 2.0, + ) + scholar_name_search_cooldown_block_threshold: int = _env_int( + "SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD", + 1, + ) + scholar_name_search_cooldown_seconds: int = _env_int( + "SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS", + 1800, + ) + scholar_name_search_alert_retry_count_threshold: int = _env_int( + "SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD", + 2, + ) + scholar_name_search_alert_cooldown_rejections_threshold: int = _env_int( + "SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD", + 3, + ) + unpaywall_enabled: bool = _env_bool("UNPAYWALL_ENABLED", True) + unpaywall_email: str = _env_str("UNPAYWALL_EMAIL", "") + unpaywall_timeout_seconds: float = _env_float("UNPAYWALL_TIMEOUT_SECONDS", 4.0) + unpaywall_min_interval_seconds: float = _env_float("UNPAYWALL_MIN_INTERVAL_SECONDS", 0.6) + unpaywall_max_items_per_request: int = _env_int("UNPAYWALL_MAX_ITEMS_PER_REQUEST", 20) + unpaywall_retry_cooldown_seconds: int = _env_int("UNPAYWALL_RETRY_COOLDOWN_SECONDS", 1800) + pdf_auto_retry_interval_seconds: int = _env_int( + "PDF_AUTO_RETRY_INTERVAL_SECONDS", + 86_400, + ) + pdf_auto_retry_first_interval_seconds: int = _env_int( + "PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS", + 3_600, + ) + pdf_auto_retry_max_attempts: int = _env_int("PDF_AUTO_RETRY_MAX_ATTEMPTS", 3) + unpaywall_pdf_discovery_enabled: bool = _env_bool("UNPAYWALL_PDF_DISCOVERY_ENABLED", True) + unpaywall_pdf_discovery_max_candidates: int = _env_int("UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES", 5) + unpaywall_pdf_discovery_max_html_bytes: int = _env_int("UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES", 500_000) + crossref_enabled: bool = _env_bool("CROSSREF_ENABLED", True) + crossref_max_rows: int = _env_int("CROSSREF_MAX_ROWS", 10) + crossref_timeout_seconds: float = _env_float("CROSSREF_TIMEOUT_SECONDS", 8.0) + crossref_min_interval_seconds: float = _env_float("CROSSREF_MIN_INTERVAL_SECONDS", 0.6) + crossref_max_lookups_per_request: int = _env_int("CROSSREF_MAX_LOOKUPS_PER_REQUEST", 8) + + openalex_api_key: str | None = os.getenv("OPENALEX_API_KEY") + crossref_api_token: str | None = os.getenv("CROSSREF_API_TOKEN") + crossref_api_mailto: str | None = os.getenv("CROSSREF_API_MAILTO") + + +settings = Settings() diff --git a/diag_pubs.py b/diag_pubs.py new file mode 100644 index 0000000..2417001 --- /dev/null +++ b/diag_pubs.py @@ -0,0 +1,39 @@ + +import asyncio +from sqlalchemy import select, func +from app.db.session import get_session_factory +from app.db.models import Publication, ScholarPublication + +async def check_publications(): + session_factory = get_session_factory() + async with session_factory() as session: + # Total publications + total_pubs = await session.execute(select(func.count()).select_from(Publication)) + print(f"Total Publications: {total_pubs.scalar()}") + + # Total scholar links + total_links = await session.execute(select(func.count()).select_from(ScholarPublication)) + print(f"Total Scholar-Publication Links: {total_links.scalar()}") + + # Check for orphans + orphans = await session.execute( + select(func.count()) + .select_from(Publication) + .outerjoin(ScholarPublication, Publication.id == ScholarPublication.publication_id) + .where(ScholarPublication.publication_id == None) + ) + print(f"Orphaned Publications (no link): {orphans.scalar()}") + + # Check sorting behavior + latest_links = await session.execute( + select(Publication.title_raw, ScholarPublication.created_at) + .join(ScholarPublication, Publication.id == ScholarPublication.publication_id) + .order_by(ScholarPublication.created_at.desc()) + .limit(10) + ) + print("\nLatest 10 links by created_at:") + for row in latest_links: + print(f"- {row.created_at}: {row.title_raw[:50]}") + +if __name__ == "__main__": + asyncio.run(check_publications()) diff --git a/docs/developer/api-contract.md b/docs/developer/api-contract.md new file mode 100644 index 0000000..6884bc6 --- /dev/null +++ b/docs/developer/api-contract.md @@ -0,0 +1,12 @@ +# API Contracts + +`Scholarr` is designed with strictly typed Pydantic V2 models, serialized across the network through standard OpenAPI `v3` specs via FastAPI. + +## DTO Models Structure +FastAPI routes dynamically ingest request/response Data Transfer Objects. +- **Example**: `PublicationListItem` no longer contains a hard-coded `.doi`. It contains a `.display_identifier` property resolving the highest confidence identifier regardless of backend origin. +- **Testing**: Run `./scripts/check_frontend_api_contract.py` or equivalent integration steps mapped in CI to ensure that backend python routes strictly map to the TypeScript types compiled by the Frontend. + +## Frontend Contract Requirements +The UI exclusively calls routes exposed by FastAPI's `APIRouter`. +All frontend integration tests enforce that Vue 3 Components do not assume hard properties directly outside the bounded TS schemas. This avoids type mismatch runtime explosions if the database is scaled horizontally. diff --git a/docs/developer/architecture.md b/docs/developer/architecture.md index 0a169e6..afa6b05 100644 --- a/docs/developer/architecture.md +++ b/docs/developer/architecture.md @@ -20,6 +20,32 @@ Canonical business logic belongs in `app/services/domains/*`. ## Frontend Behavior Notes -- Mobile primary nav is in a left drawer and closes on route change or logout. -- Long list views use internal scroll containers. -- Name search remains intentionally constrained due upstream anti-bot behavior; production onboarding should prefer scholar ID/profile URL. +- Navigation: Mobile primary nav is in a left drawer and closes on route change or logout. +- Lists: Long list views use internal scroll containers to prevent viewport overflow. +- Rate Limiting: Name search remains intentionally constrained due to upstream anti-bot behavior; production onboarding should prefer scholar ID/profile URLs directly if possible. + +## Data Integration & Acquisition Flow + +```mermaid +graph TD + UI[Frontend Vue App] --> API[FastAPI Backend] + API --> Scheduler[Background Celery/Async Scheduler] + + Scheduler -->|1. Fetch HTML| Scholar[Google Scholar HTML Parser] + Scholar -->|2. Extract Metadata| IdentifierModule[Identifier Gathering] + + IdentifierModule -->|Search arXiv API| API1(arXiv) + IdentifierModule -->|Search Crossref API| API2(Crossref) + + IdentifierModule -->|3. Save Identifiers| DB[(PostgreSQL)] + + Scheduler -->|4. Resolve PDF| PDFPipeline[PDF Resolution Pipeline] + DB --> |Identified DOIs| PDFPipeline + PDFPipeline -->|Search Open Access APIs| Unpaywall(Unpaywall API) + + Unpaywall --> |Acquire PDF URL| PDFWorker[PDF Download Worker] + PDFWorker --> |Store PDF Metadata| DB +``` + +### Identifier Engine Philosophy +The platform previously treated the `DOI` as a hardcoded 1:1 property of a publication. It now utilizes a decoupled *Identifier Gathering* module (`PublicationIdentifier` table). A single publication can have multiple identifiers (ex. `doi`, `arxiv`, `pmid`, `pmcid`). This creates high resilience when integrating with external APIs, allowing systems like Unpaywall to be fed explicitly with high-confidence DOIs, rather than relying on unstructured search heuristics. diff --git a/docs/developer/ingestion.md b/docs/developer/ingestion.md new file mode 100644 index 0000000..b906f15 --- /dev/null +++ b/docs/developer/ingestion.md @@ -0,0 +1,26 @@ +# Ingestion System & Backoff Strategies + +The `ScholarIngestionService` drives the primary data acquisition loop. Since Google Scholar utilizes heavy bot protection and rate limits, this package contains nuanced backoffs to protect user networks from automated IP bans. + +## Ingestion Overview +The underlying ingestion process: +1. Receives an explicit request or background cron trigger to resolve a `scholar_profile_id`. +2. Connects asynchronously using configured HTTPX adapters with strict browser headers. +3. Downloads the paginated HTML feed for a user across multiple page iterations. +4. Uses `regex` and DOM-invariants (e.g. `gsc_vcd_cib`) to pull individual publication blocks. + +## Handling 429 Too Many Requests +Google Scholar aggressively throws HTTP 429 responses if multiple concurrent tabs or rapidly sequential commands query the same IP address for specific API endpoints (like `citations?view_op=view_citation...`). + +Scholarr treats these distinct from random network timeouts. +- **Network Error Retries**: Handled via `ingestion_network_error_retries` with a base backoff of `ingestion_retry_backoff_seconds` (Default 1.0s). +- **Rate Limit 429 Retries**: When `ParseState.BLOCKED_OR_CAPTCHA` captures `blocked_http_429_rate_limited`, the system applies a dedicated cooldown. It respects `ingestion_rate_limit_retries` multiplied by `ingestion_rate_limit_backoff_seconds` (Default 30.0s). This prevents the pipeline from fatalizing a user's job completely, pausing operations seamlessly instead. + +## Publication Identifiers Loop +Once a publication is built, the `gather_identifiers_for_publication` module isolates keys explicitly. +- **Local Parsing**: Searches for direct identifiers within the HTML parameters (DOI patterns, arXiv regexes). +- **API Fetching**: Queries secondary bibliographic platforms sequentially: + - `export.arxiv.org/api/query` (Queries by Title and Author strings). + - `crossref.restful` APIs (Queries by Title and Author strings). + +These identifiers are accumulated in `publication_identifiers` instead of being bound as hard-coded properties, maximizing matching resilience in the automated Unpaywall PDF acquisition stage. diff --git a/docs/developer/overview.md b/docs/developer/overview.md index 4d9880e..7ab91cc 100644 --- a/docs/developer/overview.md +++ b/docs/developer/overview.md @@ -5,5 +5,7 @@ Use this section if you are contributing code or running full quality gates. - [Documentation Standards](./documentation-standards.md) - [Local Development](./local-development.md) - [Architecture Boundaries](./architecture.md) +- [Ingestion API & Backoff Strategies](./ingestion.md) +- [API Contracts](./api-contract.md) - [Contributing](./contributing.md) - [Frontend Theme Inventory](./frontend-theme-inventory.md) diff --git a/frontend/src/components/layout/AppNav.vue b/frontend/src/components/layout/AppNav.vue index 788e5bf..1679059 100644 --- a/frontend/src/components/layout/AppNav.vue +++ b/frontend/src/components/layout/AppNav.vue @@ -39,6 +39,10 @@ const navRunStateStatus = computed(() => { if (runStatus.isSubmitting && !runStatus.isLikelyRunning) { return "starting"; } + const s = runStatus.latestRun?.status; + if (s === "resolving") { + return "resolving"; + } if (runStatus.isLikelyRunning) { return "running"; } diff --git a/frontend/src/components/patterns/RunStatusBadge.vue b/frontend/src/components/patterns/RunStatusBadge.vue index 575b5c7..da5accf 100644 --- a/frontend/src/components/patterns/RunStatusBadge.vue +++ b/frontend/src/components/patterns/RunStatusBadge.vue @@ -20,9 +20,15 @@ const toneClass = computed(() => { if (props.status === "running") { return "border-state-info-border bg-state-info-bg text-state-info-text"; } + if (props.status === "resolving") { + return "border-state-info-border bg-state-info-bg text-state-info-text"; + } if (props.status === "starting") { return "border-state-info-border bg-state-info-bg text-state-info-text"; } + if (props.status === "canceled") { + return "border-state-warning-border bg-state-warning-bg text-state-warning-text"; + } return "border-stroke-default bg-surface-card-muted text-ink-secondary"; }); diff --git a/frontend/src/features/admin_dbops/index.ts b/frontend/src/features/admin_dbops/index.ts index 2f042f5..0b16b3a 100644 --- a/frontend/src/features/admin_dbops/index.ts +++ b/frontend/src/features/admin_dbops/index.ts @@ -41,7 +41,6 @@ export interface AdminDbRepairJob { export interface AdminPdfQueueItem { publication_id: number; title: string; - doi: string | null; display_identifier: DisplayIdentifier | null; pdf_url: string | null; status: string; @@ -160,3 +159,19 @@ export async function requeueAllAdminPdfLookups(limit = 1000): Promise { + const response = await apiRequest( + "/admin/db/drop-all-publications", + { + method: "POST", + body: { confirmation_text: confirmationText }, + }, + ); + return response.data; +} diff --git a/frontend/src/features/publications/index.ts b/frontend/src/features/publications/index.ts index f238a0e..3f80806 100644 --- a/frontend/src/features/publications/index.ts +++ b/frontend/src/features/publications/index.ts @@ -19,7 +19,6 @@ export interface PublicationItem { citation_count: number; venue_text: string | null; pub_url: string | null; - doi: string | null; display_identifier: DisplayIdentifier | null; pdf_url: string | null; pdf_status: "untracked" | "queued" | "running" | "resolved" | "failed"; @@ -44,6 +43,7 @@ export interface PublicationsResult { total_count: number; page: number; page_size: number; + snapshot: string; has_next: boolean; has_prev: boolean; publications: PublicationItem[]; @@ -53,8 +53,12 @@ export interface PublicationsQuery { mode?: PublicationMode; favoriteOnly?: boolean; scholarProfileId?: number; + search?: string; + sortBy?: string; + sortDir?: "asc" | "desc"; page?: number; pageSize?: number; + snapshot?: string; } export interface PublicationSelection { @@ -74,12 +78,24 @@ export async function listPublications(query: PublicationsQuery = {}): Promise

      0) { + params.set("search", query.search.trim()); + } + if (query.sortBy) { + params.set("sort_by", query.sortBy); + } + if (query.sortDir) { + params.set("sort_dir", query.sortDir); + } const parsedPage = Number.isFinite(query.page) ? Math.max(1, Math.trunc(Number(query.page))) : 1; const parsedPageSize = Number.isFinite(query.pageSize) ? Math.max(1, Math.min(500, Math.trunc(Number(query.pageSize)))) : 100; params.set("page", String(parsedPage)); params.set("page_size", String(parsedPageSize)); + if (query.snapshot && query.snapshot.trim().length > 0) { + params.set("snapshot", query.snapshot.trim()); + } const suffix = params.toString(); const response = await apiRequest( diff --git a/frontend/src/features/runs/index.ts b/frontend/src/features/runs/index.ts index 97d49c6..735776d 100644 --- a/frontend/src/features/runs/index.ts +++ b/frontend/src/features/runs/index.ts @@ -145,6 +145,13 @@ export async function triggerManualRun(): Promise<{ return response.data; } +export async function cancelRun(runId: number): Promise { + const response = await apiRequest(`/runs/${runId}/cancel`, { + method: "POST", + }); + return response.data; +} + export async function listQueueItems(limit = 200): Promise { const response = await apiRequest(`/runs/queue/items?limit=${limit}`, { method: "GET", diff --git a/frontend/src/features/scholars/index.ts b/frontend/src/features/scholars/index.ts index db19e76..ef3c8c4 100644 --- a/frontend/src/features/scholars/index.ts +++ b/frontend/src/features/scholars/index.ts @@ -54,7 +54,6 @@ export interface PublicationExportItem { author_text: string | null; venue_text: string | null; pub_url: string | null; - doi: string | null; pdf_url: string | null; is_read: boolean; } @@ -86,7 +85,7 @@ interface ScholarsListData { scholars: ScholarProfile[]; } -interface ScholarSearchData extends ScholarSearchResult {} +interface ScholarSearchData extends ScholarSearchResult { } export async function listScholars(): Promise { const response = await apiRequest("/scholars", { method: "GET" }); diff --git a/frontend/src/features/settings/SettingsAdminPanel.vue b/frontend/src/features/settings/SettingsAdminPanel.vue index 7edf67b..e5edfb4 100644 --- a/frontend/src/features/settings/SettingsAdminPanel.vue +++ b/frontend/src/features/settings/SettingsAdminPanel.vue @@ -14,6 +14,7 @@ import AppRefreshButton from "@/components/ui/AppRefreshButton.vue"; import AppSelect from "@/components/ui/AppSelect.vue"; import AppTable from "@/components/ui/AppTable.vue"; import { + dropAllPublications, getAdminDbIntegrityReport, listAdminDbRepairJobs, listAdminPdfQueue, @@ -42,6 +43,7 @@ const SECTION_PDF = "pdf"; const SCOPE_SINGLE_USER = "single_user"; const SCOPE_ALL_USERS = "all_users"; const APPLY_ALL_USERS_CONFIRM_TEXT = "REPAIR ALL USERS"; +const DROP_PUBLICATIONS_CONFIRM_TEXT = "DROP ALL PUBLICATIONS"; type RepairScopeMode = typeof SCOPE_SINGLE_USER | typeof SCOPE_ALL_USERS; @@ -81,6 +83,13 @@ const repairDryRun = ref(true); const repairGcOrphans = ref(false); const repairConfirmationText = ref(""); +const droppingPublications = ref(false); +const dropConfirmationText = ref(""); +const dropResult = ref<{ deleted_count: number; message: string } | null>(null); +const dropConfirmationValid = computed( + () => dropConfirmationText.value.trim() === DROP_PUBLICATIONS_CONFIRM_TEXT, +); + const refreshingPdfQueue = ref(false); const requeueingPublicationId = ref(null); const requeueingAllPdfs = ref(false); @@ -433,6 +442,22 @@ async function onRequeueAllPdfs(): Promise { } } +async function onDropAllPublications(): Promise { + droppingPublications.value = true; + clearAlerts(); + dropResult.value = null; + try { + const result = await dropAllPublications(dropConfirmationText.value.trim()); + dropResult.value = result; + successMessage.value = result.message; + dropConfirmationText.value = ""; + } catch (error) { + assignError(error, "Unable to drop publications."); + } finally { + droppingPublications.value = false; + } +} + onMounted(async () => { loading.value = true; clearAlerts(); @@ -686,6 +711,45 @@ watch( + + +

      +

      Drop All Publications

      + +
      + +

      + This action is irreversible. It deletes every publication record across all users. + The next ingestion run will re-populate all data from scratch. +

      + +
      + +
      + + {{ droppingPublications ? "Dropping..." : "Drop all publications" }} + +
      +
      + +
      +
      + Deleted: {{ dropResult.deleted_count }} +
      +

      {{ dropResult.message }}

      +
      + diff --git a/frontend/src/features/settings/index.ts b/frontend/src/features/settings/index.ts index e8ef325..6ff6c94 100644 --- a/frontend/src/features/settings/index.ts +++ b/frontend/src/features/settings/index.ts @@ -19,6 +19,9 @@ export interface UserSettings { nav_visible_pages: string[]; policy: UserSettingsPolicy; safety_state: ScrapeSafetyState; + openalex_api_key: string | null; + crossref_api_token: string | null; + crossref_api_mailto: string | null; } export interface UserSettingsUpdate { @@ -26,6 +29,9 @@ export interface UserSettingsUpdate { run_interval_minutes: number; request_delay_seconds: number; nav_visible_pages: string[]; + openalex_api_key: string | null; + crossref_api_token: string | null; + crossref_api_mailto: string | null; } export interface ChangePasswordPayload { diff --git a/frontend/src/pages/DashboardPage.vue b/frontend/src/pages/DashboardPage.vue index e61df8d..90d8513 100644 --- a/frontend/src/pages/DashboardPage.vue +++ b/frontend/src/pages/DashboardPage.vue @@ -64,6 +64,7 @@ const startCheckLabel = computed(() => { const isStartCheckAnimating = computed( () => runStatus.isSubmitting || runStatus.isLikelyRunning, ); +const isCancelAnimating = ref(false); const displayedLatestRun = computed(() => { const snapshotLatest = snapshot.value?.latestRun ?? null; @@ -102,6 +103,25 @@ const recentRuns = computed(() => { mergedRuns.splice(existingIndex, 1, latest); return mergedRuns; }); + +const recentPublications = computed(() => { + const stream = runStatus.livePublications; + // DashboardSnapshot's recentPublications is compatible with PublicationItem, + // but let's cast it slightly to guarantee iteration. + const base = snapshot.value?.recentPublications ?? []; + const merged = [...stream, ...base]; + + const seenIds = new Set(); + const deduped: any[] = []; + for (const item of merged) { + if (!seenIds.has(item.publication_id)) { + seenIds.add(item.publication_id); + deduped.push(item); + } + } + return deduped; +}); + const activeRunId = computed(() => runStatus.latestRun?.status === "running" ? runStatus.latestRun.id : null); const showRunningHint = computed( () => runStatus.isLikelyRunning && displayedLatestRun.value?.status !== "running", @@ -178,7 +198,6 @@ async function onTriggerRun(): Promise { return; } - errorMessage.value = result.message; errorRequestId.value = result.requestId; } catch { errorMessage.value = "Unable to start an update check."; @@ -186,6 +205,28 @@ async function onTriggerRun(): Promise { } } +async function onCancelRun(): Promise { + if (!activeRunId.value) return; + errorMessage.value = null; + errorRequestId.value = null; + successMessage.value = null; + isCancelAnimating.value = true; + + try { + const result = await runStatus.cancelActiveCheck(); + if (result.kind === "success") { + successMessage.value = "Update check canceled successfully."; + await loadSnapshot(); + } else { + errorMessage.value = result.message; + } + } catch { + errorMessage.value = "Unable to cancel the update check."; + } finally { + isCancelAnimating.value = false; + } +} + onMounted(() => { void loadSnapshot(); void runStatus.syncLatest(); @@ -252,14 +293,14 @@ watch(
  • @@ -328,6 +369,22 @@ watch( />
    + + + + + + + Cancel check + + ("first_seen"); const sortDirection = ref<"asc" | "desc">("desc"); const currentPage = ref(1); const pageSize = ref("50"); +const publicationSnapshot = ref(null); const scholars = ref([]); const listState = ref(null); @@ -190,27 +189,42 @@ const selectedScholarName = computed(() => { }); const filteredPublications = computed(() => { + let stream = [...runStatus.livePublications]; + if (favoriteOnly.value) { + stream = stream.filter((p) => p.is_favorite); + } + if (mode.value === "unread") { + stream = stream.filter((p) => !p.is_read); + } + const selectedScholarId = Number(selectedScholarFilter.value); + if (Number.isInteger(selectedScholarId) && selectedScholarId > 0) { + stream = stream.filter((p) => p.scholar_profile_id === selectedScholarId); + } + const base = listState.value?.publications ?? []; + const merged = [...stream, ...base]; + const seenIds = new Set(); + const deduped: typeof base = []; + for (const item of merged) { + if (!seenIds.has(item.publication_id)) { + seenIds.add(item.publication_id); + deduped.push(item); + } + } + const normalized = searchQuery.value.trim().toLowerCase(); if (!normalized) { - return base; + return deduped; } - return base.filter((item) => { - const year = item.year === null ? "" : String(item.year); - return [item.title, item.scholar_label, item.venue_text || "", year] - .join(" ") - .toLowerCase() - .includes(normalized); - }); + // Client-side fallback: filter live-discovered publications that haven't been + // server-round-tripped yet. The main server query already filters by search. + return deduped; }); function publicationSortValue(item: PublicationItem, key: PublicationSortKey): number | string { if (key === "title") { return item.title; } - if (key === "favorite") { - return item.is_favorite ? 1 : 0; - } if (key === "scholar") { return item.scholar_label; } @@ -220,38 +234,14 @@ function publicationSortValue(item: PublicationItem, key: PublicationSortKey): n if (key === "citations") { return item.citation_count; } - if (key === "status") { - if (item.is_read) { - return 2; - } - if (item.is_new_in_latest_run) { - return 0; - } - return 1; - } const timestamp = Date.parse(item.first_seen_at); return Number.isNaN(timestamp) ? 0 : timestamp; } const sortedPublications = computed(() => { - const sorted = [...filteredPublications.value]; - sorted.sort((a, b) => { - const left = publicationSortValue(a, sortKey.value); - const right = publicationSortValue(b, sortKey.value); - - let comparison: number; - if (typeof left === "string" && typeof right === "string") { - comparison = textCollator.compare(left, right); - } else { - comparison = Number(left) - Number(right); - } - - if (comparison === 0) { - comparison = textCollator.compare(a.title, b.title); - } - return sortDirection.value === "asc" ? comparison : comparison * -1; - }); - return sorted; + // Server already returns data in the correct sort order. + // We just pass through the filtered/merged list without client-side re-sorting. + return filteredPublications.value; }); const visibleUnreadKeys = computed(() => { @@ -437,13 +427,16 @@ watch(hasSelection, (nextHasSelection) => { bulkAction.value = nextHasSelection ? "mark_selected_read" : "mark_all_unread_read"; }); -function toggleSort(nextKey: PublicationSortKey): void { +async function toggleSort(nextKey: PublicationSortKey): Promise { if (sortKey.value === nextKey) { sortDirection.value = sortDirection.value === "asc" ? "desc" : "asc"; - return; + } else { + sortKey.value = nextKey; + sortDirection.value = nextKey === "first_seen" ? "desc" : "asc"; } - sortKey.value = nextKey; - sortDirection.value = nextKey === "first_seen" ? "desc" : "asc"; + currentPage.value = 1; + publicationSnapshot.value = null; + await loadPublications(); } function sortMarker(key: PublicationSortKey): string { @@ -476,9 +469,14 @@ async function loadPublications(): Promise { mode: mode.value, favoriteOnly: favoriteOnly.value, scholarProfileId: selectedScholarId(), + search: searchQuery.value.trim() || undefined, + sortBy: sortKey.value, + sortDir: sortDirection.value, page: currentPage.value, pageSize: pageSizeValue.value, + snapshot: publicationSnapshot.value ?? undefined, }); + publicationSnapshot.value = listState.value.snapshot; currentPage.value = listState.value.page; pageSize.value = String(listState.value.page_size); selectedPublicationKeys.value = new Set(); @@ -498,12 +496,14 @@ async function loadPublications(): Promise { async function onModeChanged(): Promise { currentPage.value = 1; + publicationSnapshot.value = null; await syncFiltersToRoute(); await loadPublications(); } async function onScholarFilterChanged(): Promise { currentPage.value = 1; + publicationSnapshot.value = null; await syncFiltersToRoute(); await loadPublications(); } @@ -511,12 +511,14 @@ async function onScholarFilterChanged(): Promise { async function onFavoriteOnlyChanged(): Promise { favoriteOnly.value = !favoriteOnly.value; currentPage.value = 1; + publicationSnapshot.value = null; await syncFiltersToRoute(); await loadPublications(); } async function onPageSizeChanged(): Promise { currentPage.value = 1; + publicationSnapshot.value = null; await syncFiltersToRoute(); await loadPublications(); } @@ -605,15 +607,15 @@ function canRetryPublicationPdf(item: PublicationItem): boolean { function pdfPendingLabel(item: PublicationItem): string { if (item.pdf_status === "queued") { - return "queued"; + return "Queued"; } if (item.pdf_status === "running") { - return "resolving"; + return "Resolving..."; } if (item.pdf_status === "failed") { - return "failed"; + return "Missing"; } - return "untracked"; + return "Untracked"; } function replacePublication(updated: PublicationItem): void { @@ -784,6 +786,18 @@ function resetSearchQuery(): void { searchQuery.value = ""; } +let searchDebounceTimer: ReturnType | null = null; +watch(searchQuery, () => { + if (searchDebounceTimer !== null) { + clearTimeout(searchDebounceTimer); + } + searchDebounceTimer = setTimeout(() => { + currentPage.value = 1; + publicationSnapshot.value = null; + void loadPublications(); + }, 300); +}); + onMounted(() => { syncFiltersFromRoute(); void Promise.all([loadScholarFilters(), loadPublications(), runStatus.syncLatest()]); @@ -792,10 +806,18 @@ onMounted(() => { watch( () => [route.query.scholar, route.query.favorite, route.query.page], async () => { + const previousScholar = selectedScholarFilter.value; + const previousFavorite = favoriteOnly.value; const changed = syncFiltersFromRoute(); if (!changed) { return; } + if ( + selectedScholarFilter.value !== previousScholar + || favoriteOnly.value !== previousFavorite + ) { + publicationSnapshot.value = null; + } await loadPublications(); }, ); @@ -843,7 +865,7 @@ watch(