justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

temp commit

Browse source
This commit is contained in:
Justin Visser 2026-02-26 12:54:19 +01:00
parent 8760f27b51
commit 0e9e49df16
193 changed files with 23228 additions and 935 deletions
Download patch file Download diff file Expand all files Collapse all files

4
app/services/domains/publications/application.py
View file

@ -24,7 +24,7 @@ from app.services.domains.publications.modes import (
resolve_publication_view_mode,
)
from app.services.domains.publications.queries import (
get_latest_completed_run_id_for_user,
get_latest_run_id_for_user,
get_publication_item_for_user,
publications_query,
)
@ -50,7 +50,7 @@ __all__ = [
"PublicationListItem",
"UnreadPublicationItem",
"resolve_publication_view_mode",
"get_latest_completed_run_id_for_user",
"get_latest_run_id_for_user",
"publications_query",
"get_publication_item_for_user",
"list_for_user",

19
app/services/domains/publications/counts.py
View file

@ -1,6 +1,8 @@
from __future__ import annotations
from sqlalchemy import func, select
from datetime import datetime
from sqlalchemy import distinct, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import ScholarProfile, ScholarPublication
@ -10,7 +12,7 @@ from app.services.domains.publications.modes import (
MODE_UNREAD,
resolve_publication_view_mode,
)
from app.services.domains.publications.queries import get_latest_completed_run_id_for_user
from app.services.domains.publications.queries import get_latest_run_id_for_user
async def count_for_user(
@ -20,11 +22,12 @@ async def count_for_user(
mode: str = MODE_ALL,
scholar_profile_id: int | None = None,
favorite_only: bool = False,
snapshot_before: datetime | None = None,
) -> int:
resolved_mode = resolve_publication_view_mode(mode)
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
latest_run_id = await get_latest_run_id_for_user(db_session, user_id=user_id)
stmt = (
select(func.count())
select(func.count(distinct(ScholarPublication.publication_id)))
.select_from(ScholarPublication)
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
.where(ScholarProfile.user_id == user_id)
@ -33,6 +36,8 @@ async def count_for_user(
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
if favorite_only:
stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
if snapshot_before is not None:
stmt = stmt.where(ScholarPublication.created_at <= snapshot_before)
if resolved_mode == MODE_UNREAD:
stmt = stmt.where(ScholarPublication.is_read.is_(False))
if resolved_mode == MODE_LATEST:
@ -49,6 +54,7 @@ async def count_unread_for_user(
user_id: int,
scholar_profile_id: int | None = None,
favorite_only: bool = False,
snapshot_before: datetime | None = None,
) -> int:
return await count_for_user(
db_session,
@ -56,6 +62,7 @@ async def count_unread_for_user(
mode=MODE_UNREAD,
scholar_profile_id=scholar_profile_id,
favorite_only=favorite_only,
snapshot_before=snapshot_before,
)
@ -65,6 +72,7 @@ async def count_latest_for_user(
user_id: int,
scholar_profile_id: int | None = None,
favorite_only: bool = False,
snapshot_before: datetime | None = None,
) -> int:
return await count_for_user(
db_session,
@ -72,6 +80,7 @@ async def count_latest_for_user(
mode=MODE_LATEST,
scholar_profile_id=scholar_profile_id,
favorite_only=favorite_only,
snapshot_before=snapshot_before,
)
@ -80,6 +89,7 @@ async def count_favorite_for_user(
*,
user_id: int,
scholar_profile_id: int | None = None,
snapshot_before: datetime | None = None,
) -> int:
return await count_for_user(
db_session,
@ -87,4 +97,5 @@ async def count_favorite_for_user(
mode=MODE_ALL,
scholar_profile_id=scholar_profile_id,
favorite_only=True,
snapshot_before=snapshot_before,
)

103
app/services/domains/publications/dedup.py Normal file
View file

@ -0,0 +1,103 @@
from __future__ import annotations
import logging
from sqlalchemy import delete, select
from sqlalchemy.orm import aliased
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Publication, PublicationIdentifier, ScholarPublication
logger = logging.getLogger(__name__)
async def find_identifier_duplicate_pairs(
db_session: AsyncSession,
) -> list[tuple[int, int]]:
"""Return (winner_id, dup_id) pairs where two publications share the same identifier.
Winner is always the lower publication_id (earlier-created). Uses the existing
ix_publication_identifiers_kind_value index for the self-join.
"""
pi1 = aliased(PublicationIdentifier, name="pi1")
pi2 = aliased(PublicationIdentifier, name="pi2")
rows = await db_session.execute(
select(pi1.publication_id, pi2.publication_id)
.join(
pi2,
(pi1.kind == pi2.kind)
& (pi1.value_normalized == pi2.value_normalized)
& (pi1.publication_id < pi2.publication_id),
)
.distinct()
)
return [(winner_id, dup_id) for winner_id, dup_id in rows]
async def merge_duplicate_publication(
db_session: AsyncSession,
*,
winner_id: int,
dup_id: int,
) -> None:
"""Merge dup_id into winner_id: migrate scholar links, then delete the dup."""
await _migrate_scholar_links(db_session, winner_id=winner_id, dup_id=dup_id)
await db_session.execute(
delete(Publication).where(Publication.id == dup_id)
)
logger.info(
"publications.identifier_merge",
extra={
"event": "publications.identifier_merge",
"winner_id": winner_id,
"dup_id": dup_id,
},
)
async def _migrate_scholar_links(
db_session: AsyncSession,
*,
winner_id: int,
dup_id: int,
) -> None:
"""Move ScholarPublication links from dup to winner, dropping conflicts."""
dup_links_result = await db_session.execute(
select(ScholarPublication).where(ScholarPublication.publication_id == dup_id)
)
dup_links = dup_links_result.scalars().all()
winner_profiles_result = await db_session.execute(
select(ScholarPublication.scholar_profile_id).where(
ScholarPublication.publication_id == winner_id
)
)
winner_profiles: set[int] = {row for (row,) in winner_profiles_result}
for link in dup_links:
if link.scholar_profile_id in winner_profiles:
await db_session.delete(link)
else:
link.publication_id = winner_id
async def sweep_identifier_duplicates(db_session: AsyncSession) -> int:
"""Find publications sharing an identifier and merge duplicates into the winner.
Returns the number of duplicate publications removed.
"""
pairs = await find_identifier_duplicate_pairs(db_session)
if not pairs:
return 0
# Deduplicate the pairs — a dup may appear multiple times if it shares
# several identifiers with the winner; process each dup only once.
processed_dups: set[int] = set()
for winner_id, dup_id in pairs:
if dup_id in processed_dups:
continue
processed_dups.add(dup_id)
await merge_duplicate_publication(db_session, winner_id=winner_id, dup_id=dup_id)
await db_session.flush()
return len(processed_dups)

16
app/services/domains/publications/listing.py
View file

@ -1,5 +1,7 @@
from __future__ import annotations
from datetime import datetime
from sqlalchemy.ext.asyncio import AsyncSession
from app.services.domains.publication_identifiers import application as identifier_service
@ -9,7 +11,7 @@ from app.services.domains.publications.modes import (
resolve_publication_view_mode,
)
from app.services.domains.publications.queries import (
get_latest_completed_run_id_for_user,
get_latest_run_id_for_user,
get_publication_item_for_user,
publication_list_item_from_row,
publications_query,
@ -25,11 +27,15 @@ async def list_for_user(
mode: str = MODE_ALL,
scholar_profile_id: int | None = None,
favorite_only: bool = False,
limit: int = 300,
search: str | None = None,
sort_by: str = "first_seen",
sort_dir: str = "desc",
limit: int = 100,
offset: int = 0,
snapshot_before: datetime | None = None,
) -> list[PublicationListItem]:
resolved_mode = resolve_publication_view_mode(mode)
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
latest_run_id = await get_latest_run_id_for_user(db_session, user_id=user_id)
result = await db_session.execute(
publications_query(
user_id=user_id,
@ -37,8 +43,12 @@ async def list_for_user(
latest_run_id=latest_run_id,
scholar_profile_id=scholar_profile_id,
favorite_only=favorite_only,
search=search,
sort_by=sort_by,
sort_dir=sort_dir,
limit=limit,
offset=offset,
snapshot_before=snapshot_before,
)
)
rows = [

140
app/services/domains/publications/pdf_queue.py
View file

@ -2,10 +2,10 @@ from __future__ import annotations
import asyncio
from dataclasses import dataclass
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
import logging
from sqlalchemy import Select, func, literal, or_, select, union_all
from sqlalchemy import Select, and_, func, literal, or_, select, union_all
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import (
@ -20,7 +20,6 @@ from app.db.session import get_session_factory
from app.services.domains.publication_identifiers import application as identifier_service
from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.domains.publications.pdf_resolution_pipeline import (
PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE,
resolve_publication_pdf_outcome_for_row,
)
from app.services.domains.publications.types import PublicationListItem
@ -49,7 +48,6 @@ _scheduled_tasks: set[asyncio.Task[None]] = set()
class PdfQueueListItem:
publication_id: int
title: str
doi: str | None
pdf_url: str | None
status: str
attempt_count: int
@ -114,7 +112,6 @@ def _item_from_row_and_job(
citation_count=row.citation_count,
venue_text=row.venue_text,
pub_url=row.pub_url,
doi=row.doi,
pdf_url=row.pdf_url,
is_read=row.is_read,
is_favorite=row.is_favorite,
@ -360,7 +357,7 @@ def _failed_outcome(
) -> OaResolutionOutcome:
return OaResolutionOutcome(
publication_id=row.publication_id,
doi=row.doi,
doi=None,
pdf_url=None,
failure_reason=FAILURE_RESOLUTION_EXCEPTION,
source=None,
@ -372,10 +369,12 @@ async def _fetch_outcome_for_row(
*,
row: PublicationListItem,
request_email: str | None,
openalex_api_key: str | None = None,
) -> OaResolutionOutcome:
pipeline_result = await resolve_publication_pdf_outcome_for_row(
row=row,
request_email=request_email,
openalex_api_key=openalex_api_key,
)
outcome = pipeline_result.outcome
if outcome is not None:
@ -386,11 +385,8 @@ async def _fetch_outcome_for_row(
def _apply_publication_update(
publication: Publication,
*,
doi: str | None,
pdf_url: str | None,
) -> None:
if doi and publication.doi != doi:
publication.doi = doi
if pdf_url and publication.pdf_url != pdf_url:
publication.pdf_url = pdf_url
@ -426,7 +422,7 @@ async def _persist_outcome(
job = await db_session.get(PublicationPdfJob, publication_id)
if publication is None or job is None:
return
_apply_publication_update(publication, doi=outcome.doi, pdf_url=outcome.pdf_url)
_apply_publication_update(publication, pdf_url=outcome.pdf_url)
await identifier_service.sync_identifiers_for_publication_resolution(
db_session,
publication=publication,
@ -453,10 +449,26 @@ async def _resolve_publication_row(
user_id: int,
request_email: str | None,
row: PublicationListItem,
openalex_api_key: str | None = None,
) -> None:
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
from app.services.domains.arxiv.application import ArxivRateLimitError
await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
try:
outcome = await _fetch_outcome_for_row(row=row, request_email=request_email)
outcome = await _fetch_outcome_for_row(
row=row,
request_email=request_email,
openalex_api_key=openalex_api_key,
)
except (OpenAlexBudgetExhaustedError, ArxivRateLimitError):
# Persist a terminal outcome so jobs do not remain stuck in "running".
await _persist_outcome(
publication_id=row.publication_id,
user_id=user_id,
outcome=_failed_outcome(row=row),
)
# Propagate upward so the batch loop can stop immediately.
raise
except Exception as exc: # pragma: no cover - defensive network boundary
logger.warning(
"publications.pdf_queue.resolve_failed",
@ -480,12 +492,44 @@ async def _run_resolution_task(
request_email: str | None,
rows: list[PublicationListItem],
) -> None:
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
from app.services.domains.arxiv.application import ArxivRateLimitError
from app.services.domains.settings import application as user_settings_service
# Resolve the best available API key: per-user setting → env var fallback.
openalex_api_key: str | None = None
try:
session_factory = get_session_factory()
async with session_factory() as key_session:
user_settings = await user_settings_service.get_or_create_settings(key_session, user_id=user_id)
openalex_api_key = getattr(user_settings, "openalex_api_key", None) or settings.openalex_api_key
except Exception:
openalex_api_key = settings.openalex_api_key
for row in rows:
await _resolve_publication_row(
user_id=user_id,
request_email=request_email,
row=row,
)
try:
await _resolve_publication_row(
user_id=user_id,
request_email=request_email,
row=row,
openalex_api_key=openalex_api_key,
)
except OpenAlexBudgetExhaustedError:
logger.warning(
"publications.pdf_queue.budget_exhausted",
extra={"event": "publications.pdf_queue.budget_exhausted",
"detail": "Stopping PDF resolution batch — OpenAlex daily budget exhausted"},
)
break
except ArxivRateLimitError:
logger.warning(
"publications.pdf_queue.arxiv_rate_limited",
extra={
"event": "publications.pdf_queue.arxiv_rate_limited",
"detail": "Stopping PDF resolution batch — arXiv rate limit hit (429)",
},
)
break
def _schedule_rows(
@ -571,7 +615,6 @@ def _retry_item_from_publication(
citation_count=int(publication.citation_count or 0),
venue_text=publication.venue_text,
pub_url=publication.pub_url,
doi=publication.doi,
pdf_url=publication.pdf_url,
is_read=is_read,
first_seen_at=first_seen_at,
@ -629,13 +672,12 @@ def _queue_candidate_from_publication(publication: Publication) -> PublicationLi
return PublicationListItem(
publication_id=int(publication.id),
scholar_profile_id=0,
scholar_label="admin",
scholar_label="",
title=publication.title_raw,
year=publication.year,
citation_count=int(publication.citation_count or 0),
venue_text=publication.venue_text,
pub_url=publication.pub_url,
doi=publication.doi,
pdf_url=publication.pdf_url,
is_read=True,
first_seen_at=publication.created_at or _utcnow(),
@ -649,6 +691,9 @@ async def _missing_pdf_candidates(
limit: int,
) -> list[PublicationListItem]:
bounded_limit = max(1, min(int(limit), 5000))
now = datetime.now(timezone.utc)
cooldown_threshold = now - timedelta(days=7)
result = await db_session.execute(
select(Publication)
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
@ -656,7 +701,13 @@ async def _missing_pdf_candidates(
.where(
or_(
PublicationPdfJob.publication_id.is_(None),
PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
and_(
PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
or_(
PublicationPdfJob.last_attempt_at.is_(None),
PublicationPdfJob.last_attempt_at < cooldown_threshold,
),
),
)
)
.order_by(Publication.updated_at.desc(), Publication.id.desc())
@ -694,7 +745,6 @@ def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
select(
PublicationPdfJob.publication_id,
Publication.title_raw,
Publication.doi,
Publication.pdf_url,
PublicationPdfJob.status,
PublicationPdfJob.attempt_count,
@ -730,7 +780,6 @@ def _untracked_queue_select_base() -> Select[tuple]:
select(
Publication.id,
Publication.title_raw,
Publication.doi,
Publication.pdf_url,
literal(PDF_STATUS_UNTRACKED),
literal(0),
@ -793,19 +842,18 @@ def _queue_item_from_row(row: tuple) -> PdfQueueListItem:
return PdfQueueListItem(
publication_id=int(row[0]),
title=str(row[1] or ""),
doi=row[2],
pdf_url=row[3],
status=str(row[4] or PDF_STATUS_UNTRACKED),
attempt_count=int(row[5] or 0),
last_failure_reason=row[6],
last_failure_detail=row[7],
last_source=row[8],
requested_by_user_id=int(row[9]) if row[9] is not None else None,
requested_by_email=row[10],
queued_at=row[11],
last_attempt_at=row[12],
resolved_at=row[13],
updated_at=row[14],
pdf_url=row[2],
status=str(row[3] or PDF_STATUS_UNTRACKED),
attempt_count=int(row[4] or 0),
last_failure_reason=row[5],
last_failure_detail=row[6],
last_source=row[7],
requested_by_user_id=int(row[8]) if row[8] is not None else None,
requested_by_email=row[9],
queued_at=row[10],
last_attempt_at=row[11],
resolved_at=row[12],
updated_at=row[13],
)
@ -902,3 +950,25 @@ async def list_pdf_queue_page(
limit=bounded_limit,
offset=bounded_offset,
)
async def drain_ready_jobs(
db_session: AsyncSession,
*,
limit: int,
max_attempts: int,
) -> int:
result = await db_session.execute(
select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1)
)
system_user_id = result.scalar_one_or_none()
if system_user_id is None:
return 0
bulk_result = await enqueue_all_missing_pdf_jobs(
db_session,
user_id=system_user_id,
request_email=settings.unpaywall_email,
limit=limit,
)
return bulk_result.queued_count

201
app/services/domains/publications/pdf_resolution_pipeline.py
View file

@ -2,100 +2,114 @@ from __future__ import annotations
from dataclasses import dataclass
import logging
from urllib.parse import urlparse
import httpx
from typing import Any
from app.services.domains.arxiv.application import ArxivRateLimitError
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.scholar.publication_pdf import (
ScholarPublicationLinkCandidate,
ScholarPublicationLinkCandidates,
fetch_link_candidates_from_scholar_publication_page,
)
from app.services.domains.unpaywall import pdf_discovery as pdf_discovery_service
from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
from app.settings import settings
logger = logging.getLogger(__name__)
PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE = "scholar_publication_page"
PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED = "scholar_publication_page_unlabeled_fallback"
PDF_PATH_TOKEN = "/pdf/"
HTTP_TIMEOUT_FLOOR_SECONDS = 0.5
@dataclass(frozen=True)
class PipelineOutcome:
outcome: OaResolutionOutcome | None
scholar_candidates: ScholarPublicationLinkCandidates | None
scholar_candidates: Any | None # Kept for backward compatibility with calling signatures
async def resolve_publication_pdf_outcome_for_row(
*,
row: PublicationListItem,
request_email: str | None,
openalex_api_key: str | None = None,
) -> PipelineOutcome:
candidates = await _safe_scholar_candidates(row.pub_url)
labeled = _labeled_candidate(candidates)
if labeled is not None:
return PipelineOutcome(_scholar_outcome(row=row, candidate=labeled), candidates)
# 1. OpenAlex OA — raises OpenAlexBudgetExhaustedError if budget is gone
openalex_outcome = await _openalex_outcome(row, request_email=request_email, openalex_api_key=openalex_api_key)
if openalex_outcome and openalex_outcome.pdf_url:
return PipelineOutcome(openalex_outcome, None)
# 2. arXiv
arxiv_outcome = await _arxiv_outcome(row, request_email=request_email)
if arxiv_outcome and arxiv_outcome.pdf_url:
return PipelineOutcome(arxiv_outcome, None)
# 3. Unpaywall (which falls back to Crossref)
oa_outcome = await _oa_outcome(row=row, request_email=request_email)
if _oa_has_pdf(oa_outcome):
return PipelineOutcome(oa_outcome, candidates)
unlabeled = _unlabeled_candidate(candidates)
if unlabeled is None:
return PipelineOutcome(oa_outcome, candidates)
fallback_outcome = await _unlabeled_fallback_outcome(row=row, candidate=unlabeled)
if fallback_outcome is not None:
return PipelineOutcome(fallback_outcome, candidates)
return PipelineOutcome(oa_outcome, candidates)
return PipelineOutcome(oa_outcome, None)
async def _safe_scholar_candidates(pub_url: str | None) -> ScholarPublicationLinkCandidates | None:
try:
return await fetch_link_candidates_from_scholar_publication_page(pub_url)
except Exception as exc: # pragma: no cover - defensive boundary
logger.warning(
"publications.pdf_resolution.scholar_candidates_failed",
extra={"event": "publications.pdf_resolution.scholar_candidates_failed", "error": str(exc)},
)
return None
def _labeled_candidate(
candidates: ScholarPublicationLinkCandidates | None,
) -> ScholarPublicationLinkCandidate | None:
if candidates is None:
return None
return candidates.labeled_candidate
def _unlabeled_candidate(
candidates: ScholarPublicationLinkCandidates | None,
) -> ScholarPublicationLinkCandidate | None:
if candidates is None:
return None
return candidates.fallback_candidate
def _scholar_outcome(
*,
async def _openalex_outcome(
row: PublicationListItem,
candidate: ScholarPublicationLinkCandidate,
) -> OaResolutionOutcome:
source = (
PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE
if candidate.label_present
else PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED
)
return OaResolutionOutcome(
publication_id=row.publication_id,
doi=row.doi,
pdf_url=candidate.url,
failure_reason=None,
source=source,
used_crossref=False,
)
request_email: str | None,
openalex_api_key: str | None = None,
) -> OaResolutionOutcome | None:
from app.services.domains.openalex.client import OpenAlexClient
from app.services.domains.openalex.matching import find_best_match
if not row.title:
return None
import re
safe_title = re.sub(r"[^\w\s]", " ", row.title)
safe_title = " ".join(safe_title.split())
if not safe_title:
return None
api_key = openalex_api_key or settings.openalex_api_key
client = OpenAlexClient(api_key=api_key, mailto=request_email or settings.crossref_api_mailto)
try:
openalex_works = await client.get_works_by_filter({"title.search": safe_title}, limit=5)
match = find_best_match(
target_title=row.title,
target_year=row.year,
target_authors=row.scholar_label,
candidates=openalex_works,
)
if match and match.oa_url:
return OaResolutionOutcome(
publication_id=row.publication_id,
doi=match.doi,
pdf_url=match.oa_url,
failure_reason=None,
source="openalex",
used_crossref=False,
)
except OpenAlexBudgetExhaustedError:
# Re-raise so the caller's batch loop can stop hitting the API.
raise
except Exception as exc:
logger.warning(
"publications.pdf_resolution.openalex_failed",
extra={"event": "publications.pdf_resolution.openalex_failed", "error": str(exc)},
)
return None
async def _arxiv_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None:
from app.services.domains.arxiv.application import discover_arxiv_id_for_publication
try:
arxiv_id = await discover_arxiv_id_for_publication(item=row, request_email=request_email)
if arxiv_id:
pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
return OaResolutionOutcome(
publication_id=row.publication_id,
doi=None,
pdf_url=pdf_url,
failure_reason=None,
source="arxiv",
used_crossref=False,
)
except ArxivRateLimitError:
raise # propagate so the batch loop can stop
except Exception as exc:
logger.warning(
"publications.pdf_resolution.arxiv_failed",
extra={"event": "publications.pdf_resolution.arxiv_failed", "error": str(exc)},
)
return None
async def _oa_outcome(
@ -105,46 +119,3 @@ async def _oa_outcome(
) -> OaResolutionOutcome | None:
outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email)
return outcomes.get(row.publication_id)
def _oa_has_pdf(outcome: OaResolutionOutcome | None) -> bool:
return bool(outcome and outcome.pdf_url)
async def _unlabeled_fallback_outcome(
*,
row: PublicationListItem,
candidate: ScholarPublicationLinkCandidate,
) -> OaResolutionOutcome | None:
pdf_url = await _validated_pdf_url(candidate.url)
if pdf_url is None:
return None
return _scholar_outcome(row=row, candidate=ScholarPublicationLinkCandidate(
url=pdf_url,
confidence_score=candidate.confidence_score,
label_present=False,
reason=candidate.reason,
))
async def _validated_pdf_url(candidate_url: str) -> str | None:
if _looks_direct_pdf(candidate_url):
return candidate_url
timeout_seconds = _discovery_timeout_seconds()
async with httpx.AsyncClient(timeout=timeout_seconds) as client:
if await pdf_discovery_service._candidate_is_pdf(client, candidate_url=candidate_url):
return candidate_url
return await pdf_discovery_service.resolve_pdf_from_landing_page(client, page_url=candidate_url)
def _looks_direct_pdf(url: str | None) -> bool:
if pdf_discovery_service.looks_like_pdf_url(url):
return True
if not isinstance(url, str):
return False
path = (urlparse(url).path or "").lower()
return PDF_PATH_TOKEN in path
def _discovery_timeout_seconds() -> float:
return max(float(settings.unpaywall_timeout_seconds), HTTP_TIMEOUT_FLOOR_SECONDS)

56
app/services/domains/publications/queries.py
View file

@ -1,5 +1,7 @@
from __future__ import annotations
from datetime import datetime
from sqlalchemy import Select, func, select
from sqlalchemy.ext.asyncio import AsyncSession
@ -15,15 +17,24 @@ def _normalized_citation_count(value: object) -> int:
return 0
async def get_latest_completed_run_id_for_user(
async def get_latest_run_id_for_user(
db_session: AsyncSession,
*,
user_id: int,
) -> int | None:
# We include RUNNING and RESOLVING statuses so that the "New" tab shows
# results in real-time as they are discovered.
result = await db_session.execute(
select(func.max(CrawlRun.id)).where(
CrawlRun.user_id == user_id,
CrawlRun.status != RunStatus.RUNNING,
CrawlRun.status.in_(
[
RunStatus.RUNNING,
RunStatus.RESOLVING,
RunStatus.SUCCESS,
RunStatus.PARTIAL_FAILURE,
]
),
)
)
latest_run_id = result.scalar_one_or_none()
@ -39,7 +50,19 @@ def publications_query(
favorite_only: bool,
limit: int,
offset: int = 0,
search: str | None = None,
sort_by: str = "first_seen",
sort_dir: str = "desc",
snapshot_before: datetime | None = None,
) -> Select[tuple]:
_SORT_COLUMNS = {
"first_seen": ScholarPublication.created_at,
"title": Publication.title_raw,
"year": Publication.year,
"citations": Publication.citation_count,
"scholar": ScholarProfile.display_name,
}
scholar_label = ScholarProfile.display_name
stmt = (
select(
@ -52,7 +75,6 @@ def publications_query(
Publication.citation_count,
Publication.venue_text,
Publication.pub_url,
Publication.doi,
Publication.pdf_url,
ScholarPublication.is_read,
ScholarPublication.is_favorite,
@ -62,10 +84,15 @@ def publications_query(
.join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
.where(ScholarProfile.user_id == user_id)
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
.offset(max(int(offset), 0))
.limit(limit)
)
if search:
safe_search = search.replace("%", r"\%").replace("_", r"\_")
pat = f"%{safe_search}%"
stmt = stmt.where(
Publication.title_raw.ilike(pat)
| ScholarProfile.display_name.ilike(pat)
| Publication.venue_text.ilike(pat)
)
if scholar_profile_id is not None:
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
if favorite_only:
@ -76,6 +103,16 @@ def publications_query(
if latest_run_id is None:
return stmt.where(False)
stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
if snapshot_before is not None:
stmt = stmt.where(ScholarPublication.created_at <= snapshot_before)
sort_col = _SORT_COLUMNS.get(sort_by, ScholarPublication.created_at)
order = sort_col.desc() if sort_dir == "desc" else sort_col.asc()
stmt = stmt.order_by(order, Publication.id.desc())
if limit is not None:
stmt = stmt.offset(max(int(offset), 0)).limit(limit)
return stmt
@ -96,7 +133,6 @@ def publication_query_for_user(
Publication.citation_count,
Publication.venue_text,
Publication.pub_url,
Publication.doi,
Publication.pdf_url,
ScholarPublication.is_read,
ScholarPublication.is_favorite,
@ -121,7 +157,7 @@ async def get_publication_item_for_user(
scholar_profile_id: int,
publication_id: int,
) -> PublicationListItem | None:
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
latest_run_id = await get_latest_run_id_for_user(db_session, user_id=user_id)
result = await db_session.execute(
publication_query_for_user(
user_id=user_id,
@ -150,7 +186,6 @@ def publication_list_item_from_row(
citation_count,
venue_text,
pub_url,
doi,
pdf_url,
is_read,
is_favorite,
@ -166,7 +201,6 @@ def publication_list_item_from_row(
citation_count=_normalized_citation_count(citation_count),
venue_text=venue_text,
pub_url=pub_url,
doi=doi,
pdf_url=pdf_url,
is_read=bool(is_read),
is_favorite=bool(is_favorite),
@ -188,7 +222,6 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
citation_count,
venue_text,
pub_url,
doi,
pdf_url,
_is_read,
_is_favorite,
@ -204,6 +237,5 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
citation_count=_normalized_citation_count(citation_count),
venue_text=venue_text,
pub_url=pub_url,
doi=doi,
pdf_url=pdf_url,
)

2
app/services/domains/publications/types.py
View file

@ -16,7 +16,6 @@ class PublicationListItem:
citation_count: int
venue_text: str | None
pub_url: str | None
doi: str | None
pdf_url: str | None
is_read: bool
first_seen_at: datetime
@ -39,5 +38,4 @@ class UnreadPublicationItem:
citation_count: int
venue_text: str | None
pub_url: str | None
doi: str | None
pdf_url: str | None