justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

temp commit

Browse source
This commit is contained in:
Justin Visser 2026-02-26 12:54:19 +01:00
parent 8760f27b51
commit 0e9e49df16
193 changed files with 23228 additions and 935 deletions
Download patch file Download diff file Expand all files Collapse all files

110
app/services/domains/arxiv/application.py Normal file
View file

@ -0,0 +1,110 @@
from __future__ import annotations
import asyncio
import logging
from typing import TYPE_CHECKING
import xml.etree.ElementTree as ET
import httpx
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
from app.settings import settings
if TYPE_CHECKING:
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
logger = logging.getLogger(__name__)
# arXiv API terms: max 1 request per 3 seconds, single connection at a time.
_ARXIV_LOCK = asyncio.Lock()
_ARXIV_MIN_INTERVAL_SECONDS = 4.0
# Global cooldown: when arXiv returns 429, all batches back off for this long.
_ARXIV_RATE_LIMIT_COOLDOWN_SECONDS = 60.0
_arxiv_rate_limited_until: float = 0.0 # asyncio monotonic time
class ArxivRateLimitError(Exception):
"""arXiv returned 429 — stop the batch to avoid hammering."""
pass
def _build_arxiv_query(title: str, author_surname: str | None) -> str | None:
parts = []
if title:
clean_title = title.replace('"', '').replace("'", "")
parts.append(f'ti:"{clean_title}"')
if author_surname:
parts.append(f'au:"{author_surname}"')
if not parts:
return None
return " AND ".join(parts)
async def discover_arxiv_id_for_publication(
*,
item: PublicationListItem | UnreadPublicationItem,
request_email: str | None = None,
timeout_seconds: float = 3.0,
) -> str | None:
title = (item.title or "").strip()
if not title:
return None
author_surname = None
if item.scholar_label:
tokens = [t for t in item.scholar_label.strip().split() if t]
if tokens:
author_surname = tokens[-1].lower()
query = _build_arxiv_query(title, author_surname)
if not query:
return None
url = "https://export.arxiv.org/api/query"
params = {"search_query": query, "start": 0, "max_results": 3}
headers = {
"User-Agent": (
f"scholar-scraper/1.0 "
f"(mailto:{request_email or settings.crossref_api_mailto or 'unknown@example.com'})"
)
}
try:
async with _ARXIV_LOCK:
global _arxiv_rate_limited_until
now = asyncio.get_running_loop().time()
if now < _arxiv_rate_limited_until:
remaining = _arxiv_rate_limited_until - now
raise ArxivRateLimitError(f"arXiv global cooldown active ({remaining:.0f}s remaining)")
async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True, headers=headers) as client:
response = await client.get(url, params=params)
if response.status_code == 429:
_arxiv_rate_limited_until = asyncio.get_running_loop().time() + _ARXIV_RATE_LIMIT_COOLDOWN_SECONDS
raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
await asyncio.sleep(_ARXIV_MIN_INTERVAL_SECONDS)
response.raise_for_status()
root = ET.fromstring(response.text)
namespace = {"atom": "http://www.w3.org/2005/Atom"}
for entry in root.findall("atom:entry", namespace):
id_elem = entry.find("atom:id", namespace)
if id_elem is not None and id_elem.text:
candidate = str(id_elem.text)
if "/abs/" in candidate:
candidate = candidate.split("/abs/")[-1]
normalized = normalize_arxiv_id(candidate)
if normalized:
logger.debug("arxiv.id_discovered", extra={"event": "arxiv.id_discovered", "arxiv_id": normalized})
return normalized
except ArxivRateLimitError:
raise # propagate so the batch loop can stop
except Exception as exc:
logger.debug(f"Failed to query arXiv API: {exc}")
return None

862
app/services/domains/ingestion/application.py
View file

File diff suppressed because it is too large Load diff

141
app/services/domains/ingestion/fingerprints.py
View file

@ -15,12 +15,59 @@ from app.services.domains.ingestion.constants import (
)
from app.services.domains.scholar.parser import ParseState, ParsedProfilePage, PublicationCandidate
# Scholar-specific noise patterns stripped before canonical comparison.
# Applied in order; each targets a different Scholar metadata injection style.
_NOISE_DOI_RE = re.compile(r"[,.\s]+doi\s*:\s*\S+.*$", re.IGNORECASE)
_NOISE_ARXIV_RE = re.compile(r"[,.\s]+arxiv\b.*$", re.IGNORECASE)
_NOISE_PREPRINT_RE = re.compile(
r"[,\s]+(?:preprint|extended\s+version|technical\s+report|working\s+paper)\b.*$",
re.IGNORECASE,
)
_NOISE_TRAILING_YEAR_RE = re.compile(r"\s*[,(]\s*\d{4}\s*[),]?\s*$")
# Strips ". Capitalised sentence" appended as venue: ". Comput. Sci…", ". Journal of…"
_NOISE_VENUE_SENTENCE_RE = re.compile(r"(?<=\w{3})\.\s+[A-Z][a-z].*$")
_CANONICAL_DEDUP_THRESHOLD = 0.82
def normalize_title(value: str) -> str:
lowered = value.lower()
return TITLE_ALNUM_RE.sub("", lowered)
def canonical_title_for_dedup(title: str) -> str:
"""Strip Scholar-specific noise suffixes then normalize for dedup comparison."""
t = title.strip()
t = _NOISE_DOI_RE.sub("", t)
t = _NOISE_ARXIV_RE.sub("", t)
t = _NOISE_PREPRINT_RE.sub("", t)
t = _NOISE_TRAILING_YEAR_RE.sub("", t)
t = _NOISE_VENUE_SENTENCE_RE.sub("", t)
return normalize_title(t.strip())
def _stripped_title_for_canonical(title: str) -> str:
"""Apply noise-stripping and lowercase but PRESERVE spaces (for later tokenization)."""
t = title.strip()
t = _NOISE_DOI_RE.sub("", t)
t = _NOISE_ARXIV_RE.sub("", t)
t = _NOISE_PREPRINT_RE.sub("", t)
t = _NOISE_TRAILING_YEAR_RE.sub("", t)
t = _NOISE_VENUE_SENTENCE_RE.sub("", t)
return t.lower().strip()
def _canonical_title_tokens(title: str) -> set[str]:
"""Word tokens of the noise-stripped title (preserves token boundaries)."""
return set(WORD_RE.findall(_stripped_title_for_canonical(title)))
def _jaccard(a: set[str], b: set[str]) -> float:
if not a or not b:
return 0.0
return len(a & b) / len(a | b)
def _first_author_last_name(authors_text: str | None) -> str:
if not authors_text:
return ""
@ -100,31 +147,91 @@ def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int:
return int(fallback)
def _title_tokens(value: str) -> set[str]:
"""Extract normalized word tokens for fuzzy title comparison."""
return set(WORD_RE.findall(value.lower()))
def fuzzy_titles_match(
title_a: str,
title_b: str,
*,
threshold: float = 0.85,
) -> bool:
"""Return True if two titles are near-duplicates by token-level Jaccard similarity.
A threshold of 0.85 catches common academic duplicate patterns:
differences in punctuation, minor word variations, subtitle changes.
"""
tokens_a = _title_tokens(title_a)
tokens_b = _title_tokens(title_b)
return _jaccard(tokens_a, tokens_b) >= threshold
def _dedupe_publication_candidates(
publications: list[PublicationCandidate],
*,
seen_canonical: set[str] | None = None,
) -> list[PublicationCandidate]:
"""Deduplicate candidates using canonical title matching.
Args:
publications: candidates to filter
seen_canonical: optional mutable set shared across pages. Stores the
noise-stripped *lowercased* (but space-preserved) canonical string
so it can be tokenized on the next page for cross-page fuzzy dedup.
Accepted canonicals are added; existing entries are consulted.
"""
deduped: list[PublicationCandidate] = []
seen: set[str] = set()
for publication in publications:
if publication.cluster_id:
identity = f"cluster:{publication.cluster_id}"
else:
identity = "|".join(
[
"fallback",
normalize_title(publication.title),
str(publication.year) if publication.year is not None else "",
publication.authors_text or "",
publication.venue_text or "",
]
)
if identity in seen:
seen_exact: set[str] = set()
# Token sets for fuzzy comparison; seeded from cross-page state.
seen_tokens: list[set[str]] = []
if seen_canonical:
for stripped in seen_canonical:
seen_tokens.append(set(WORD_RE.findall(stripped)))
for pub in publications:
identity = _publication_identity(pub)
if identity in seen_exact:
continue
seen.add(identity)
deduped.append(publication)
# Use space-preserving stripped form for token-level fuzzy match.
tokens = _canonical_title_tokens(pub.title)
if _is_fuzzy_dup(tokens, seen_tokens):
continue
seen_exact.add(identity)
seen_tokens.append(tokens)
if seen_canonical is not None:
# Store the noise-stripped lowercased (space-preserved) form.
seen_canonical.add(_stripped_title_for_canonical(pub.title))
deduped.append(pub)
return deduped
def _publication_identity(pub: PublicationCandidate) -> str:
if pub.cluster_id:
return f"cluster:{pub.cluster_id}"
canonical = canonical_title_for_dedup(pub.title)
return "|".join(
[
"fallback",
canonical,
str(pub.year) if pub.year is not None else "",
_first_author_last_name(pub.authors_text),
]
)
def _is_fuzzy_dup(tokens: set[str], seen: list[set[str]]) -> bool:
for existing in seen:
if _jaccard(tokens, existing) >= _CANONICAL_DEDUP_THRESHOLD:
return True
return False
def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None:
if not body:
return None

26
app/services/domains/ingestion/scheduler.py
View file

@ -146,6 +146,9 @@ class SchedulerService:
async def _tick_once(self) -> None:
if self._continuation_queue_enabled:
await self._drain_continuation_queue()
await self._drain_pdf_queue()
candidates = await self._load_candidates()
if not candidates:
return
@ -488,6 +491,8 @@ class SchedulerService:
request_delay_seconds=request_delay_seconds,
network_error_retries=self._network_error_retries,
retry_backoff_seconds=self._retry_backoff_seconds,
rate_limit_retries=settings.ingestion_rate_limit_retries,
rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds,
max_pages_per_scholar=self._max_pages_per_scholar,
page_size=self._page_size,
scholar_profile_ids={job.scholar_profile_id},
@ -583,6 +588,27 @@ class SchedulerService:
return
await self._finalize_queue_job_after_run(job, run_summary)
async def _drain_pdf_queue(self) -> None:
from app.services.domains.publications.pdf_queue import drain_ready_jobs
session_factory = get_session_factory()
async with session_factory() as session:
try:
processed = await drain_ready_jobs(
session,
limit=settings.scheduler_pdf_queue_batch_size,
max_attempts=settings.pdf_auto_retry_max_attempts,
)
if processed > 0:
logger.info("scheduler.pdf_queue_drain_completed", extra={
"event": "scheduler.pdf_queue_drain_completed",
"processed_count": processed,
})
except Exception:
logger.exception("scheduler.pdf_queue_drain_failed", extra={
"event": "scheduler.pdf_queue_drain_failed",
})
async def _load_request_delay_for_user(
self,
db_session: AsyncSession,

2
app/services/domains/ingestion/types.py
View file

@ -35,6 +35,7 @@ class PagedParseResult:
pagination_truncated_reason: str | None
continuation_cstart: int | None
skipped_no_change: bool
discovered_publication_count: int
@dataclass
@ -88,6 +89,7 @@ class PagedLoopState:
has_more_remaining: bool = False
pagination_truncated_reason: str | None = None
continuation_cstart: int | None = None
discovered_publication_count: int = 0
class RunAlreadyInProgressError(RuntimeError):

5
app/services/domains/openalex/__init__.py Normal file
View file

@ -0,0 +1,5 @@
from __future__ import annotations
import logging
logger = logging.getLogger(__name__)

151
app/services/domains/openalex/client.py Normal file
View file

@ -0,0 +1,151 @@
import asyncio
import logging
from typing import Any, Mapping
import httpx
from tenacity import (
retry,
retry_if_exception_type,
stop_after_attempt,
wait_exponential,
)
from app.services.domains.openalex.types import OpenAlexWork
logger = logging.getLogger(__name__)
OPENALEX_BASE_URL = "https://api.openalex.org"
class OpenAlexClientError(Exception):
pass
class OpenAlexRateLimitError(OpenAlexClientError):
"""Transient rate limit (too many requests per second)."""
pass
class OpenAlexBudgetExhaustedError(OpenAlexClientError):
"""Daily API budget exhausted — retrying is futile until midnight UTC."""
pass
class OpenAlexClient:
def __init__(
self,
api_key: str | None = None,
mailto: str | None = None,
timeout: float = 10.0,
) -> None:
self.api_key = api_key
self.mailto = mailto
self.timeout = timeout
@property
def _base_params(self) -> dict[str, str]:
params = {}
if self.mailto:
params["mailto"] = self.mailto
if self.api_key:
params["api_key"] = self.api_key
return params
@retry(
retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException)),
stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1, min=2, max=10),
reraise=True,
)
async def get_work_by_doi(self, doi: str) -> OpenAlexWork | None:
"""Fetch a single work by DOI directly."""
clean_doi = doi.replace("https://doi.org/", "")
if not clean_doi:
return None
url = f"{OPENALEX_BASE_URL}/works/{clean_doi}"
headers = {}
if self.mailto:
headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
else:
headers["User-Agent"] = "scholar-scraper/1.0"
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client:
response = await client.get(url, params=self._base_params)
if response.status_code == 404:
return None
if response.status_code == 429:
remaining = response.headers.get("X-RateLimit-Remaining-USD", "")
if remaining == "0" or remaining.startswith("-"):
raise OpenAlexBudgetExhaustedError(
"Daily API budget exhausted; retrying won't help until midnight UTC"
)
raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI")
if response.status_code >= 400:
logger.warning("OpenAlex API error: %s %s", response.status_code, response.text)
raise OpenAlexClientError(f"API Error {response.status_code}")
data = response.json()
return OpenAlexWork.from_api_dict(data)
@retry(
retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException)),
stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1, min=2, max=10),
reraise=True,
)
async def get_works_by_filter(
self,
filters: dict[str, str],
limit: int = 50,
) -> list[OpenAlexWork]:
"""
Fetch works using the ?filter= query parameter.
Supports fetching multiple records by joining filters with | (OR logic).
"""
if not filters:
return []
# Example: {"doi": "10.foo|10.bar", "title.search": "query"}
filter_str = ",".join(f"{k}:{v}" for k, v in filters.items())
params = self._base_params.copy()
params["filter"] = filter_str
params["per-page"] = str(limit)
url = f"{OPENALEX_BASE_URL}/works"
headers = {}
if self.mailto:
headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
else:
headers["User-Agent"] = "scholar-scraper/1.0"
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client:
response = await client.get(url, params=params)
if response.status_code == 429:
remaining = response.headers.get("X-RateLimit-Remaining-USD", "")
if remaining == "0" or remaining.startswith("-"):
raise OpenAlexBudgetExhaustedError(
"Daily API budget exhausted; retrying won't help until midnight UTC"
)
raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list")
if response.status_code >= 400:
logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text)
raise OpenAlexClientError(f"API Error {response.status_code}")
data = response.json()
results = data.get("results") or []
parsed_works = []
for raw_work in results:
try:
parsed_works.append(OpenAlexWork.from_api_dict(raw_work))
except Exception as e:
logger.warning("Failed to parse OpenAlex raw dict: %s", e)
continue
return parsed_works

108
app/services/domains/openalex/matching.py Normal file
View file

@ -0,0 +1,108 @@
import logging
import re
from rapidfuzz import fuzz
from app.services.domains.openalex.types import OpenAlexWork
logger = logging.getLogger(__name__)
# A minimum similarity score out of 100 for a title to be considered a match candidate.
TITLE_MATCH_THRESHOLD = 90.0
# The margin within the top score where a secondary tiebreaker (author/year) is necessary.
TIEBREAKER_MARGIN = 5.0
def _clean_string(s: str | None) -> str:
if not s:
return ""
# Strip non-alphanumeric (keep spaces), lowercase, and collapse whitespace
cleaned = re.sub(r"[^a-z0-9\s]", " ", s.lower())
return " ".join(cleaned.split())
def _author_overlap_score(target_authors: str | None, candidate_authors: list[str]) -> bool:
if not target_authors or not candidate_authors:
return False
target_clean = _clean_string(target_authors)
if not target_clean:
return False
for candidate in candidate_authors:
cand_clean = _clean_string(candidate)
if cand_clean and (cand_clean in target_clean or target_clean in cand_clean):
return True
# Alternatively check rapidfuzz token_set_ratio
if cand_clean and fuzz.token_set_ratio(target_clean, cand_clean) > 80:
return True
return False
def find_best_match(
target_title: str,
target_year: int | None,
target_authors: str | None,
candidates: list[OpenAlexWork],
) -> OpenAlexWork | None:
"""
Finds the best matching OpenAlexWork from a list of candidates, prioritizing title similarity (>90%)
with year and author overlap as tiebreakers for close candidates.
"""
if not target_title or not candidates:
return None
clean_target = _clean_string(target_title)
if not clean_target:
return None
scored_candidates: list[tuple[float, OpenAlexWork]] = []
for cand in candidates:
if not cand.title:
continue
clean_cand = _clean_string(cand.title)
# Primary sort: string similarity ratio
score = fuzz.ratio(clean_target, clean_cand)
if score >= TITLE_MATCH_THRESHOLD:
scored_candidates.append((score, cand))
if not scored_candidates:
return None
# Sort descending by score
scored_candidates.sort(key=lambda x: x[0], reverse=True)
best_score = scored_candidates[0][0]
# Extract all candidates within the tiebreaker margin
top_scored_candidates = [
(score, cand) for score, cand in scored_candidates
if best_score - score <= TIEBREAKER_MARGIN
]
if len(top_scored_candidates) == 1:
return top_scored_candidates[0][1]
# We have a tie or near-tie. Use year and author overlap to break the tie.
# Score candidates: +1 for year match (within 1 year), +1 for author overlap
tiebreaker_scores: list[tuple[int, float, OpenAlexWork]] = []
for original_score, cand in top_scored_candidates:
tb_score = 0
if target_year is not None and cand.publication_year is not None:
if abs(target_year - cand.publication_year) <= 1:
tb_score += 1
candidate_author_names = [a.display_name for a in cand.authors if a.display_name]
if _author_overlap_score(target_authors, candidate_author_names):
tb_score += 1
tiebreaker_scores.append((tb_score, original_score, cand))
tiebreaker_scores.sort(key=lambda x: (x[0], x[1]), reverse=True)
return tiebreaker_scores[0][2]

71
app/services/domains/openalex/types.py Normal file
View file

@ -0,0 +1,71 @@
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any, Mapping
@dataclass(frozen=True)
class OpenAlexAuthor:
openalex_id: str | None
display_name: str | None
@dataclass(frozen=True)
class OpenAlexWork:
openalex_id: str
doi: str | None
pmid: str | None
pmcid: str | None
title: str | None
publication_year: int | None
cited_by_count: int
is_oa: bool
oa_url: str | None
authors: list[OpenAlexAuthor] = field(default_factory=list)
raw_data: Mapping[str, Any] = field(default_factory=dict, repr=False)
@classmethod
def from_api_dict(cls, data: Mapping[str, Any]) -> OpenAlexWork:
ids = data.get("ids") or {}
# Extract DOI without the https://doi.org/ prefix
doi = ids.get("doi")
if doi and doi.startswith("https://doi.org/"):
doi = doi[16:]
# Extract PMID without the url prefix
pmid = ids.get("pmid")
if pmid and pmid.startswith("https://pubmed.ncbi.nlm.nih.gov/"):
pmid = pmid[32:]
# Extract PMCID without the url prefix
pmcid = ids.get("pmcid")
if pmcid and pmcid.startswith("https://www.ncbi.nlm.nih.gov/pmc/articles/"):
pmcid = pmcid[42:]
open_access = data.get("open_access") or {}
authors = []
for authorship in data.get("authorships") or []:
author_data = authorship.get("author") or {}
authors.append(
OpenAlexAuthor(
openalex_id=author_data.get("id"),
display_name=author_data.get("display_name"),
)
)
return cls(
openalex_id=data.get("id", ""),
doi=doi,
pmid=pmid,
pmcid=pmcid,
title=data.get("title"),
publication_year=data.get("publication_year"),
cited_by_count=data.get("cited_by_count", 0),
is_oa=bool(open_access.get("is_oa")),
oa_url=open_access.get("oa_url"),
authors=authors,
raw_data=dict(data),
)

3
app/services/domains/portability/exporting.py
View file

@ -34,7 +34,6 @@ def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]:
author_text,
venue_text,
pub_url,
doi,
pdf_url,
is_read,
) = row
@ -48,7 +47,6 @@ def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]:
"author_text": author_text,
"venue_text": venue_text,
"pub_url": pub_url,
"doi": doi,
"pdf_url": pdf_url,
"is_read": bool(is_read),
}
@ -75,7 +73,6 @@ async def export_user_data(
Publication.author_text,
Publication.venue_text,
Publication.pub_url,
Publication.doi,
Publication.pdf_url,
ScholarPublication.is_read,
)

8
app/services/domains/portability/publication_import.py
View file

@ -71,7 +71,6 @@ def _apply_imported_publication_values(
author_text: str | None,
venue_text: str | None,
pub_url: str | None,
doi: str | None,
pdf_url: str | None,
cluster_id: str | None,
) -> bool:
@ -98,9 +97,6 @@ def _apply_imported_publication_values(
if pub_url and publication.pub_url != pub_url:
publication.pub_url = pub_url
updated = True
if doi and publication.doi != doi:
publication.doi = doi
updated = True
if pdf_url and publication.pdf_url != pdf_url:
publication.pdf_url = pdf_url
updated = True
@ -117,7 +113,6 @@ def _new_publication(
author_text: str | None,
venue_text: str | None,
pub_url: str | None,
doi: str | None,
pdf_url: str | None,
) -> Publication:
return Publication(
@ -130,7 +125,6 @@ def _new_publication(
author_text=author_text,
venue_text=venue_text,
pub_url=pub_url,
doi=doi,
pdf_url=pdf_url,
)
@ -285,7 +279,6 @@ async def _create_import_publication(
author_text=payload.author_text,
venue_text=payload.venue_text,
pub_url=payload.pub_url,
doi=payload.doi,
pdf_url=payload.pdf_url,
)
db_session.add(publication)
@ -306,7 +299,6 @@ def _update_import_publication(
author_text=payload.author_text,
venue_text=payload.venue_text,
pub_url=payload.pub_url,
doi=payload.doi,
pdf_url=payload.pdf_url,
cluster_id=payload.cluster_id,
)

90
app/services/domains/publication_identifiers/application.py
View file

@ -21,7 +21,7 @@ from app.services.domains.publication_identifiers.types import (
if TYPE_CHECKING:
from app.services.domains.publications.pdf_queue import PdfQueueListItem
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
CONFIDENCE_HIGH = 0.98
CONFIDENCE_MEDIUM = 0.9
@ -117,9 +117,76 @@ async def sync_identifiers_for_publication_fields(
await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=candidates)
async def discover_and_sync_identifiers_for_publication(
db_session: AsyncSession,
*,
publication: Publication,
scholar_label: str,
) -> None:
await sync_identifiers_for_publication_fields(db_session, publication=publication)
existing_doi = await _existing_identifier_by_kind(
db_session,
publication_id=int(publication.id),
kind=IdentifierKind.DOI.value,
)
if existing_doi is not None:
return
from app.services.domains.crossref import application as crossref_service
from app.services.domains.publications.types import UnreadPublicationItem
item = UnreadPublicationItem(
publication_id=int(publication.id),
scholar_profile_id=0,
scholar_label=scholar_label,
title=str(publication.title_raw or ""),
year=publication.year,
citation_count=publication.citation_count,
venue_text=publication.venue_text,
pub_url=publication.pub_url,
pdf_url=publication.pdf_url,
)
discovered_doi = await crossref_service.discover_doi_for_publication(item=item)
if discovered_doi:
normalized_doi = normalize_doi(discovered_doi)
if normalized_doi:
candidate = _candidate(
IdentifierKind.DOI,
discovered_doi,
normalized_doi,
"crossref_api",
CONFIDENCE_MEDIUM,
None,
)
await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate)
existing_arxiv = await _existing_identifier_by_kind(
db_session,
publication_id=int(publication.id),
kind=IdentifierKind.ARXIV.value,
)
if existing_arxiv is None:
from app.services.domains.arxiv import application as arxiv_service
discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item)
if discovered_arxiv:
normalized_arxiv = normalize_arxiv_id(discovered_arxiv)
if normalized_arxiv:
candidate = _candidate(
IdentifierKind.ARXIV,
discovered_arxiv,
normalized_arxiv,
"arxiv_api",
CONFIDENCE_MEDIUM,
None,
)
await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate)
def _publication_field_candidates(publication: Publication) -> list[IdentifierCandidate]:
return _fallback_candidates_from_values(
doi=publication.doi,
doi=None,
pub_url=publication.pub_url,
pdf_url=publication.pdf_url,
)
@ -194,6 +261,21 @@ async def _existing_identifier(
return result.scalar_one_or_none()
async def _existing_identifier_by_kind(
db_session: AsyncSession,
*,
publication_id: int,
kind: str,
) -> PublicationIdentifier | None:
result = await db_session.execute(
select(PublicationIdentifier).where(
PublicationIdentifier.publication_id == publication_id,
PublicationIdentifier.kind == kind,
).order_by(PublicationIdentifier.confidence_score.desc()).limit(1)
)
return result.scalar_one_or_none()
def _new_identifier_row(
*,
publication_id: int,
@ -234,7 +316,7 @@ def _overlay_publication_item(
item: PublicationListItem,
display_identifier: DisplayIdentifier | None,
) -> PublicationListItem:
fallback = display_identifier or derive_display_identifier_from_values(doi=item.doi, pub_url=item.pub_url, pdf_url=item.pdf_url)
fallback = display_identifier or derive_display_identifier_from_values(doi=None, pub_url=item.pub_url, pdf_url=item.pdf_url)
return replace(item, display_identifier=fallback)
@ -253,7 +335,7 @@ def _overlay_queue_item(
item: PdfQueueListItem,
display_identifier: DisplayIdentifier | None,
) -> PdfQueueListItem:
fallback = display_identifier or derive_display_identifier_from_values(doi=item.doi, pdf_url=item.pdf_url)
fallback = display_identifier or derive_display_identifier_from_values(doi=None, pdf_url=item.pdf_url)
return replace(item, display_identifier=fallback)

2
app/services/domains/publication_identifiers/normalize.py
View file

@ -7,7 +7,7 @@ from app.services.domains.doi.normalize import normalize_doi
from app.services.domains.publication_identifiers.types import IdentifierKind
ARXIV_ABS_RE = re.compile(r"\barxiv:\s*([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?\b", re.I)
ARXIV_PATH_RE = re.compile(r"^/(?:abs|pdf)/([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?(?:\.pdf)?/?$", re.I)
ARXIV_PATH_RE = re.compile(r"^/(?:abs|pdf|html|ps|format)/([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?(?:\.pdf)?/?$", re.I)
PMCID_RE = re.compile(r"\b(PMC\d+)\b", re.I)
PUBMED_PATH_RE = re.compile(r"^/(\d+)/?$")

4
app/services/domains/publications/application.py
View file

@ -24,7 +24,7 @@ from app.services.domains.publications.modes import (
resolve_publication_view_mode,
)
from app.services.domains.publications.queries import (
get_latest_completed_run_id_for_user,
get_latest_run_id_for_user,
get_publication_item_for_user,
publications_query,
)
@ -50,7 +50,7 @@ __all__ = [
"PublicationListItem",
"UnreadPublicationItem",
"resolve_publication_view_mode",
"get_latest_completed_run_id_for_user",
"get_latest_run_id_for_user",
"publications_query",
"get_publication_item_for_user",
"list_for_user",

19
app/services/domains/publications/counts.py
View file

@ -1,6 +1,8 @@
from __future__ import annotations
from sqlalchemy import func, select
from datetime import datetime
from sqlalchemy import distinct, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import ScholarProfile, ScholarPublication
@ -10,7 +12,7 @@ from app.services.domains.publications.modes import (
MODE_UNREAD,
resolve_publication_view_mode,
)
from app.services.domains.publications.queries import get_latest_completed_run_id_for_user
from app.services.domains.publications.queries import get_latest_run_id_for_user
async def count_for_user(
@ -20,11 +22,12 @@ async def count_for_user(
mode: str = MODE_ALL,
scholar_profile_id: int | None = None,
favorite_only: bool = False,
snapshot_before: datetime | None = None,
) -> int:
resolved_mode = resolve_publication_view_mode(mode)
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
latest_run_id = await get_latest_run_id_for_user(db_session, user_id=user_id)
stmt = (
select(func.count())
select(func.count(distinct(ScholarPublication.publication_id)))
.select_from(ScholarPublication)
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
.where(ScholarProfile.user_id == user_id)
@ -33,6 +36,8 @@ async def count_for_user(
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
if favorite_only:
stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
if snapshot_before is not None:
stmt = stmt.where(ScholarPublication.created_at <= snapshot_before)
if resolved_mode == MODE_UNREAD:
stmt = stmt.where(ScholarPublication.is_read.is_(False))
if resolved_mode == MODE_LATEST:
@ -49,6 +54,7 @@ async def count_unread_for_user(
user_id: int,
scholar_profile_id: int | None = None,
favorite_only: bool = False,
snapshot_before: datetime | None = None,
) -> int:
return await count_for_user(
db_session,
@ -56,6 +62,7 @@ async def count_unread_for_user(
mode=MODE_UNREAD,
scholar_profile_id=scholar_profile_id,
favorite_only=favorite_only,
snapshot_before=snapshot_before,
)
@ -65,6 +72,7 @@ async def count_latest_for_user(
user_id: int,
scholar_profile_id: int | None = None,
favorite_only: bool = False,
snapshot_before: datetime | None = None,
) -> int:
return await count_for_user(
db_session,
@ -72,6 +80,7 @@ async def count_latest_for_user(
mode=MODE_LATEST,
scholar_profile_id=scholar_profile_id,
favorite_only=favorite_only,
snapshot_before=snapshot_before,
)
@ -80,6 +89,7 @@ async def count_favorite_for_user(
*,
user_id: int,
scholar_profile_id: int | None = None,
snapshot_before: datetime | None = None,
) -> int:
return await count_for_user(
db_session,
@ -87,4 +97,5 @@ async def count_favorite_for_user(
mode=MODE_ALL,
scholar_profile_id=scholar_profile_id,
favorite_only=True,
snapshot_before=snapshot_before,
)

103
app/services/domains/publications/dedup.py Normal file
View file

@ -0,0 +1,103 @@
from __future__ import annotations
import logging
from sqlalchemy import delete, select
from sqlalchemy.orm import aliased
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Publication, PublicationIdentifier, ScholarPublication
logger = logging.getLogger(__name__)
async def find_identifier_duplicate_pairs(
db_session: AsyncSession,
) -> list[tuple[int, int]]:
"""Return (winner_id, dup_id) pairs where two publications share the same identifier.
Winner is always the lower publication_id (earlier-created). Uses the existing
ix_publication_identifiers_kind_value index for the self-join.
"""
pi1 = aliased(PublicationIdentifier, name="pi1")
pi2 = aliased(PublicationIdentifier, name="pi2")
rows = await db_session.execute(
select(pi1.publication_id, pi2.publication_id)
.join(
pi2,
(pi1.kind == pi2.kind)
& (pi1.value_normalized == pi2.value_normalized)
& (pi1.publication_id < pi2.publication_id),
)
.distinct()
)
return [(winner_id, dup_id) for winner_id, dup_id in rows]
async def merge_duplicate_publication(
db_session: AsyncSession,
*,
winner_id: int,
dup_id: int,
) -> None:
"""Merge dup_id into winner_id: migrate scholar links, then delete the dup."""
await _migrate_scholar_links(db_session, winner_id=winner_id, dup_id=dup_id)
await db_session.execute(
delete(Publication).where(Publication.id == dup_id)
)
logger.info(
"publications.identifier_merge",
extra={
"event": "publications.identifier_merge",
"winner_id": winner_id,
"dup_id": dup_id,
},
)
async def _migrate_scholar_links(
db_session: AsyncSession,
*,
winner_id: int,
dup_id: int,
) -> None:
"""Move ScholarPublication links from dup to winner, dropping conflicts."""
dup_links_result = await db_session.execute(
select(ScholarPublication).where(ScholarPublication.publication_id == dup_id)
)
dup_links = dup_links_result.scalars().all()
winner_profiles_result = await db_session.execute(
select(ScholarPublication.scholar_profile_id).where(
ScholarPublication.publication_id == winner_id
)
)
winner_profiles: set[int] = {row for (row,) in winner_profiles_result}
for link in dup_links:
if link.scholar_profile_id in winner_profiles:
await db_session.delete(link)
else:
link.publication_id = winner_id
async def sweep_identifier_duplicates(db_session: AsyncSession) -> int:
"""Find publications sharing an identifier and merge duplicates into the winner.
Returns the number of duplicate publications removed.
"""
pairs = await find_identifier_duplicate_pairs(db_session)
if not pairs:
return 0
# Deduplicate the pairs — a dup may appear multiple times if it shares
# several identifiers with the winner; process each dup only once.
processed_dups: set[int] = set()
for winner_id, dup_id in pairs:
if dup_id in processed_dups:
continue
processed_dups.add(dup_id)
await merge_duplicate_publication(db_session, winner_id=winner_id, dup_id=dup_id)
await db_session.flush()
return len(processed_dups)

16
app/services/domains/publications/listing.py
View file

@ -1,5 +1,7 @@
from __future__ import annotations
from datetime import datetime
from sqlalchemy.ext.asyncio import AsyncSession
from app.services.domains.publication_identifiers import application as identifier_service
@ -9,7 +11,7 @@ from app.services.domains.publications.modes import (
resolve_publication_view_mode,
)
from app.services.domains.publications.queries import (
get_latest_completed_run_id_for_user,
get_latest_run_id_for_user,
get_publication_item_for_user,
publication_list_item_from_row,
publications_query,
@ -25,11 +27,15 @@ async def list_for_user(
mode: str = MODE_ALL,
scholar_profile_id: int | None = None,
favorite_only: bool = False,
limit: int = 300,
search: str | None = None,
sort_by: str = "first_seen",
sort_dir: str = "desc",
limit: int = 100,
offset: int = 0,
snapshot_before: datetime | None = None,
) -> list[PublicationListItem]:
resolved_mode = resolve_publication_view_mode(mode)
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
latest_run_id = await get_latest_run_id_for_user(db_session, user_id=user_id)
result = await db_session.execute(
publications_query(
user_id=user_id,
@ -37,8 +43,12 @@ async def list_for_user(
latest_run_id=latest_run_id,
scholar_profile_id=scholar_profile_id,
favorite_only=favorite_only,
search=search,
sort_by=sort_by,
sort_dir=sort_dir,
limit=limit,
offset=offset,
snapshot_before=snapshot_before,
)
)
rows = [

140
app/services/domains/publications/pdf_queue.py
View file

@ -2,10 +2,10 @@ from __future__ import annotations
import asyncio
from dataclasses import dataclass
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
import logging
from sqlalchemy import Select, func, literal, or_, select, union_all
from sqlalchemy import Select, and_, func, literal, or_, select, union_all
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import (
@ -20,7 +20,6 @@ from app.db.session import get_session_factory
from app.services.domains.publication_identifiers import application as identifier_service
from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.domains.publications.pdf_resolution_pipeline import (
PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE,
resolve_publication_pdf_outcome_for_row,
)
from app.services.domains.publications.types import PublicationListItem
@ -49,7 +48,6 @@ _scheduled_tasks: set[asyncio.Task[None]] = set()
class PdfQueueListItem:
publication_id: int
title: str
doi: str | None
pdf_url: str | None
status: str
attempt_count: int
@ -114,7 +112,6 @@ def _item_from_row_and_job(
citation_count=row.citation_count,
venue_text=row.venue_text,
pub_url=row.pub_url,
doi=row.doi,
pdf_url=row.pdf_url,
is_read=row.is_read,
is_favorite=row.is_favorite,
@ -360,7 +357,7 @@ def _failed_outcome(
) -> OaResolutionOutcome:
return OaResolutionOutcome(
publication_id=row.publication_id,
doi=row.doi,
doi=None,
pdf_url=None,
failure_reason=FAILURE_RESOLUTION_EXCEPTION,
source=None,
@ -372,10 +369,12 @@ async def _fetch_outcome_for_row(
*,
row: PublicationListItem,
request_email: str | None,
openalex_api_key: str | None = None,
) -> OaResolutionOutcome:
pipeline_result = await resolve_publication_pdf_outcome_for_row(
row=row,
request_email=request_email,
openalex_api_key=openalex_api_key,
)
outcome = pipeline_result.outcome
if outcome is not None:
@ -386,11 +385,8 @@ async def _fetch_outcome_for_row(
def _apply_publication_update(
publication: Publication,
*,
doi: str | None,
pdf_url: str | None,
) -> None:
if doi and publication.doi != doi:
publication.doi = doi
if pdf_url and publication.pdf_url != pdf_url:
publication.pdf_url = pdf_url
@ -426,7 +422,7 @@ async def _persist_outcome(
job = await db_session.get(PublicationPdfJob, publication_id)
if publication is None or job is None:
return
_apply_publication_update(publication, doi=outcome.doi, pdf_url=outcome.pdf_url)
_apply_publication_update(publication, pdf_url=outcome.pdf_url)
await identifier_service.sync_identifiers_for_publication_resolution(
db_session,
publication=publication,
@ -453,10 +449,26 @@ async def _resolve_publication_row(
user_id: int,
request_email: str | None,
row: PublicationListItem,
openalex_api_key: str | None = None,
) -> None:
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
from app.services.domains.arxiv.application import ArxivRateLimitError
await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
try:
outcome = await _fetch_outcome_for_row(row=row, request_email=request_email)
outcome = await _fetch_outcome_for_row(
row=row,
request_email=request_email,
openalex_api_key=openalex_api_key,
)
except (OpenAlexBudgetExhaustedError, ArxivRateLimitError):
# Persist a terminal outcome so jobs do not remain stuck in "running".
await _persist_outcome(
publication_id=row.publication_id,
user_id=user_id,
outcome=_failed_outcome(row=row),
)
# Propagate upward so the batch loop can stop immediately.
raise
except Exception as exc: # pragma: no cover - defensive network boundary
logger.warning(
"publications.pdf_queue.resolve_failed",
@ -480,12 +492,44 @@ async def _run_resolution_task(
request_email: str | None,
rows: list[PublicationListItem],
) -> None:
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
from app.services.domains.arxiv.application import ArxivRateLimitError
from app.services.domains.settings import application as user_settings_service
# Resolve the best available API key: per-user setting → env var fallback.
openalex_api_key: str | None = None
try:
session_factory = get_session_factory()
async with session_factory() as key_session:
user_settings = await user_settings_service.get_or_create_settings(key_session, user_id=user_id)
openalex_api_key = getattr(user_settings, "openalex_api_key", None) or settings.openalex_api_key
except Exception:
openalex_api_key = settings.openalex_api_key
for row in rows:
await _resolve_publication_row(
user_id=user_id,
request_email=request_email,
row=row,
)
try:
await _resolve_publication_row(
user_id=user_id,
request_email=request_email,
row=row,
openalex_api_key=openalex_api_key,
)
except OpenAlexBudgetExhaustedError:
logger.warning(
"publications.pdf_queue.budget_exhausted",
extra={"event": "publications.pdf_queue.budget_exhausted",
"detail": "Stopping PDF resolution batch — OpenAlex daily budget exhausted"},
)
break
except ArxivRateLimitError:
logger.warning(
"publications.pdf_queue.arxiv_rate_limited",
extra={
"event": "publications.pdf_queue.arxiv_rate_limited",
"detail": "Stopping PDF resolution batch — arXiv rate limit hit (429)",
},
)
break
def _schedule_rows(
@ -571,7 +615,6 @@ def _retry_item_from_publication(
citation_count=int(publication.citation_count or 0),
venue_text=publication.venue_text,
pub_url=publication.pub_url,
doi=publication.doi,
pdf_url=publication.pdf_url,
is_read=is_read,
first_seen_at=first_seen_at,
@ -629,13 +672,12 @@ def _queue_candidate_from_publication(publication: Publication) -> PublicationLi
return PublicationListItem(
publication_id=int(publication.id),
scholar_profile_id=0,
scholar_label="admin",
scholar_label="",
title=publication.title_raw,
year=publication.year,
citation_count=int(publication.citation_count or 0),
venue_text=publication.venue_text,
pub_url=publication.pub_url,
doi=publication.doi,
pdf_url=publication.pdf_url,
is_read=True,
first_seen_at=publication.created_at or _utcnow(),
@ -649,6 +691,9 @@ async def _missing_pdf_candidates(
limit: int,
) -> list[PublicationListItem]:
bounded_limit = max(1, min(int(limit), 5000))
now = datetime.now(timezone.utc)
cooldown_threshold = now - timedelta(days=7)
result = await db_session.execute(
select(Publication)
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
@ -656,7 +701,13 @@ async def _missing_pdf_candidates(
.where(
or_(
PublicationPdfJob.publication_id.is_(None),
PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
and_(
PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
or_(
PublicationPdfJob.last_attempt_at.is_(None),
PublicationPdfJob.last_attempt_at < cooldown_threshold,
),
),
)
)
.order_by(Publication.updated_at.desc(), Publication.id.desc())
@ -694,7 +745,6 @@ def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
select(
PublicationPdfJob.publication_id,
Publication.title_raw,
Publication.doi,
Publication.pdf_url,
PublicationPdfJob.status,
PublicationPdfJob.attempt_count,
@ -730,7 +780,6 @@ def _untracked_queue_select_base() -> Select[tuple]:
select(
Publication.id,
Publication.title_raw,
Publication.doi,
Publication.pdf_url,
literal(PDF_STATUS_UNTRACKED),
literal(0),
@ -793,19 +842,18 @@ def _queue_item_from_row(row: tuple) -> PdfQueueListItem:
return PdfQueueListItem(
publication_id=int(row[0]),
title=str(row[1] or ""),
doi=row[2],
pdf_url=row[3],
status=str(row[4] or PDF_STATUS_UNTRACKED),
attempt_count=int(row[5] or 0),
last_failure_reason=row[6],
last_failure_detail=row[7],
last_source=row[8],
requested_by_user_id=int(row[9]) if row[9] is not None else None,
requested_by_email=row[10],
queued_at=row[11],
last_attempt_at=row[12],
resolved_at=row[13],
updated_at=row[14],
pdf_url=row[2],
status=str(row[3] or PDF_STATUS_UNTRACKED),
attempt_count=int(row[4] or 0),
last_failure_reason=row[5],
last_failure_detail=row[6],
last_source=row[7],
requested_by_user_id=int(row[8]) if row[8] is not None else None,
requested_by_email=row[9],
queued_at=row[10],
last_attempt_at=row[11],
resolved_at=row[12],
updated_at=row[13],
)
@ -902,3 +950,25 @@ async def list_pdf_queue_page(
limit=bounded_limit,
offset=bounded_offset,
)
async def drain_ready_jobs(
db_session: AsyncSession,
*,
limit: int,
max_attempts: int,
) -> int:
result = await db_session.execute(
select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1)
)
system_user_id = result.scalar_one_or_none()
if system_user_id is None:
return 0
bulk_result = await enqueue_all_missing_pdf_jobs(
db_session,
user_id=system_user_id,
request_email=settings.unpaywall_email,
limit=limit,
)
return bulk_result.queued_count

201
app/services/domains/publications/pdf_resolution_pipeline.py
View file

@ -2,100 +2,114 @@ from __future__ import annotations
from dataclasses import dataclass
import logging
from urllib.parse import urlparse
import httpx
from typing import Any
from app.services.domains.arxiv.application import ArxivRateLimitError
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.scholar.publication_pdf import (
ScholarPublicationLinkCandidate,
ScholarPublicationLinkCandidates,
fetch_link_candidates_from_scholar_publication_page,
)
from app.services.domains.unpaywall import pdf_discovery as pdf_discovery_service
from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
from app.settings import settings
logger = logging.getLogger(__name__)
PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE = "scholar_publication_page"
PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED = "scholar_publication_page_unlabeled_fallback"
PDF_PATH_TOKEN = "/pdf/"
HTTP_TIMEOUT_FLOOR_SECONDS = 0.5
@dataclass(frozen=True)
class PipelineOutcome:
outcome: OaResolutionOutcome | None
scholar_candidates: ScholarPublicationLinkCandidates | None
scholar_candidates: Any | None # Kept for backward compatibility with calling signatures
async def resolve_publication_pdf_outcome_for_row(
*,
row: PublicationListItem,
request_email: str | None,
openalex_api_key: str | None = None,
) -> PipelineOutcome:
candidates = await _safe_scholar_candidates(row.pub_url)
labeled = _labeled_candidate(candidates)
if labeled is not None:
return PipelineOutcome(_scholar_outcome(row=row, candidate=labeled), candidates)
# 1. OpenAlex OA — raises OpenAlexBudgetExhaustedError if budget is gone
openalex_outcome = await _openalex_outcome(row, request_email=request_email, openalex_api_key=openalex_api_key)
if openalex_outcome and openalex_outcome.pdf_url:
return PipelineOutcome(openalex_outcome, None)
# 2. arXiv
arxiv_outcome = await _arxiv_outcome(row, request_email=request_email)
if arxiv_outcome and arxiv_outcome.pdf_url:
return PipelineOutcome(arxiv_outcome, None)
# 3. Unpaywall (which falls back to Crossref)
oa_outcome = await _oa_outcome(row=row, request_email=request_email)
if _oa_has_pdf(oa_outcome):
return PipelineOutcome(oa_outcome, candidates)
unlabeled = _unlabeled_candidate(candidates)
if unlabeled is None:
return PipelineOutcome(oa_outcome, candidates)
fallback_outcome = await _unlabeled_fallback_outcome(row=row, candidate=unlabeled)
if fallback_outcome is not None:
return PipelineOutcome(fallback_outcome, candidates)
return PipelineOutcome(oa_outcome, candidates)
return PipelineOutcome(oa_outcome, None)
async def _safe_scholar_candidates(pub_url: str | None) -> ScholarPublicationLinkCandidates | None:
try:
return await fetch_link_candidates_from_scholar_publication_page(pub_url)
except Exception as exc: # pragma: no cover - defensive boundary
logger.warning(
"publications.pdf_resolution.scholar_candidates_failed",
extra={"event": "publications.pdf_resolution.scholar_candidates_failed", "error": str(exc)},
)
return None
def _labeled_candidate(
candidates: ScholarPublicationLinkCandidates | None,
) -> ScholarPublicationLinkCandidate | None:
if candidates is None:
return None
return candidates.labeled_candidate
def _unlabeled_candidate(
candidates: ScholarPublicationLinkCandidates | None,
) -> ScholarPublicationLinkCandidate | None:
if candidates is None:
return None
return candidates.fallback_candidate
def _scholar_outcome(
*,
async def _openalex_outcome(
row: PublicationListItem,
candidate: ScholarPublicationLinkCandidate,
) -> OaResolutionOutcome:
source = (
PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE
if candidate.label_present
else PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED
)
return OaResolutionOutcome(
publication_id=row.publication_id,
doi=row.doi,
pdf_url=candidate.url,
failure_reason=None,
source=source,
used_crossref=False,
)
request_email: str | None,
openalex_api_key: str | None = None,
) -> OaResolutionOutcome | None:
from app.services.domains.openalex.client import OpenAlexClient
from app.services.domains.openalex.matching import find_best_match
if not row.title:
return None
import re
safe_title = re.sub(r"[^\w\s]", " ", row.title)
safe_title = " ".join(safe_title.split())
if not safe_title:
return None
api_key = openalex_api_key or settings.openalex_api_key
client = OpenAlexClient(api_key=api_key, mailto=request_email or settings.crossref_api_mailto)
try:
openalex_works = await client.get_works_by_filter({"title.search": safe_title}, limit=5)
match = find_best_match(
target_title=row.title,
target_year=row.year,
target_authors=row.scholar_label,
candidates=openalex_works,
)
if match and match.oa_url:
return OaResolutionOutcome(
publication_id=row.publication_id,
doi=match.doi,
pdf_url=match.oa_url,
failure_reason=None,
source="openalex",
used_crossref=False,
)
except OpenAlexBudgetExhaustedError:
# Re-raise so the caller's batch loop can stop hitting the API.
raise
except Exception as exc:
logger.warning(
"publications.pdf_resolution.openalex_failed",
extra={"event": "publications.pdf_resolution.openalex_failed", "error": str(exc)},
)
return None
async def _arxiv_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None:
from app.services.domains.arxiv.application import discover_arxiv_id_for_publication
try:
arxiv_id = await discover_arxiv_id_for_publication(item=row, request_email=request_email)
if arxiv_id:
pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
return OaResolutionOutcome(
publication_id=row.publication_id,
doi=None,
pdf_url=pdf_url,
failure_reason=None,
source="arxiv",
used_crossref=False,
)
except ArxivRateLimitError:
raise # propagate so the batch loop can stop
except Exception as exc:
logger.warning(
"publications.pdf_resolution.arxiv_failed",
extra={"event": "publications.pdf_resolution.arxiv_failed", "error": str(exc)},
)
return None
async def _oa_outcome(
@ -105,46 +119,3 @@ async def _oa_outcome(
) -> OaResolutionOutcome | None:
outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email)
return outcomes.get(row.publication_id)
def _oa_has_pdf(outcome: OaResolutionOutcome | None) -> bool:
return bool(outcome and outcome.pdf_url)
async def _unlabeled_fallback_outcome(
*,
row: PublicationListItem,
candidate: ScholarPublicationLinkCandidate,
) -> OaResolutionOutcome | None:
pdf_url = await _validated_pdf_url(candidate.url)
if pdf_url is None:
return None
return _scholar_outcome(row=row, candidate=ScholarPublicationLinkCandidate(
url=pdf_url,
confidence_score=candidate.confidence_score,
label_present=False,
reason=candidate.reason,
))
async def _validated_pdf_url(candidate_url: str) -> str | None:
if _looks_direct_pdf(candidate_url):
return candidate_url
timeout_seconds = _discovery_timeout_seconds()
async with httpx.AsyncClient(timeout=timeout_seconds) as client:
if await pdf_discovery_service._candidate_is_pdf(client, candidate_url=candidate_url):
return candidate_url
return await pdf_discovery_service.resolve_pdf_from_landing_page(client, page_url=candidate_url)
def _looks_direct_pdf(url: str | None) -> bool:
if pdf_discovery_service.looks_like_pdf_url(url):
return True
if not isinstance(url, str):
return False
path = (urlparse(url).path or "").lower()
return PDF_PATH_TOKEN in path
def _discovery_timeout_seconds() -> float:
return max(float(settings.unpaywall_timeout_seconds), HTTP_TIMEOUT_FLOOR_SECONDS)

56
app/services/domains/publications/queries.py
View file

@ -1,5 +1,7 @@
from __future__ import annotations
from datetime import datetime
from sqlalchemy import Select, func, select
from sqlalchemy.ext.asyncio import AsyncSession
@ -15,15 +17,24 @@ def _normalized_citation_count(value: object) -> int:
return 0
async def get_latest_completed_run_id_for_user(
async def get_latest_run_id_for_user(
db_session: AsyncSession,
*,
user_id: int,
) -> int | None:
# We include RUNNING and RESOLVING statuses so that the "New" tab shows
# results in real-time as they are discovered.
result = await db_session.execute(
select(func.max(CrawlRun.id)).where(
CrawlRun.user_id == user_id,
CrawlRun.status != RunStatus.RUNNING,
CrawlRun.status.in_(
[
RunStatus.RUNNING,
RunStatus.RESOLVING,
RunStatus.SUCCESS,
RunStatus.PARTIAL_FAILURE,
]
),
)
)
latest_run_id = result.scalar_one_or_none()
@ -39,7 +50,19 @@ def publications_query(
favorite_only: bool,
limit: int,
offset: int = 0,
search: str | None = None,
sort_by: str = "first_seen",
sort_dir: str = "desc",
snapshot_before: datetime | None = None,
) -> Select[tuple]:
_SORT_COLUMNS = {
"first_seen": ScholarPublication.created_at,
"title": Publication.title_raw,
"year": Publication.year,
"citations": Publication.citation_count,
"scholar": ScholarProfile.display_name,
}
scholar_label = ScholarProfile.display_name
stmt = (
select(
@ -52,7 +75,6 @@ def publications_query(
Publication.citation_count,
Publication.venue_text,
Publication.pub_url,
Publication.doi,
Publication.pdf_url,
ScholarPublication.is_read,
ScholarPublication.is_favorite,
@ -62,10 +84,15 @@ def publications_query(
.join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
.where(ScholarProfile.user_id == user_id)
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
.offset(max(int(offset), 0))
.limit(limit)
)
if search:
safe_search = search.replace("%", r"\%").replace("_", r"\_")
pat = f"%{safe_search}%"
stmt = stmt.where(
Publication.title_raw.ilike(pat)
| ScholarProfile.display_name.ilike(pat)
| Publication.venue_text.ilike(pat)
)
if scholar_profile_id is not None:
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
if favorite_only:
@ -76,6 +103,16 @@ def publications_query(
if latest_run_id is None:
return stmt.where(False)
stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
if snapshot_before is not None:
stmt = stmt.where(ScholarPublication.created_at <= snapshot_before)
sort_col = _SORT_COLUMNS.get(sort_by, ScholarPublication.created_at)
order = sort_col.desc() if sort_dir == "desc" else sort_col.asc()
stmt = stmt.order_by(order, Publication.id.desc())
if limit is not None:
stmt = stmt.offset(max(int(offset), 0)).limit(limit)
return stmt
@ -96,7 +133,6 @@ def publication_query_for_user(
Publication.citation_count,
Publication.venue_text,
Publication.pub_url,
Publication.doi,
Publication.pdf_url,
ScholarPublication.is_read,
ScholarPublication.is_favorite,
@ -121,7 +157,7 @@ async def get_publication_item_for_user(
scholar_profile_id: int,
publication_id: int,
) -> PublicationListItem | None:
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
latest_run_id = await get_latest_run_id_for_user(db_session, user_id=user_id)
result = await db_session.execute(
publication_query_for_user(
user_id=user_id,
@ -150,7 +186,6 @@ def publication_list_item_from_row(
citation_count,
venue_text,
pub_url,
doi,
pdf_url,
is_read,
is_favorite,
@ -166,7 +201,6 @@ def publication_list_item_from_row(
citation_count=_normalized_citation_count(citation_count),
venue_text=venue_text,
pub_url=pub_url,
doi=doi,
pdf_url=pdf_url,
is_read=bool(is_read),
is_favorite=bool(is_favorite),
@ -188,7 +222,6 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
citation_count,
venue_text,
pub_url,
doi,
pdf_url,
_is_read,
_is_favorite,
@ -204,6 +237,5 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
citation_count=_normalized_citation_count(citation_count),
venue_text=venue_text,
pub_url=pub_url,
doi=doi,
pdf_url=pdf_url,
)

2
app/services/domains/publications/types.py
View file

@ -16,7 +16,6 @@ class PublicationListItem:
citation_count: int
venue_text: str | None
pub_url: str | None
doi: str | None
pdf_url: str | None
is_read: bool
first_seen_at: datetime
@ -39,5 +38,4 @@ class UnreadPublicationItem:
citation_count: int
venue_text: str | None
pub_url: str | None
doi: str | None
pdf_url: str | None

59
app/services/domains/runs/events.py Normal file
View file

@ -0,0 +1,59 @@
import asyncio
import json
import logging
from typing import Any, AsyncGenerator, Dict, Set
logger = logging.getLogger(__name__)
class RunEventPublisher:
def __init__(self) -> None:
# Maps run_id to a set of subscriber queues
self._subscribers: Dict[int, Set[asyncio.Queue]] = {}
def subscribe(self, run_id: int) -> asyncio.Queue:
if run_id not in self._subscribers:
self._subscribers[run_id] = set()
queue: asyncio.Queue[Any] = asyncio.Queue()
self._subscribers[run_id].add(queue)
logger.debug(f"New subscriber for run {run_id}. Total: {len(self._subscribers[run_id])}")
return queue
def unsubscribe(self, run_id: int, queue: asyncio.Queue) -> None:
if run_id in self._subscribers:
self._subscribers[run_id].discard(queue)
if not self._subscribers[run_id]:
self._subscribers.pop(run_id, None)
async def publish(self, run_id: int, event_type: str, data: dict[str, Any]) -> None:
if run_id not in self._subscribers:
return
message = {
"type": event_type,
"data": data
}
# Fan-out to all active subscribers for this run
for queue in list(self._subscribers[run_id]):
try:
queue.put_nowait(message)
except asyncio.QueueFull:
logger.warning(f"Subscriber queue full for run {run_id}, dropping message")
run_events = RunEventPublisher()
async def event_generator(run_id: int) -> AsyncGenerator[str, None]:
queue = run_events.subscribe(run_id)
try:
while True:
# Wait for a new event
message = await queue.get()
# Server-Sent Events format: "event: <type>\ndata: <json>\n\n"
event_type = message["type"]
data_str = json.dumps(message["data"])
yield f"event: {event_type}\ndata: {data_str}\n\n"
except asyncio.CancelledError:
logger.debug(f"Client disconnected from SSE stream for run {run_id}")
raise
finally:
run_events.unsubscribe(run_id, queue)

232
app/services/domains/scholar/publication_pdf.py
View file

@ -1,232 +0,0 @@
from __future__ import annotations
from dataclasses import dataclass
from html.parser import HTMLParser
from urllib.parse import parse_qs, urlparse
from app.services.domains.scholar.parser_types import ScholarDomInvariantError
from app.services.domains.scholar.parser_utils import attr_href, normalize_space
from app.services.domains.scholar.rate_limit import wait_for_scholar_slot
from app.services.domains.scholar.source import LiveScholarSource
from app.services.domains.settings.application import resolve_request_delay_minimum
from app.settings import settings
CONTAINER_ID = "gsc_oci_title_gg"
PDF_LABEL_TOKEN = "[pdf]"
SCHOLAR_PDF_LABELED_CONFIDENCE = 0.98
SCHOLAR_PDF_UNLABELED_CONFIDENCE = 0.2
ALLOWED_URL_SCHEMES = frozenset({"http", "https"})
@dataclass(frozen=True)
class ScholarPublicationLinkCandidate:
url: str
confidence_score: float
label_present: bool
reason: str
@dataclass(frozen=True)
class ScholarPublicationLinkCandidates:
container_seen: bool
labeled_candidate: ScholarPublicationLinkCandidate | None
fallback_candidate: ScholarPublicationLinkCandidate | None
warnings: tuple[str, ...] = ()
@dataclass(frozen=True)
class _ParsedAnchor:
href: str | None
text: str
class _ScholarPublicationPdfParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.container_seen = False
self.anchors: list[_ParsedAnchor] = []
self._container_depth = 0
self._anchor_depth = 0
self._anchor_href: str | None = None
self._anchor_parts: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
self._increment_depths()
if self._starts_container(tag, attrs):
self.container_seen = True
self._container_depth = 1
return
if self._container_depth <= 0 or tag != "a":
return
if self._anchor_depth > 0:
return
self._anchor_href = attr_href(attrs)
self._anchor_parts = []
self._anchor_depth = 1
def handle_data(self, data: str) -> None:
if self._anchor_depth > 0:
self._anchor_parts.append(data)
def handle_endtag(self, tag: str) -> None:
if self._anchor_depth > 0:
self._anchor_depth -= 1
if self._anchor_depth == 0:
self._finish_anchor()
if self._container_depth > 0:
self._container_depth -= 1
def _increment_depths(self) -> None:
if self._container_depth > 0:
self._container_depth += 1
if self._anchor_depth > 0:
self._anchor_depth += 1
def _starts_container(self, tag: str, attrs: list[tuple[str, str | None]]) -> bool:
if tag != "div":
return False
attrs_map = {name.lower(): (value or "") for name, value in attrs}
return attrs_map.get("id") == CONTAINER_ID
def _finish_anchor(self) -> None:
self.anchors.append(
_ParsedAnchor(
href=self._anchor_href,
text=normalize_space("".join(self._anchor_parts)),
)
)
self._anchor_href = None
self._anchor_parts = []
def is_scholar_publication_detail_url(url: str | None) -> bool:
if not isinstance(url, str) or not url.strip():
return False
parsed = urlparse(url)
if parsed.scheme not in ALLOWED_URL_SCHEMES:
return False
if parsed.netloc.lower() != "scholar.google.com":
return False
query = parse_qs(parsed.query)
return _has_view_citation_params(query)
def _has_view_citation_params(query: dict[str, list[str]]) -> bool:
view_op = (query.get("view_op") or [""])[0]
citation = (query.get("citation_for_view") or [""])[0].strip()
return view_op == "view_citation" and bool(citation)
def extract_link_candidates_from_publication_detail_html(html: str) -> ScholarPublicationLinkCandidates:
parser = _parsed_publication_detail(html)
if not parser.container_seen:
return ScholarPublicationLinkCandidates(False, None, None)
anchors = _validated_container_anchors(parser.anchors)
labeled = _select_labeled_candidate(anchors)
fallback = _select_fallback_candidate(anchors, labeled=labeled)
warnings = _candidate_warnings(labeled=labeled, fallback=fallback)
return ScholarPublicationLinkCandidates(True, labeled, fallback, warnings)
def _parsed_publication_detail(html: str) -> _ScholarPublicationPdfParser:
parser = _ScholarPublicationPdfParser()
parser.feed(html)
parser.close()
return parser
def _validated_container_anchors(anchors: list[_ParsedAnchor]) -> list[_ParsedAnchor]:
if not anchors:
raise ScholarDomInvariantError(
code="layout_publication_link_container_missing_anchor",
message="Scholar publication link container was present without an anchor.",
)
validated: list[_ParsedAnchor] = []
for anchor in anchors:
validated.append(_validated_anchor(anchor))
return validated
def _validated_anchor(anchor: _ParsedAnchor) -> _ParsedAnchor:
href = (anchor.href or "").strip()
if not href:
raise ScholarDomInvariantError(
code="layout_publication_link_missing_href",
message="Scholar publication link container anchor was missing href.",
)
parsed = urlparse(href)
if parsed.scheme not in ALLOWED_URL_SCHEMES:
raise ScholarDomInvariantError(
code="layout_publication_link_invalid_scheme",
message="Scholar publication link used a non-http URL.",
)
return _ParsedAnchor(href=href, text=anchor.text)
def _select_labeled_candidate(anchors: list[_ParsedAnchor]) -> ScholarPublicationLinkCandidate | None:
for anchor in anchors:
if PDF_LABEL_TOKEN in anchor.text.lower():
return ScholarPublicationLinkCandidate(
url=str(anchor.href),
confidence_score=SCHOLAR_PDF_LABELED_CONFIDENCE,
label_present=True,
reason="scholar_link_labeled_pdf",
)
return None
def _select_fallback_candidate(
anchors: list[_ParsedAnchor],
*,
labeled: ScholarPublicationLinkCandidate | None,
) -> ScholarPublicationLinkCandidate | None:
for anchor in anchors:
if labeled and anchor.href == labeled.url:
continue
return ScholarPublicationLinkCandidate(
url=str(anchor.href),
confidence_score=SCHOLAR_PDF_UNLABELED_CONFIDENCE,
label_present=False,
reason="scholar_link_unlabeled_fallback",
)
if labeled is None and anchors:
anchor = anchors[0]
return ScholarPublicationLinkCandidate(
url=str(anchor.href),
confidence_score=SCHOLAR_PDF_UNLABELED_CONFIDENCE,
label_present=False,
reason="scholar_link_unlabeled_fallback",
)
return None
def _candidate_warnings(
*,
labeled: ScholarPublicationLinkCandidate | None,
fallback: ScholarPublicationLinkCandidate | None,
) -> tuple[str, ...]:
warnings: list[str] = []
if labeled is None and fallback is not None:
warnings.append("scholar_publication_link_unlabeled_only")
return tuple(warnings)
def _scholar_request_delay_seconds() -> int:
return resolve_request_delay_minimum(settings.ingestion_min_request_delay_seconds)
def _fetch_succeeded(fetch_result) -> bool:
return int(fetch_result.status_code or 0) == 200 and not fetch_result.error
async def fetch_link_candidates_from_scholar_publication_page(
publication_url: str | None,
) -> ScholarPublicationLinkCandidates | None:
if not is_scholar_publication_detail_url(publication_url):
return None
await wait_for_scholar_slot(min_interval_seconds=float(_scholar_request_delay_seconds()))
source = LiveScholarSource()
fetch_result = await source.fetch_publication_html(str(publication_url))
if not _fetch_succeeded(fetch_result):
return None
return extract_link_candidates_from_publication_detail_html(fetch_result.body)

15
app/services/domains/settings/application.py
View file

@ -109,6 +109,8 @@ def parse_nav_visible_pages(value: object) -> list[str]:
return deduped
from app.settings import settings as app_settings
async def get_or_create_settings(
db_session: AsyncSession,
*,
@ -121,7 +123,12 @@ async def get_or_create_settings(
if settings is not None:
return settings
settings = UserSetting(user_id=user_id)
settings = UserSetting(
user_id=user_id,
openalex_api_key=app_settings.openalex_api_key,
crossref_api_token=app_settings.crossref_api_token,
crossref_api_mailto=app_settings.crossref_api_mailto,
)
db_session.add(settings)
await db_session.commit()
await db_session.refresh(settings)
@ -136,11 +143,17 @@ async def update_settings(
run_interval_minutes: int,
request_delay_seconds: int,
nav_visible_pages: list[str],
openalex_api_key: str | None,
crossref_api_token: str | None,
crossref_api_mailto: str | None,
) -> UserSetting:
settings.auto_run_enabled = auto_run_enabled
settings.run_interval_minutes = run_interval_minutes
settings.request_delay_seconds = request_delay_seconds
settings.nav_visible_pages = nav_visible_pages
settings.openalex_api_key = openalex_api_key
settings.crossref_api_token = crossref_api_token
settings.crossref_api_mailto = crossref_api_mailto
await db_session.commit()
await db_session.refresh(settings)
return settings

10
app/services/domains/unpaywall/application.py
View file

@ -63,7 +63,10 @@ def _extract_explicit_doi(text: str | None) -> str | None:
def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str | None:
stored = normalize_doi(item.doi)
stored = None
if getattr(item, "display_identifier", None) and item.display_identifier.kind == "doi":
stored = normalize_doi(item.display_identifier.value)
explicit_doi = (
_extract_explicit_doi(item.pub_url)
or _extract_explicit_doi(item.venue_text)
@ -169,9 +172,11 @@ async def _fetch_unpaywall_payload_by_doi(
email: str,
) -> dict | None:
await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{email})"}
response = await client.get(
UNPAYWALL_URL_TEMPLATE.format(doi=doi),
params={"email": email},
headers=headers,
)
if response.status_code != 200:
return None
@ -427,7 +432,8 @@ async def resolve_publication_oa_outcomes(
timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5)
targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)]
async with httpx.AsyncClient(timeout=timeout_seconds) as client:
headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{email})"}
async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True, headers=headers) as client:
outcomes = await _resolve_outcomes_with_client(
client=client,
targets=targets,