justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

ci: add ruff linting and mypy type checking

Browse source 
Add ruff and mypy to dev dependencies with configuration in pyproject.toml.
Add a lint CI job that runs ruff check, ruff format --check, and mypy.
Auto-fix import sorting and formatting across the codebase. Exclude
alembic/versions from linting (auto-generated migrations). Ignore B008
(FastAPI Depends pattern) and RUF001 (unicode in user-facing strings).

21 ruff lint errors and 50 mypy errors remain for manual review.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-02-26 22:11:41 +01:00
parent 399276ea69
commit bf04c77aa9
137 changed files with 3066 additions and 1900 deletions
Download patch file Download diff file Expand all files Collapse all files

20
app/services/domains/openalex/client.py
View file

@ -1,6 +1,4 @@
import asyncio
import logging
from typing import Any, Mapping
import httpx
from tenacity import (
@ -23,11 +21,13 @@ class OpenAlexClientError(Exception):
class OpenAlexRateLimitError(OpenAlexClientError):
"""Transient rate limit (too many requests per second)."""
pass
class OpenAlexBudgetExhaustedError(OpenAlexClientError):
"""Daily API budget exhausted — retrying is futile until midnight UTC."""
pass
@ -64,7 +64,7 @@ class OpenAlexClient:
return None
url = f"{OPENALEX_BASE_URL}/works/{clean_doi}"
headers = {}
if self.mailto:
headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
@ -79,9 +79,7 @@ class OpenAlexClient:
if response.status_code == 429:
remaining = response.headers.get("X-RateLimit-Remaining-USD", "")
if remaining == "0" or remaining.startswith("-"):
raise OpenAlexBudgetExhaustedError(
"Daily API budget exhausted; retrying won't help until midnight UTC"
)
raise OpenAlexBudgetExhaustedError("Daily API budget exhausted; retrying won't help until midnight UTC")
raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI")
if response.status_code >= 400:
logger.warning("OpenAlex API error: %s %s", response.status_code, response.text[:500])
@ -110,13 +108,13 @@ class OpenAlexClient:
# Example: {"doi": "10.foo|10.bar", "title.search": "query"}
filter_str = ",".join(f"{k}:{v}" for k, v in filters.items())
params = self._base_params.copy()
params["filter"] = filter_str
params["per-page"] = str(limit)
url = f"{OPENALEX_BASE_URL}/works"
headers = {}
if self.mailto:
headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
@ -129,9 +127,7 @@ class OpenAlexClient:
if response.status_code == 429:
remaining = response.headers.get("X-RateLimit-Remaining-USD", "")
if remaining == "0" or remaining.startswith("-"):
raise OpenAlexBudgetExhaustedError(
"Daily API budget exhausted; retrying won't help until midnight UTC"
)
raise OpenAlexBudgetExhaustedError("Daily API budget exhausted; retrying won't help until midnight UTC")
raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list")
if response.status_code >= 400:
logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text[:500])
@ -139,7 +135,7 @@ class OpenAlexClient:
data = response.json()
results = data.get("results") or []
parsed_works = []
for raw_work in results:
try:

27
app/services/domains/openalex/matching.py
View file

@ -24,11 +24,11 @@ def _clean_string(s: str | None) -> str:
def _author_overlap_score(target_authors: str | None, candidate_authors: list[str]) -> bool:
if not target_authors or not candidate_authors:
return False
target_clean = _clean_string(target_authors)
if not target_clean:
return False
for candidate in candidate_authors:
cand_clean = _clean_string(candidate)
if cand_clean and (cand_clean in target_clean or target_clean in cand_clean):
@ -36,7 +36,7 @@ def _author_overlap_score(target_authors: str | None, candidate_authors: list[st
# Alternatively check rapidfuzz token_set_ratio
if cand_clean and fuzz.token_set_ratio(target_clean, cand_clean) > 80:
return True
return False
@ -62,12 +62,12 @@ def find_best_match(
for cand in candidates:
if not cand.title:
continue
clean_cand = _clean_string(cand.title)
# Primary sort: string similarity ratio
score = fuzz.ratio(clean_target, clean_cand)
if score >= TITLE_MATCH_THRESHOLD:
scored_candidates.append((score, cand))
@ -76,13 +76,12 @@ def find_best_match(
# Sort descending by score
scored_candidates.sort(key=lambda x: x[0], reverse=True)
best_score = scored_candidates[0][0]
# Extract all candidates within the tiebreaker margin
top_scored_candidates = [
(score, cand) for score, cand in scored_candidates
if best_score - score <= TIEBREAKER_MARGIN
(score, cand) for score, cand in scored_candidates if best_score - score <= TIEBREAKER_MARGIN
]
if len(top_scored_candidates) == 1:
@ -91,18 +90,18 @@ def find_best_match(
# We have a tie or near-tie. Use year and author overlap to break the tie.
# Score candidates: +1 for year match (within 1 year), +1 for author overlap
tiebreaker_scores: list[tuple[int, float, OpenAlexWork]] = []
for original_score, cand in top_scored_candidates:
tb_score = 0
if target_year is not None and cand.publication_year is not None:
if abs(target_year - cand.publication_year) <= 1:
tb_score += 1
candidate_author_names = [a.display_name for a in cand.authors if a.display_name]
if _author_overlap_score(target_authors, candidate_author_names):
tb_score += 1
tiebreaker_scores.append((tb_score, original_score, cand))
tiebreaker_scores.sort(key=lambda x: (x[0], x[1]), reverse=True)
return tiebreaker_scores[0][2]

12
app/services/domains/openalex/types.py
View file

@ -1,8 +1,8 @@
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any, Mapping
from typing import Any
@dataclass(frozen=True)
@ -28,24 +28,24 @@ class OpenAlexWork:
@classmethod
def from_api_dict(cls, data: Mapping[str, Any]) -> OpenAlexWork:
ids = data.get("ids") or {}
# Extract DOI without the https://doi.org/ prefix
doi = ids.get("doi")
if doi and doi.startswith("https://doi.org/"):
doi = doi[16:]
# Extract PMID without the url prefix
pmid = ids.get("pmid")
if pmid and pmid.startswith("https://pubmed.ncbi.nlm.nih.gov/"):
pmid = pmid[32:]
# Extract PMCID without the url prefix
pmcid = ids.get("pmcid")
if pmcid and pmcid.startswith("https://www.ncbi.nlm.nih.gov/pmc/articles/"):
pmcid = pmcid[42:]
open_access = data.get("open_access") or {}
authors = []
for authorship in data.get("authorships") or []:
author_data = authorship.get("author") or {}