ci: add CodeQL security scanning and Dependabot

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app/services/openalex/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger(__name__)

app/services/openalex/client.py

@@ -0,0 +1,147 @@
+import logging
+
+import httpx
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+from app.services.domains.openalex.types import OpenAlexWork
+
+logger = logging.getLogger(__name__)
+
+OPENALEX_BASE_URL = "https://api.openalex.org"
+
+
+class OpenAlexClientError(Exception):
+    pass
+
+
+class OpenAlexRateLimitError(OpenAlexClientError):
+    """Transient rate limit (too many requests per second)."""
+
+    pass
+
+
+class OpenAlexBudgetExhaustedError(OpenAlexClientError):
+    """Daily API budget exhausted — retrying is futile until midnight UTC."""
+
+    pass
+
+
+class OpenAlexClient:
+    def __init__(
+        self,
+        api_key: str | None = None,
+        mailto: str | None = None,
+        timeout: float = 10.0,
+    ) -> None:
+        self.api_key = api_key
+        self.mailto = mailto
+        self.timeout = timeout
+
+    @property
+    def _base_params(self) -> dict[str, str]:
+        params = {}
+        if self.mailto:
+            params["mailto"] = self.mailto
+        if self.api_key:
+            params["api_key"] = self.api_key
+        return params
+
+    @retry(
+        retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException)),
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=2, max=10),
+        reraise=True,
+    )
+    async def get_work_by_doi(self, doi: str) -> OpenAlexWork | None:
+        """Fetch a single work by DOI directly."""
+        clean_doi = doi.replace("https://doi.org/", "")
+        if not clean_doi:
+            return None
+
+        url = f"{OPENALEX_BASE_URL}/works/{clean_doi}"
+
+        headers = {}
+        if self.mailto:
+            headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
+        else:
+            headers["User-Agent"] = "scholar-scraper/1.0"
+
+        async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client:
+            response = await client.get(url, params=self._base_params)
+
+        if response.status_code == 404:
+            return None
+        if response.status_code == 429:
+            remaining = response.headers.get("X-RateLimit-Remaining-USD", "")
+            if remaining == "0" or remaining.startswith("-"):
+                raise OpenAlexBudgetExhaustedError("Daily API budget exhausted; retrying won't help until midnight UTC")
+            raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI")
+        if response.status_code >= 400:
+            logger.warning("OpenAlex API error: %s %s", response.status_code, response.text[:500])
+            raise OpenAlexClientError(f"API Error {response.status_code}")
+
+        data = response.json()
+        return OpenAlexWork.from_api_dict(data)
+
+    @retry(
+        retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException)),
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=2, max=10),
+        reraise=True,
+    )
+    async def get_works_by_filter(
+        self,
+        filters: dict[str, str],
+        limit: int = 50,
+    ) -> list[OpenAlexWork]:
+        """
+        Fetch works using the ?filter= query parameter.
+        Supports fetching multiple records by joining filters with | (OR logic).
+        """
+        if not filters:
+            return []
+
+        # Example: {"doi": "10.foo|10.bar", "title.search": "query"}
+        filter_str = ",".join(f"{k}:{v}" for k, v in filters.items())
+
+        params = self._base_params.copy()
+        params["filter"] = filter_str
+        params["per-page"] = str(limit)
+
+        url = f"{OPENALEX_BASE_URL}/works"
+
+        headers = {}
+        if self.mailto:
+            headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
+        else:
+            headers["User-Agent"] = "scholar-scraper/1.0"
+
+        async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client:
+            response = await client.get(url, params=params)
+
+        if response.status_code == 429:
+            remaining = response.headers.get("X-RateLimit-Remaining-USD", "")
+            if remaining == "0" or remaining.startswith("-"):
+                raise OpenAlexBudgetExhaustedError("Daily API budget exhausted; retrying won't help until midnight UTC")
+            raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list")
+        if response.status_code >= 400:
+            logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text[:500])
+            raise OpenAlexClientError(f"API Error {response.status_code}")
+
+        data = response.json()
+        results = data.get("results") or []
+
+        parsed_works = []
+        for raw_work in results:
+            try:
+                parsed_works.append(OpenAlexWork.from_api_dict(raw_work))
+            except Exception as e:
+                logger.warning("Failed to parse OpenAlex raw dict: %s", e)
+                continue
+
+        return parsed_works

app/services/openalex/matching.py

@@ -0,0 +1,106 @@
+import logging
+import re
+
+from rapidfuzz import fuzz
+
+from app.services.domains.openalex.types import OpenAlexWork
+
+logger = logging.getLogger(__name__)
+
+# A minimum similarity score out of 100 for a title to be considered a match candidate.
+TITLE_MATCH_THRESHOLD = 90.0
+# The margin within the top score where a secondary tiebreaker (author/year) is necessary.
+TIEBREAKER_MARGIN = 5.0
+
+
+def _clean_string(s: str | None) -> str:
+    if not s:
+        return ""
+    # Strip non-alphanumeric (keep spaces), lowercase, and collapse whitespace
+    cleaned = re.sub(r"[^a-z0-9\s]", " ", s.lower())
+    return " ".join(cleaned.split())
+
+
+def _author_overlap_score(target_authors: str | None, candidate_authors: list[str]) -> bool:
+    if not target_authors or not candidate_authors:
+        return False
+
+    target_clean = _clean_string(target_authors)
+    if not target_clean:
+        return False
+
+    for candidate in candidate_authors:
+        cand_clean = _clean_string(candidate)
+        if cand_clean and (cand_clean in target_clean or target_clean in cand_clean):
+            return True
+        # Alternatively check rapidfuzz token_set_ratio
+        if cand_clean and fuzz.token_set_ratio(target_clean, cand_clean) > 80:
+            return True
+
+    return False
+
+
+def find_best_match(
+    target_title: str,
+    target_year: int | None,
+    target_authors: str | None,
+    candidates: list[OpenAlexWork],
+) -> OpenAlexWork | None:
+    """
+    Finds the best matching OpenAlexWork from a list of candidates, prioritizing title similarity (>90%)
+    with year and author overlap as tiebreakers for close candidates.
+    """
+    if not target_title or not candidates:
+        return None
+
+    clean_target = _clean_string(target_title)
+    if not clean_target:
+        return None
+
+    scored_candidates: list[tuple[float, OpenAlexWork]] = []
+
+    for cand in candidates:
+        if not cand.title:
+            continue
+
+        clean_cand = _clean_string(cand.title)
+
+        # Primary sort: string similarity ratio
+        score = fuzz.ratio(clean_target, clean_cand)
+
+        if score >= TITLE_MATCH_THRESHOLD:
+            scored_candidates.append((score, cand))
+
+    if not scored_candidates:
+        return None
+
+    # Sort descending by score
+    scored_candidates.sort(key=lambda x: x[0], reverse=True)
+
+    best_score = scored_candidates[0][0]
+
+    # Extract all candidates within the tiebreaker margin
+    top_scored_candidates = [
+        (score, cand) for score, cand in scored_candidates if best_score - score <= TIEBREAKER_MARGIN
+    ]
+
+    if len(top_scored_candidates) == 1:
+        return top_scored_candidates[0][1]
+
+    # We have a tie or near-tie. Use year and author overlap to break the tie.
+    # Score candidates: +1 for year match (within 1 year), +1 for author overlap
+    tiebreaker_scores: list[tuple[int, float, OpenAlexWork]] = []
+
+    for original_score, cand in top_scored_candidates:
+        tb_score = 0
+        if target_year is not None and cand.publication_year is not None and abs(target_year - cand.publication_year) <= 1:
+                tb_score += 1
+
+        candidate_author_names = [a.display_name for a in cand.authors if a.display_name]
+        if _author_overlap_score(target_authors, candidate_author_names):
+            tb_score += 1
+
+        tiebreaker_scores.append((tb_score, original_score, cand))
+
+    tiebreaker_scores.sort(key=lambda x: (x[0], x[1]), reverse=True)
+    return tiebreaker_scores[0][2]

app/services/openalex/types.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class OpenAlexAuthor:
+    openalex_id: str | None
+    display_name: str | None
+
+
+@dataclass(frozen=True)
+class OpenAlexWork:
+    openalex_id: str
+    doi: str | None
+    pmid: str | None
+    pmcid: str | None
+    title: str | None
+    publication_year: int | None
+    cited_by_count: int
+    is_oa: bool
+    oa_url: str | None
+    authors: list[OpenAlexAuthor] = field(default_factory=list)
+    raw_data: Mapping[str, Any] = field(default_factory=dict, repr=False)
+
+    @classmethod
+    def from_api_dict(cls, data: Mapping[str, Any]) -> OpenAlexWork:
+        ids = data.get("ids") or {}
+
+        # Extract DOI without the https://doi.org/ prefix
+        doi = ids.get("doi")
+        if doi and doi.startswith("https://doi.org/"):
+            doi = doi[16:]
+
+        # Extract PMID without the url prefix
+        pmid = ids.get("pmid")
+        if pmid and pmid.startswith("https://pubmed.ncbi.nlm.nih.gov/"):
+            pmid = pmid[32:]
+
+        # Extract PMCID without the url prefix
+        pmcid = ids.get("pmcid")
+        if pmcid and pmcid.startswith("https://www.ncbi.nlm.nih.gov/pmc/articles/"):
+            pmcid = pmcid[42:]
+
+        open_access = data.get("open_access") or {}
+
+        authors = []
+        for authorship in data.get("authorships") or []:
+            author_data = authorship.get("author") or {}
+            authors.append(
+                OpenAlexAuthor(
+                    openalex_id=author_data.get("id"),
+                    display_name=author_data.get("display_name"),
+                )
+            )
+
+        return cls(
+            openalex_id=data.get("id", ""),
+            doi=doi,
+            pmid=pmid,
+            pmcid=pmcid,
+            title=data.get("title"),
+            publication_year=data.get("publication_year"),
+            cited_by_count=data.get("cited_by_count", 0),
+            is_oa=bool(open_access.get("is_oa")),
+            oa_url=open_access.get("oa_url"),
+            authors=authors,
+            raw_data=dict(data),
+        )