135 lines
4.3 KiB
Python
135 lines
4.3 KiB
Python
import asyncio
|
|
import logging
|
|
from typing import Any, Mapping
|
|
|
|
import httpx
|
|
from tenacity import (
|
|
retry,
|
|
retry_if_exception_type,
|
|
stop_after_attempt,
|
|
wait_exponential,
|
|
)
|
|
|
|
from app.services.domains.openalex.types import OpenAlexWork
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
OPENALEX_BASE_URL = "https://api.openalex.org"
|
|
|
|
|
|
class OpenAlexClientError(Exception):
|
|
pass
|
|
|
|
|
|
class OpenAlexRateLimitError(OpenAlexClientError):
|
|
pass
|
|
|
|
|
|
class OpenAlexClient:
|
|
def __init__(
|
|
self,
|
|
api_key: str | None = None,
|
|
mailto: str | None = None,
|
|
timeout: float = 10.0,
|
|
) -> None:
|
|
self.api_key = api_key
|
|
self.mailto = mailto
|
|
self.timeout = timeout
|
|
|
|
@property
|
|
def _base_params(self) -> dict[str, str]:
|
|
params = {}
|
|
if self.mailto:
|
|
params["mailto"] = self.mailto
|
|
if self.api_key:
|
|
params["api_key"] = self.api_key
|
|
return params
|
|
|
|
@retry(
|
|
retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException, OpenAlexRateLimitError)),
|
|
stop=stop_after_attempt(3),
|
|
wait=wait_exponential(multiplier=1, min=2, max=10),
|
|
reraise=True,
|
|
)
|
|
async def get_work_by_doi(self, doi: str) -> OpenAlexWork | None:
|
|
"""Fetch a single work by DOI directly."""
|
|
clean_doi = doi.replace("https://doi.org/", "")
|
|
if not clean_doi:
|
|
return None
|
|
|
|
url = f"{OPENALEX_BASE_URL}/works/{clean_doi}"
|
|
|
|
headers = {}
|
|
if self.mailto:
|
|
headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
|
|
else:
|
|
headers["User-Agent"] = "scholar-scraper/1.0"
|
|
|
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client:
|
|
response = await client.get(url, params=self._base_params)
|
|
|
|
if response.status_code == 404:
|
|
return None
|
|
if response.status_code == 429:
|
|
raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI")
|
|
if response.status_code >= 400:
|
|
logger.warning("OpenAlex API error: %s %s", response.status_code, response.text)
|
|
raise OpenAlexClientError(f"API Error {response.status_code}")
|
|
|
|
data = response.json()
|
|
return OpenAlexWork.from_api_dict(data)
|
|
|
|
@retry(
|
|
retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException, OpenAlexRateLimitError)),
|
|
stop=stop_after_attempt(3),
|
|
wait=wait_exponential(multiplier=1, min=2, max=10),
|
|
reraise=True,
|
|
)
|
|
async def get_works_by_filter(
|
|
self,
|
|
filters: dict[str, str],
|
|
limit: int = 50,
|
|
) -> list[OpenAlexWork]:
|
|
"""
|
|
Fetch works using the ?filter= query parameter.
|
|
Supports fetching multiple records by joining filters with | (OR logic).
|
|
"""
|
|
if not filters:
|
|
return []
|
|
|
|
# Example: {"doi": "10.foo|10.bar", "title.search": "query"}
|
|
filter_str = ",".join(f"{k}:{v}" for k, v in filters.items())
|
|
|
|
params = self._base_params.copy()
|
|
params["filter"] = filter_str
|
|
params["per-page"] = str(limit)
|
|
|
|
url = f"{OPENALEX_BASE_URL}/works"
|
|
|
|
headers = {}
|
|
if self.mailto:
|
|
headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
|
|
else:
|
|
headers["User-Agent"] = "scholar-scraper/1.0"
|
|
|
|
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client:
|
|
response = await client.get(url, params=params)
|
|
|
|
if response.status_code == 429:
|
|
raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list")
|
|
if response.status_code >= 400:
|
|
logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text)
|
|
raise OpenAlexClientError(f"API Error {response.status_code}")
|
|
|
|
data = response.json()
|
|
results = data.get("results") or []
|
|
|
|
parsed_works = []
|
|
for raw_work in results:
|
|
try:
|
|
parsed_works.append(OpenAlexWork.from_api_dict(raw_work))
|
|
except Exception as e:
|
|
logger.warning("Failed to parse OpenAlex raw dict: %s", e)
|
|
continue
|
|
|
|
return parsed_works
|