ci: add CodeQL security scanning and Dependabot
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
ac002131d6
commit
3866c6d6f0
90 changed files with 40 additions and 1 deletions
5
app/services/crossref/__init__.py
Normal file
5
app/services/crossref/__init__.py
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.crossref.application import discover_doi_for_publication
|
||||
|
||||
__all__ = ["discover_doi_for_publication"]
|
||||
383
app/services/crossref/application.py
Normal file
383
app/services/crossref/application.py
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crossref.restful import Etiquette, Works
|
||||
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.doi.normalize import normalize_doi
|
||||
from app.settings import settings
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
TOKEN_RE = re.compile(r"[a-z0-9]+")
|
||||
NON_ALNUM_RE = re.compile(r"[^a-z0-9\\s]+")
|
||||
STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis"}
|
||||
_RATE_LOCK = threading.Lock()
|
||||
_LAST_REQUEST_AT = 0.0
|
||||
logger = logging.getLogger(__name__)
|
||||
STRICT_TITLE_MATCH_THRESHOLD = 0.75
|
||||
RELAXED_TITLE_MATCH_THRESHOLD = 0.85
|
||||
|
||||
|
||||
def _rate_limit_wait(min_interval_seconds: float) -> None:
|
||||
global _LAST_REQUEST_AT
|
||||
interval = max(float(min_interval_seconds), 0.0)
|
||||
with _RATE_LOCK:
|
||||
elapsed = time.monotonic() - _LAST_REQUEST_AT
|
||||
remaining = interval - elapsed
|
||||
if remaining > 0:
|
||||
time.sleep(remaining)
|
||||
_LAST_REQUEST_AT = time.monotonic()
|
||||
|
||||
|
||||
def _normalized_tokens(value: str) -> list[str]:
|
||||
lowered = value.lower().replace("’", "'").replace("“", '"').replace("”", '"')
|
||||
lowered = NON_ALNUM_RE.sub(" ", lowered)
|
||||
return [token for token in TOKEN_RE.findall(lowered) if len(token) >= 3]
|
||||
|
||||
|
||||
def _normalized_query(value: str) -> str:
|
||||
tokens = [token for token in _normalized_tokens(value) if token not in STOP_WORDS]
|
||||
if len(tokens) < 3:
|
||||
tokens = _normalized_tokens(value)
|
||||
if len(tokens) < 3:
|
||||
return ""
|
||||
return " ".join(tokens[:12]).strip()
|
||||
|
||||
|
||||
def _query_author(value: str) -> str | None:
|
||||
tokens = [token for token in value.strip().split() if token]
|
||||
if len(tokens) < 2:
|
||||
return None
|
||||
return " ".join(tokens[:2])[:64]
|
||||
|
||||
|
||||
def _author_surname(value: str) -> str | None:
|
||||
tokens = [token for token in value.strip().split() if token]
|
||||
if not tokens:
|
||||
return None
|
||||
return NON_ALNUM_RE.sub("", tokens[-1].lower()) or None
|
||||
|
||||
|
||||
def _query_filters(year: int | None) -> list[tuple[str, str] | None]:
|
||||
if year is None:
|
||||
return [None]
|
||||
return [
|
||||
(f"{year - 1}-01-01", f"{year + 1}-12-31"),
|
||||
(f"{year}-01-01", f"{year}-12-31"),
|
||||
None,
|
||||
]
|
||||
|
||||
|
||||
def _candidate_title(item: dict) -> str:
|
||||
titles = item.get("title")
|
||||
if isinstance(titles, list) and titles:
|
||||
return str(titles[0] or "")
|
||||
return str(item.get("title") or "")
|
||||
|
||||
|
||||
def _title_match_score(source: str, candidate: str) -> float:
|
||||
source_tokens = {token for token in _normalized_tokens(source) if len(token) >= 3}
|
||||
candidate_tokens = {token for token in _normalized_tokens(candidate) if len(token) >= 3}
|
||||
if not source_tokens or not candidate_tokens:
|
||||
return 0.0
|
||||
return len(source_tokens & candidate_tokens) / float(len(source_tokens))
|
||||
|
||||
|
||||
def _candidate_year(item: dict) -> int | None:
|
||||
issued = item.get("issued")
|
||||
if not isinstance(issued, dict):
|
||||
return None
|
||||
date_parts = issued.get("date-parts")
|
||||
if not isinstance(date_parts, list) or not date_parts:
|
||||
return None
|
||||
first = date_parts[0]
|
||||
if not isinstance(first, list) or not first:
|
||||
return None
|
||||
try:
|
||||
return int(first[0])
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _candidate_author_match(item: dict, surname: str | None) -> bool:
|
||||
if not surname:
|
||||
return True
|
||||
authors = item.get("author")
|
||||
if not isinstance(authors, list):
|
||||
return False
|
||||
for author in authors:
|
||||
if not isinstance(author, dict):
|
||||
continue
|
||||
family = NON_ALNUM_RE.sub("", str(author.get("family") or "").lower())
|
||||
if family and family == surname:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _candidate_rank(*, title: str, year: int | None, item: dict) -> tuple[float, str | None]:
|
||||
doi = normalize_doi(str(item.get("DOI") or ""))
|
||||
if doi is None:
|
||||
return 0.0, None
|
||||
score = _title_match_score(title, _candidate_title(item))
|
||||
candidate_year = _candidate_year(item)
|
||||
if year is not None and candidate_year is not None:
|
||||
if abs(year - candidate_year) > 1:
|
||||
return 0.0, None
|
||||
score += 0.1
|
||||
return score, doi
|
||||
|
||||
|
||||
def _year_delta(source_year: int | None, candidate_year: int | None) -> int | None:
|
||||
if source_year is None or candidate_year is None:
|
||||
return None
|
||||
return abs(int(source_year) - int(candidate_year))
|
||||
|
||||
|
||||
def _candidate_rank_relaxed(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
item: dict,
|
||||
author_surname: str | None,
|
||||
) -> tuple[float, str | None]:
|
||||
doi = normalize_doi(str(item.get("DOI") or ""))
|
||||
if doi is None:
|
||||
return 0.0, None
|
||||
score = _title_match_score(title, _candidate_title(item))
|
||||
if score <= 0:
|
||||
return 0.0, None
|
||||
candidate_year = _candidate_year(item)
|
||||
delta = _year_delta(year, candidate_year)
|
||||
if delta is not None:
|
||||
if delta <= 1:
|
||||
score += 0.05
|
||||
elif delta <= 3:
|
||||
score += 0.0
|
||||
elif delta <= 5:
|
||||
score -= 0.03
|
||||
else:
|
||||
score -= 0.08
|
||||
if _candidate_author_match(item, author_surname):
|
||||
score += 0.03
|
||||
return score, doi
|
||||
|
||||
|
||||
def _best_candidate_doi_strict(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
items: list[dict],
|
||||
author_surname: str | None,
|
||||
) -> str | None:
|
||||
best_score = 0.0
|
||||
best_doi: str | None = None
|
||||
best_year: int | None = None
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if not _candidate_author_match(item, author_surname):
|
||||
continue
|
||||
score, doi = _candidate_rank(title=title, year=year, item=item)
|
||||
candidate_year = _candidate_year(item)
|
||||
if doi is None or score < STRICT_TITLE_MATCH_THRESHOLD:
|
||||
continue
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_doi = doi
|
||||
best_year = candidate_year
|
||||
continue
|
||||
if abs(score - best_score) > 0.02:
|
||||
continue
|
||||
if best_year is None or candidate_year is None:
|
||||
continue
|
||||
if candidate_year < best_year:
|
||||
best_doi = doi
|
||||
best_year = candidate_year
|
||||
return best_doi
|
||||
|
||||
|
||||
def _best_candidate_doi_relaxed(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
items: list[dict],
|
||||
author_surname: str | None,
|
||||
) -> str | None:
|
||||
best_score = 0.0
|
||||
best_doi: str | None = None
|
||||
best_author_match = False
|
||||
best_delta: int | None = None
|
||||
best_year: int | None = None
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
score, doi = _candidate_rank_relaxed(
|
||||
title=title,
|
||||
year=year,
|
||||
item=item,
|
||||
author_surname=author_surname,
|
||||
)
|
||||
if doi is None or score < RELAXED_TITLE_MATCH_THRESHOLD:
|
||||
continue
|
||||
candidate_year = _candidate_year(item)
|
||||
candidate_author_match = _candidate_author_match(item, author_surname)
|
||||
candidate_delta = _year_delta(year, candidate_year)
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_doi = doi
|
||||
best_author_match = candidate_author_match
|
||||
best_delta = candidate_delta
|
||||
best_year = candidate_year
|
||||
continue
|
||||
if abs(score - best_score) > 0.02:
|
||||
continue
|
||||
if candidate_author_match and not best_author_match:
|
||||
best_doi = doi
|
||||
best_author_match = True
|
||||
best_delta = candidate_delta
|
||||
best_year = candidate_year
|
||||
continue
|
||||
if best_delta is None and candidate_delta is not None:
|
||||
best_doi = doi
|
||||
best_author_match = candidate_author_match
|
||||
best_delta = candidate_delta
|
||||
best_year = candidate_year
|
||||
continue
|
||||
if best_delta is not None and candidate_delta is not None and candidate_delta < best_delta:
|
||||
best_doi = doi
|
||||
best_author_match = candidate_author_match
|
||||
best_delta = candidate_delta
|
||||
best_year = candidate_year
|
||||
continue
|
||||
if best_year is None or candidate_year is None:
|
||||
continue
|
||||
if candidate_year < best_year:
|
||||
best_doi = doi
|
||||
best_author_match = candidate_author_match
|
||||
best_delta = candidate_delta
|
||||
best_year = candidate_year
|
||||
return best_doi
|
||||
|
||||
|
||||
def _best_candidate_doi(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
items: list[dict],
|
||||
author_surname: str | None,
|
||||
) -> str | None:
|
||||
strict_match = _best_candidate_doi_strict(
|
||||
title=title,
|
||||
year=year,
|
||||
items=items,
|
||||
author_surname=author_surname,
|
||||
)
|
||||
if strict_match:
|
||||
return strict_match
|
||||
return _best_candidate_doi_relaxed(
|
||||
title=title,
|
||||
year=year,
|
||||
items=items,
|
||||
author_surname=author_surname,
|
||||
)
|
||||
|
||||
|
||||
def _works_client(email: str | None) -> Works:
|
||||
if email:
|
||||
etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email)
|
||||
return Works(etiquette=etiquette)
|
||||
return Works()
|
||||
|
||||
|
||||
def _fetch_items_sync(
|
||||
*,
|
||||
query: str,
|
||||
author: str | None,
|
||||
date_range: tuple[str, str] | None,
|
||||
max_rows: int,
|
||||
email: str | None,
|
||||
min_interval_seconds: float,
|
||||
) -> list[dict]:
|
||||
_rate_limit_wait(min_interval_seconds)
|
||||
works = _works_client(email)
|
||||
params = {"bibliographic": query}
|
||||
if author:
|
||||
params["author"] = author
|
||||
request = works.query(**params)
|
||||
if date_range is not None:
|
||||
from_date, until_date = date_range
|
||||
request = request.filter(from_pub_date=from_date, until_pub_date=until_date)
|
||||
request = request.select(["DOI", "title", "issued", "score", "author"])
|
||||
items: list[dict] = []
|
||||
for entry in request:
|
||||
if isinstance(entry, dict):
|
||||
items.append(entry)
|
||||
if len(items) >= max(max_rows, 1):
|
||||
break
|
||||
return items
|
||||
|
||||
|
||||
async def _fetch_items(
|
||||
*,
|
||||
query: str,
|
||||
author: str | None,
|
||||
date_range: tuple[str, str] | None,
|
||||
max_rows: int,
|
||||
email: str | None,
|
||||
) -> list[dict]:
|
||||
timeout = max(float(settings.crossref_timeout_seconds), 0.5)
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
asyncio.to_thread(
|
||||
_fetch_items_sync,
|
||||
query=query,
|
||||
author=author,
|
||||
date_range=date_range,
|
||||
max_rows=max_rows,
|
||||
email=email,
|
||||
min_interval_seconds=settings.crossref_min_interval_seconds,
|
||||
),
|
||||
timeout=timeout,
|
||||
)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
async def discover_doi_for_publication(
|
||||
*,
|
||||
item: PublicationListItem | UnreadPublicationItem,
|
||||
max_rows: int = 10,
|
||||
email: str | None = None,
|
||||
) -> str | None:
|
||||
title = (item.title or "").strip()
|
||||
query = _normalized_query(title)
|
||||
if not query:
|
||||
return None
|
||||
author = _query_author(item.scholar_label)
|
||||
author_surname = _author_surname(item.scholar_label)
|
||||
for date_range in _query_filters(item.year):
|
||||
items = await _fetch_items(
|
||||
query=query,
|
||||
author=author,
|
||||
date_range=date_range,
|
||||
max_rows=max_rows,
|
||||
email=email,
|
||||
)
|
||||
doi = _best_candidate_doi(
|
||||
title=title,
|
||||
year=item.year,
|
||||
items=items,
|
||||
author_surname=author_surname,
|
||||
)
|
||||
if doi:
|
||||
structured_log(logger, "debug", "crossref.doi_discovered")
|
||||
return doi
|
||||
return None
|
||||
Loading…
Add table
Add a link
Reference in a new issue