justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

decompose application.py into enrichment, scholar processing, run completion

Browse source
This commit is contained in:
Justin Visser 2026-02-27 14:19:39 +01:00
parent 9a7fa5004e
commit 6379c97e90
28 changed files with 2982 additions and 2576 deletions
Download patch file Download diff file Expand all files Collapse all files

8
tests/unit/test_ingestion_arxiv_rate_limit.py
View file

@ -5,7 +5,7 @@ from types import SimpleNamespace
import pytest
from app.services.arxiv.errors import ArxivRateLimitError
from app.services.ingestion.application import ScholarIngestionService
from app.services.ingestion.enrichment import EnrichmentRunner
from app.services.publication_identifiers import application as identifier_service
@ -13,7 +13,7 @@ from app.services.publication_identifiers import application as identifier_servi
async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
service = ScholarIngestionService(source=object())
runner = EnrichmentRunner()
publication = SimpleNamespace(id=11, author_text="Ada Lovelace")
calls = {"sync": 0}
@ -35,9 +35,9 @@ async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit(
async def _publish_noop(*args, **kwargs) -> None:
_ = (args, kwargs)
monkeypatch.setattr(service, "_publish_identifier_update_event", _publish_noop)
monkeypatch.setattr(runner, "_publish_identifier_update_event", _publish_noop)
result = await service._discover_identifiers_for_enrichment(
result = await runner._discover_identifiers_for_enrichment(
object(),
publication=publication,
run_id=321,

8
tests/unit/test_ingestion_progress_reporting.py
View file

@ -2,7 +2,7 @@ from __future__ import annotations
import pytest
from app.services.ingestion.application import ScholarIngestionService
from app.services.ingestion.run_completion import apply_outcome_to_progress
from app.services.ingestion.types import RunProgress, ScholarProcessingOutcome
@ -31,11 +31,11 @@ def _outcome(*, scholar_profile_id: int, outcome_label: str) -> ScholarProcessin
def test_apply_outcome_to_progress_replaces_previous_scholar_outcome() -> None:
progress = RunProgress()
ScholarIngestionService._apply_outcome_to_progress(
apply_outcome_to_progress(
progress=progress,
outcome=_outcome(scholar_profile_id=42, outcome_label="partial"),
)
ScholarIngestionService._apply_outcome_to_progress(
apply_outcome_to_progress(
progress=progress,
outcome=_outcome(scholar_profile_id=42, outcome_label="success"),
)
@ -58,4 +58,4 @@ def test_apply_outcome_to_progress_rejects_invalid_scholar_id() -> None:
)
with pytest.raises(RuntimeError, match="missing valid scholar_profile_id"):
ScholarIngestionService._apply_outcome_to_progress(progress=progress, outcome=invalid)
apply_outcome_to_progress(progress=progress, outcome=invalid)

15
tests/unit/test_scholar_parser.py
View file

@ -433,3 +433,18 @@ def test_parse_author_search_page_classifies_http_429_as_blocked() -> None:
assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
assert parsed.state_reason == "blocked_http_429_rate_limited"
def test_parse_author_search_page_prefers_sorry_challenge_reason_over_generic_429() -> None:
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
status_code=429,
final_url="https://www.google.com/sorry/index?continue=scholar",
body="<html><body>Our systems have detected unusual traffic.</body></html>",
error=None,
)
parsed = parse_author_search_page(fetch_result)
assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
assert parsed.state_reason == "blocked_google_sorry_challenge"

75
tests/unit/test_scholar_source.py
View file

@ -2,6 +2,12 @@ import pytest
from app.services.scholar import rate_limit as scholar_rate_limit
from app.services.scholar.source import FetchResult, LiveScholarSource, _build_profile_url
from app.settings import settings
def _request_header(request, name: str) -> str | None:
headers = {key.lower(): value for key, value in request.header_items()}
return headers.get(name.lower())
def test_build_profile_url_includes_pagesize_for_initial_page() -> None:
@ -48,3 +54,72 @@ async def test_live_scholar_source_applies_global_throttle(monkeypatch: pytest.M
assert result == expected_result
assert captured_interval == [7.0]
def test_http_error_reason_prefers_sorry_challenge_marker() -> None:
reason = LiveScholarSource._http_error_reason(
status_code=429,
final_url="https://www.google.com/sorry/index?continue=example",
body="",
)
assert reason == "blocked_google_sorry_challenge"
def test_http_error_reason_falls_back_to_http_429_rate_limit() -> None:
reason = LiveScholarSource._http_error_reason(
status_code=429,
final_url="https://scholar.google.com/citations?hl=en&user=abc",
body="Too Many Requests",
)
assert reason == "blocked_http_429_rate_limited"
def test_build_request_uses_stable_user_agent_by_default() -> None:
previous_user_agent = settings.scholar_http_user_agent
previous_rotate = settings.scholar_http_rotate_user_agent
previous_cookie = settings.scholar_http_cookie
previous_accept_language = settings.scholar_http_accept_language
object.__setattr__(settings, "scholar_http_user_agent", "")
object.__setattr__(settings, "scholar_http_rotate_user_agent", False)
object.__setattr__(settings, "scholar_http_cookie", "")
object.__setattr__(settings, "scholar_http_accept_language", "en-US,en;q=0.9")
try:
source = LiveScholarSource(user_agents=["UA-A", "UA-B"])
request_one = source._build_request("https://example.test/one")
request_two = source._build_request("https://example.test/two")
assert _request_header(request_one, "User-Agent") == _request_header(request_two, "User-Agent")
assert _request_header(request_one, "Connection") is None
finally:
object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
object.__setattr__(settings, "scholar_http_cookie", previous_cookie)
object.__setattr__(settings, "scholar_http_accept_language", previous_accept_language)
def test_build_request_rotates_user_agent_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
previous_user_agent = settings.scholar_http_user_agent
previous_rotate = settings.scholar_http_rotate_user_agent
object.__setattr__(settings, "scholar_http_user_agent", "")
object.__setattr__(settings, "scholar_http_rotate_user_agent", True)
choices = iter(["UA-STABLE", "UA-1", "UA-2"])
monkeypatch.setattr("app.services.scholar.source.random.choice", lambda _values: next(choices))
try:
source = LiveScholarSource(user_agents=["UA-A", "UA-B"])
request_one = source._build_request("https://example.test/one")
request_two = source._build_request("https://example.test/two")
assert _request_header(request_one, "User-Agent") == "UA-1"
assert _request_header(request_two, "User-Agent") == "UA-2"
finally:
object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
def test_build_request_includes_cookie_when_configured() -> None:
previous_cookie = settings.scholar_http_cookie
object.__setattr__(settings, "scholar_http_cookie", "SID=abc123")
try:
source = LiveScholarSource(user_agents=["UA-A"])
request = source._build_request("https://example.test/one")
assert _request_header(request, "Cookie") == "SID=abc123"
finally:
object.__setattr__(settings, "scholar_http_cookie", previous_cookie)