justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

decompose application.py into enrichment, scholar processing, run completion

Browse source
This commit is contained in:
Justin Visser 2026-02-27 14:19:39 +01:00
parent 9a7fa5004e
commit 6379c97e90
28 changed files with 2982 additions and 2576 deletions
Download patch file Download diff file Expand all files Collapse all files

75
tests/unit/test_scholar_source.py
View file

@ -2,6 +2,12 @@ import pytest
from app.services.scholar import rate_limit as scholar_rate_limit
from app.services.scholar.source import FetchResult, LiveScholarSource, _build_profile_url
from app.settings import settings
def _request_header(request, name: str) -> str | None:
headers = {key.lower(): value for key, value in request.header_items()}
return headers.get(name.lower())
def test_build_profile_url_includes_pagesize_for_initial_page() -> None:
@ -48,3 +54,72 @@ async def test_live_scholar_source_applies_global_throttle(monkeypatch: pytest.M
assert result == expected_result
assert captured_interval == [7.0]
def test_http_error_reason_prefers_sorry_challenge_marker() -> None:
reason = LiveScholarSource._http_error_reason(
status_code=429,
final_url="https://www.google.com/sorry/index?continue=example",
body="",
)
assert reason == "blocked_google_sorry_challenge"
def test_http_error_reason_falls_back_to_http_429_rate_limit() -> None:
reason = LiveScholarSource._http_error_reason(
status_code=429,
final_url="https://scholar.google.com/citations?hl=en&user=abc",
body="Too Many Requests",
)
assert reason == "blocked_http_429_rate_limited"
def test_build_request_uses_stable_user_agent_by_default() -> None:
previous_user_agent = settings.scholar_http_user_agent
previous_rotate = settings.scholar_http_rotate_user_agent
previous_cookie = settings.scholar_http_cookie
previous_accept_language = settings.scholar_http_accept_language
object.__setattr__(settings, "scholar_http_user_agent", "")
object.__setattr__(settings, "scholar_http_rotate_user_agent", False)
object.__setattr__(settings, "scholar_http_cookie", "")
object.__setattr__(settings, "scholar_http_accept_language", "en-US,en;q=0.9")
try:
source = LiveScholarSource(user_agents=["UA-A", "UA-B"])
request_one = source._build_request("https://example.test/one")
request_two = source._build_request("https://example.test/two")
assert _request_header(request_one, "User-Agent") == _request_header(request_two, "User-Agent")
assert _request_header(request_one, "Connection") is None
finally:
object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
object.__setattr__(settings, "scholar_http_cookie", previous_cookie)
object.__setattr__(settings, "scholar_http_accept_language", previous_accept_language)
def test_build_request_rotates_user_agent_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
previous_user_agent = settings.scholar_http_user_agent
previous_rotate = settings.scholar_http_rotate_user_agent
object.__setattr__(settings, "scholar_http_user_agent", "")
object.__setattr__(settings, "scholar_http_rotate_user_agent", True)
choices = iter(["UA-STABLE", "UA-1", "UA-2"])
monkeypatch.setattr("app.services.scholar.source.random.choice", lambda _values: next(choices))
try:
source = LiveScholarSource(user_agents=["UA-A", "UA-B"])
request_one = source._build_request("https://example.test/one")
request_two = source._build_request("https://example.test/two")
assert _request_header(request_one, "User-Agent") == "UA-1"
assert _request_header(request_two, "User-Agent") == "UA-2"
finally:
object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
def test_build_request_includes_cookie_when_configured() -> None:
previous_cookie = settings.scholar_http_cookie
object.__setattr__(settings, "scholar_http_cookie", "SID=abc123")
try:
source = LiveScholarSource(user_agents=["UA-A"])
request = source._build_request("https://example.test/one")
assert _request_header(request, "Cookie") == "SID=abc123"
finally:
object.__setattr__(settings, "scholar_http_cookie", previous_cookie)