decompose application.py into enrichment, scholar processing, run completion

tests/integration/test_api_v1.py

@@ -391,6 +391,61 @@ async def test_api_admin_user_management_endpoints(db_session: AsyncSession) ->
     assert created_exists.scalar_one() == 1
 
 
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.asyncio
+async def test_api_admin_scholar_http_settings_endpoints(db_session: AsyncSession) -> None:
+    await insert_user(
+        db_session,
+        email="api-admin-http@example.com",
+        password="admin-password",
+        is_admin=True,
+    )
+    await insert_user(
+        db_session,
+        email="api-member-http@example.com",
+        password="member-password",
+        is_admin=False,
+    )
+    previous_user_agent = settings.scholar_http_user_agent
+    previous_rotate = settings.scholar_http_rotate_user_agent
+    previous_accept_language = settings.scholar_http_accept_language
+    previous_cookie = settings.scholar_http_cookie
+    try:
+        client = TestClient(app)
+        login_user(client, email="api-admin-http@example.com", password="admin-password")
+        headers = _api_csrf_headers(client)
+
+        read_response = client.get("/api/v1/admin/settings/scholar-http")
+        assert read_response.status_code == 200
+
+        update_response = client.put(
+            "/api/v1/admin/settings/scholar-http",
+            json={
+                "user_agent": "Mozilla/5.0 Test Runner",
+                "rotate_user_agent": False,
+                "accept_language": "en-US,en;q=0.8",
+                "cookie": "SID=test-cookie",
+            },
+            headers=headers,
+        )
+        assert update_response.status_code == 200
+        payload = update_response.json()["data"]
+        assert payload["user_agent"] == "Mozilla/5.0 Test Runner"
+        assert payload["cookie"] == "SID=test-cookie"
+        assert settings.scholar_http_user_agent == "Mozilla/5.0 Test Runner"
+
+        client.post("/api/v1/auth/logout", headers=headers)
+        login_user(client, email="api-member-http@example.com", password="member-password")
+        forbidden_response = client.get("/api/v1/admin/settings/scholar-http")
+        assert forbidden_response.status_code == 403
+    finally:
+        object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
+        object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
+        object.__setattr__(settings, "scholar_http_accept_language", previous_accept_language)
+        object.__setattr__(settings, "scholar_http_cookie", previous_cookie)
+
+
 @pytest.mark.integration
 @pytest.mark.db
 @pytest.mark.asyncio

tests/integration/test_deferred_enrichment.py

@@ -8,7 +8,7 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.models import CrawlRun, Publication, RunStatus, RunTriggerType, ScholarProfile, ScholarPublication
-from app.services.ingestion.application import ScholarIngestionService
+from app.services.ingestion.enrichment import EnrichmentRunner
 from app.services.openalex.types import OpenAlexWork
 from tests.integration.helpers import insert_user
 
@@ -79,8 +79,7 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession
         oa_url="http://example.com/grover.pdf",
     )
 
-    mock_source = MagicMock()
-    service = ScholarIngestionService(source=mock_source)
+    runner = EnrichmentRunner()
 
     # We patch the client at its source, and also mock arXiv to avoid real HTTP calls
     with (
@@ -93,7 +92,7 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession
         mock_instance.get_works_by_filter = AsyncMock(return_value=[mock_work])
 
         # 5. Execute the enrichment pass for the NEW run
-        await service._enrich_pending_publications(db_session, run_id=new_run.id)
+        await runner.enrich_pending_publications(db_session, run_id=new_run.id)
         await db_session.commit()
 
     # 6. Verification: The publication from the FAILED run should now be enriched

tests/unit/test_ingestion_arxiv_rate_limit.py

@@ -5,7 +5,7 @@ from types import SimpleNamespace
 import pytest
 
 from app.services.arxiv.errors import ArxivRateLimitError
-from app.services.ingestion.application import ScholarIngestionService
+from app.services.ingestion.enrichment import EnrichmentRunner
 from app.services.publication_identifiers import application as identifier_service
 
 
@@ -13,7 +13,7 @@ from app.services.publication_identifiers import application as identifier_servi
 async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    service = ScholarIngestionService(source=object())
+    runner = EnrichmentRunner()
     publication = SimpleNamespace(id=11, author_text="Ada Lovelace")
     calls = {"sync": 0}
 
@@ -35,9 +35,9 @@ async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit(
     async def _publish_noop(*args, **kwargs) -> None:
         _ = (args, kwargs)
 
-    monkeypatch.setattr(service, "_publish_identifier_update_event", _publish_noop)
+    monkeypatch.setattr(runner, "_publish_identifier_update_event", _publish_noop)
 
-    result = await service._discover_identifiers_for_enrichment(
+    result = await runner._discover_identifiers_for_enrichment(
         object(),
         publication=publication,
         run_id=321,

tests/unit/test_ingestion_progress_reporting.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import pytest
 
-from app.services.ingestion.application import ScholarIngestionService
+from app.services.ingestion.run_completion import apply_outcome_to_progress
 from app.services.ingestion.types import RunProgress, ScholarProcessingOutcome
 
 
@@ -31,11 +31,11 @@ def _outcome(*, scholar_profile_id: int, outcome_label: str) -> ScholarProcessin
 def test_apply_outcome_to_progress_replaces_previous_scholar_outcome() -> None:
     progress = RunProgress()
 
-    ScholarIngestionService._apply_outcome_to_progress(
+    apply_outcome_to_progress(
         progress=progress,
         outcome=_outcome(scholar_profile_id=42, outcome_label="partial"),
     )
-    ScholarIngestionService._apply_outcome_to_progress(
+    apply_outcome_to_progress(
         progress=progress,
         outcome=_outcome(scholar_profile_id=42, outcome_label="success"),
     )
@@ -58,4 +58,4 @@ def test_apply_outcome_to_progress_rejects_invalid_scholar_id() -> None:
     )
 
     with pytest.raises(RuntimeError, match="missing valid scholar_profile_id"):
-        ScholarIngestionService._apply_outcome_to_progress(progress=progress, outcome=invalid)
+        apply_outcome_to_progress(progress=progress, outcome=invalid)

tests/unit/test_scholar_parser.py

@@ -433,3 +433,18 @@ def test_parse_author_search_page_classifies_http_429_as_blocked() -> None:
 
     assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
     assert parsed.state_reason == "blocked_http_429_rate_limited"
+
+
+def test_parse_author_search_page_prefers_sorry_challenge_reason_over_generic_429() -> None:
+    fetch_result = FetchResult(
+        requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
+        status_code=429,
+        final_url="https://www.google.com/sorry/index?continue=scholar",
+        body="<html><body>Our systems have detected unusual traffic.</body></html>",
+        error=None,
+    )
+
+    parsed = parse_author_search_page(fetch_result)
+
+    assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
+    assert parsed.state_reason == "blocked_google_sorry_challenge"

tests/unit/test_scholar_source.py

@@ -2,6 +2,12 @@ import pytest
 
 from app.services.scholar import rate_limit as scholar_rate_limit
 from app.services.scholar.source import FetchResult, LiveScholarSource, _build_profile_url
+from app.settings import settings
+
+
+def _request_header(request, name: str) -> str | None:
+    headers = {key.lower(): value for key, value in request.header_items()}
+    return headers.get(name.lower())
 
 
 def test_build_profile_url_includes_pagesize_for_initial_page() -> None:
@@ -48,3 +54,72 @@ async def test_live_scholar_source_applies_global_throttle(monkeypatch: pytest.M
 
     assert result == expected_result
     assert captured_interval == [7.0]
+
+
+def test_http_error_reason_prefers_sorry_challenge_marker() -> None:
+    reason = LiveScholarSource._http_error_reason(
+        status_code=429,
+        final_url="https://www.google.com/sorry/index?continue=example",
+        body="",
+    )
+    assert reason == "blocked_google_sorry_challenge"
+
+
+def test_http_error_reason_falls_back_to_http_429_rate_limit() -> None:
+    reason = LiveScholarSource._http_error_reason(
+        status_code=429,
+        final_url="https://scholar.google.com/citations?hl=en&user=abc",
+        body="Too Many Requests",
+    )
+    assert reason == "blocked_http_429_rate_limited"
+
+
+def test_build_request_uses_stable_user_agent_by_default() -> None:
+    previous_user_agent = settings.scholar_http_user_agent
+    previous_rotate = settings.scholar_http_rotate_user_agent
+    previous_cookie = settings.scholar_http_cookie
+    previous_accept_language = settings.scholar_http_accept_language
+    object.__setattr__(settings, "scholar_http_user_agent", "")
+    object.__setattr__(settings, "scholar_http_rotate_user_agent", False)
+    object.__setattr__(settings, "scholar_http_cookie", "")
+    object.__setattr__(settings, "scholar_http_accept_language", "en-US,en;q=0.9")
+    try:
+        source = LiveScholarSource(user_agents=["UA-A", "UA-B"])
+        request_one = source._build_request("https://example.test/one")
+        request_two = source._build_request("https://example.test/two")
+        assert _request_header(request_one, "User-Agent") == _request_header(request_two, "User-Agent")
+        assert _request_header(request_one, "Connection") is None
+    finally:
+        object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
+        object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
+        object.__setattr__(settings, "scholar_http_cookie", previous_cookie)
+        object.__setattr__(settings, "scholar_http_accept_language", previous_accept_language)
+
+
+def test_build_request_rotates_user_agent_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
+    previous_user_agent = settings.scholar_http_user_agent
+    previous_rotate = settings.scholar_http_rotate_user_agent
+    object.__setattr__(settings, "scholar_http_user_agent", "")
+    object.__setattr__(settings, "scholar_http_rotate_user_agent", True)
+    choices = iter(["UA-STABLE", "UA-1", "UA-2"])
+    monkeypatch.setattr("app.services.scholar.source.random.choice", lambda _values: next(choices))
+    try:
+        source = LiveScholarSource(user_agents=["UA-A", "UA-B"])
+        request_one = source._build_request("https://example.test/one")
+        request_two = source._build_request("https://example.test/two")
+        assert _request_header(request_one, "User-Agent") == "UA-1"
+        assert _request_header(request_two, "User-Agent") == "UA-2"
+    finally:
+        object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
+        object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
+
+
+def test_build_request_includes_cookie_when_configured() -> None:
+    previous_cookie = settings.scholar_http_cookie
+    object.__setattr__(settings, "scholar_http_cookie", "SID=abc123")
+    try:
+        source = LiveScholarSource(user_agents=["UA-A"])
+        request = source._build_request("https://example.test/one")
+        assert _request_header(request, "Cookie") == "SID=abc123"
+    finally:
+        object.__setattr__(settings, "scholar_http_cookie", previous_cookie)