harden domain architecture and add lazy Crossref+Unpaywall PDF enrichment with publications workspace refactor

tests/integration/test_api_v1.py

@@ -10,8 +10,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.api.runtime_deps import get_scholar_source
 from app.main import app
-from app.services import user_settings as user_settings_service
-from app.services.scholar_source import FetchResult
+from app.services.domains.settings import application as user_settings_service
+from app.services.domains.scholar.source import FetchResult
 from app.settings import settings
 from tests.integration.helpers import insert_user, login_user
 
@@ -1465,6 +1465,182 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
     assert mark_response.json()["data"]["updated_count"] == 1
 
 
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.asyncio
+async def test_api_publications_list_schedules_background_enrichment(
+    db_session: AsyncSession,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    user_id = await insert_user(
+        db_session,
+        email="api-pubs-bg@example.com",
+        password="api-password",
+    )
+    scholar_result = await db_session.execute(
+        text(
+            """
+            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
+            VALUES (:user_id, :scholar_id, :display_name, true)
+            RETURNING id
+            """
+        ),
+        {
+            "user_id": user_id,
+            "scholar_id": "background111",
+            "display_name": "Background Scholar",
+        },
+    )
+    scholar_profile_id = int(scholar_result.scalar_one())
+    publication_result = await db_session.execute(
+        text(
+            """
+            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
+            VALUES (:fingerprint, :title_raw, :title_normalized, 3)
+            RETURNING id
+            """
+        ),
+        {
+            "fingerprint": f"{(user_id + 17):064x}",
+            "title_raw": "Background Target",
+            "title_normalized": "background target",
+        },
+    )
+    publication_id = int(publication_result.scalar_one())
+    await db_session.execute(
+        text(
+            """
+            INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
+            VALUES (:scholar_profile_id, :publication_id, false)
+            """
+        ),
+        {
+            "scholar_profile_id": scholar_profile_id,
+            "publication_id": publication_id,
+        },
+    )
+    await db_session.commit()
+
+    async def _fail_resolver(*args, **kwargs):
+        raise AssertionError("list endpoint should not resolve OA metadata inline")
+
+    async def _fake_schedule(*, user_id: int, request_email: str | None, items, max_items: int):
+        assert user_id > 0
+        assert request_email == "api-pubs-bg@example.com"
+        assert int(max_items) > 0
+        assert any(int(item.publication_id) == publication_id for item in items)
+        return 1
+
+    monkeypatch.setattr(
+        "app.services.domains.publications.listing.resolve_publication_oa_metadata",
+        _fail_resolver,
+    )
+    monkeypatch.setattr(
+        "app.services.domains.publications.application.schedule_missing_pdf_enrichment_for_user",
+        _fake_schedule,
+    )
+
+    client = TestClient(app)
+    login_user(client, email="api-pubs-bg@example.com", password="api-password")
+
+    response = client.get("/api/v1/publications?mode=all")
+    assert response.status_code == 200
+    data = response.json()["data"]
+    assert len(data["publications"]) == 1
+    assert int(data["publications"][0]["publication_id"]) == publication_id
+    assert data["publications"][0]["pdf_url"] is None
+
+
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.asyncio
+async def test_api_publication_retry_pdf_updates_publication(
+    db_session: AsyncSession,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    user_id = await insert_user(
+        db_session,
+        email="api-pubs-retry@example.com",
+        password="api-password",
+    )
+    scholar_result = await db_session.execute(
+        text(
+            """
+            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
+            VALUES (:user_id, :scholar_id, :display_name, true)
+            RETURNING id
+            """
+        ),
+        {
+            "user_id": user_id,
+            "scholar_id": "retryScholar01",
+            "display_name": "Retry Scholar",
+        },
+    )
+    scholar_profile_id = int(scholar_result.scalar_one())
+    publication_result = await db_session.execute(
+        text(
+            """
+            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
+            VALUES (:fingerprint, :title_raw, :title_normalized, 7)
+            RETURNING id
+            """
+        ),
+        {
+            "fingerprint": f"{(user_id + 9):064x}",
+            "title_raw": "Retry Target",
+            "title_normalized": "retry target",
+        },
+    )
+    publication_id = int(publication_result.scalar_one())
+    await db_session.execute(
+        text(
+            """
+            INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
+            VALUES (:scholar_profile_id, :publication_id, false)
+            """
+        ),
+        {
+            "scholar_profile_id": scholar_profile_id,
+            "publication_id": publication_id,
+        },
+    )
+    await db_session.commit()
+
+    async def _fake_resolver(items, *, request_email=None):
+        assert request_email == "api-pubs-retry@example.com"
+        return {int(items[0].publication_id): ("10.1000/retry-target", "https://oa.example/retry.pdf")}
+
+    monkeypatch.setattr(
+        "app.services.domains.publications.listing.resolve_publication_oa_metadata",
+        _fake_resolver,
+    )
+
+    client = TestClient(app)
+    login_user(client, email="api-pubs-retry@example.com", password="api-password")
+    headers = _api_csrf_headers(client)
+
+    response = client.post(
+        f"/api/v1/publications/{publication_id}/retry-pdf",
+        json={"scholar_profile_id": scholar_profile_id},
+        headers=headers,
+    )
+    assert response.status_code == 200
+    payload = response.json()["data"]
+    assert payload["resolved_pdf"] is True
+    assert payload["publication"]["publication_id"] == publication_id
+    assert payload["publication"]["pdf_url"] == "https://oa.example/retry.pdf"
+    assert payload["publication"]["doi"] == "10.1000/retry-target"
+
+    stored = await db_session.execute(
+        text("SELECT doi, pdf_url FROM publications WHERE id = :publication_id"),
+        {"publication_id": publication_id},
+    )
+    stored_doi, stored_pdf_url = stored.one()
+    assert stored_doi == "10.1000/retry-target"
+    assert stored_pdf_url == "https://oa.example/retry.pdf"
+
+
 @pytest.mark.integration
 @pytest.mark.db
 @pytest.mark.asyncio

tests/integration/test_fixture_probe_runs.py

@@ -8,8 +8,8 @@ from sqlalchemy import text
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.models import RunStatus, RunTriggerType
-from app.services.ingestion import ScholarIngestionService
-from app.services.scholar_source import FetchResult
+from app.services.domains.ingestion.application import ScholarIngestionService
+from app.services.domains.scholar.source import FetchResult
 from tests.integration.helpers import insert_user
 
 REGRESSION_FIXTURE_DIR = Path("tests/fixtures/scholar/regression")

tests/integration/test_migrations.py

@@ -16,7 +16,7 @@ EXPECTED_TABLES = {
 }
 
 EXPECTED_ENUMS = {"run_status", "run_trigger_type"}
-EXPECTED_REVISION = "20260219_0010"
+EXPECTED_REVISION = "20260220_0011"
 
 
 @pytest.mark.integration

tests/unit/test_crossref_lookup.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from app.services.domains.crossref import application as crossref_app
+
+
+def _item(*, title: str, year: int | None, scholar_label: str = "Shinya Yamanaka"):
+    return SimpleNamespace(
+        publication_id=1,
+        scholar_label=scholar_label,
+        title=title,
+        year=year,
+    )
+
+
+@pytest.mark.asyncio
+async def test_crossref_discovers_doi_from_best_title_match(monkeypatch: pytest.MonkeyPatch) -> None:
+    responses = [
+        [],
+        [
+            {
+                "DOI": "10.1000/noisy",
+                "title": ["Completely unrelated paper"],
+                "issued": {"date-parts": [[2014]]},
+                "author": [{"family": "Other"}],
+            },
+            {
+                "DOI": "10.1016/j.cell.2007.11.019",
+                "title": ["Induction of Pluripotent Stem Cells from Adult Human Fibroblasts"],
+                "issued": {"date-parts": [[2007]]},
+                "author": [{"family": "Yamanaka"}],
+            },
+        ],
+    ]
+
+    async def _fake_fetch_items(**_kwargs):
+        return responses.pop(0) if responses else []
+
+    monkeypatch.setattr(crossref_app, "_fetch_items", _fake_fetch_items)
+    doi = await crossref_app.discover_doi_for_publication(
+        item=_item(
+            title="Induction of Pluripotent Stem Cells from Adult Human Fibroblasts",
+            year=2007,
+        ),
+        max_rows=10,
+        email="user@example.com",
+    )
+    assert doi == "10.1016/j.cell.2007.11.019"
+
+
+@pytest.mark.asyncio
+async def test_crossref_rejects_large_year_mismatch(monkeypatch: pytest.MonkeyPatch) -> None:
+    async def _fake_fetch_items(**_kwargs):
+        return [
+            {
+                "DOI": "10.1000/wrong-year",
+                "title": ["Induction of Pluripotent Stem Cells from Adult Human Fibroblasts"],
+                "issued": {"date-parts": [[2014]]},
+                "author": [{"family": "Yamanaka"}],
+            }
+        ]
+
+    monkeypatch.setattr(crossref_app, "_fetch_items", _fake_fetch_items)
+    doi = await crossref_app.discover_doi_for_publication(
+        item=_item(
+            title="Induction of Pluripotent Stem Cells from Adult Human Fibroblasts",
+            year=2007,
+        ),
+        max_rows=10,
+        email=None,
+    )
+    assert doi is None

tests/unit/test_doi_normalize.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from app.services.domains.doi.normalize import first_doi_from_texts, normalize_doi
+
+
+def test_normalize_doi_extracts_and_lowercases() -> None:
+    value = "https://doi.org/10.48550/ARXIV.1412.6980"
+    assert normalize_doi(value) == "10.48550/arxiv.1412.6980"
+
+
+def test_first_doi_from_texts_prefers_first_match() -> None:
+    result = first_doi_from_texts(
+        "no doi here",
+        "venue text 10.1000/ABC-123",
+        "title with 10.9999/ignored",
+    )
+    assert result == "10.1000/abc-123"

tests/unit/test_run_safety.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from datetime import datetime, timedelta, timezone
 
 from app.db.models import UserSetting
-from app.services import run_safety
+from app.services.domains.ingestion import safety as run_safety
 
 
 def test_apply_run_safety_outcome_triggers_blocked_cooldown() -> None:

tests/unit/test_runs_summary.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from app.services.runs import extract_run_summary
+from app.services.domains.runs.application import extract_run_summary
 
 
 def test_extract_run_summary_includes_extended_metrics() -> None:

tests/unit/test_scholar_parser.py

@@ -2,12 +2,15 @@ from __future__ import annotations
 
 from pathlib import Path
 
-from app.services.scholar_parser import (
+import pytest
+
+from app.services.domains.scholar.parser import (
     ParseState,
+    ScholarDomInvariantError,
     parse_author_search_page,
     parse_profile_page,
 )
-from app.services.scholar_source import FetchResult
+from app.services.domains.scholar.source import FetchResult
 
 
 def _fixture(name: str) -> str:
@@ -98,7 +101,7 @@ def test_parse_profile_page_handles_missing_optional_metadata() -> None:
     assert publication.venue_text is None
 
 
-def test_parse_profile_page_extracts_direct_pdf_link() -> None:
+def test_parse_profile_page_ignores_direct_pdf_link_markup() -> None:
     html = """
     <html>
       <div id="gsc_prf_in">Direct PDF Test</div>
@@ -131,7 +134,7 @@ def test_parse_profile_page_extracts_direct_pdf_link() -> None:
 
     assert parsed.state == ParseState.OK
     assert len(parsed.publications) == 1
-    assert parsed.publications[0].pdf_url == "https://example.org/paper.pdf"
+    assert parsed.publications[0].pdf_url is None
 
 
 def test_parse_profile_page_fails_fast_when_citation_markup_is_unparseable() -> None:
@@ -160,10 +163,9 @@ def test_parse_profile_page_fails_fast_when_citation_markup_is_unparseable() ->
         error=None,
     )
 
-    parsed = parse_profile_page(fetch_result)
-
-    assert parsed.state == ParseState.LAYOUT_CHANGED
-    assert parsed.state_reason == "layout_row_citation_unparseable"
+    with pytest.raises(ScholarDomInvariantError) as exc:
+        parse_profile_page(fetch_result)
+    assert exc.value.code == "layout_row_citation_unparseable"
 
 
 def test_parse_profile_page_detects_layout_change_when_markers_absent() -> None:
@@ -175,11 +177,9 @@ def test_parse_profile_page_detects_layout_change_when_markers_absent() -> None:
         error=None,
     )
 
-    parsed = parse_profile_page(fetch_result)
-
-    assert parsed.state == ParseState.LAYOUT_CHANGED
-    assert parsed.state_reason == "layout_markers_missing"
-    assert "no_rows_detected" in parsed.warnings
+    with pytest.raises(ScholarDomInvariantError) as exc:
+        parse_profile_page(fetch_result)
+    assert exc.value.code == "layout_markers_missing"
 
 
 def test_parse_profile_page_reports_network_reason_when_status_missing() -> None:

tests/unit/test_scholar_search_safety.py

@@ -3,9 +3,9 @@ from __future__ import annotations
 import pytest
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.services import scholars as scholar_service
-from app.services.scholar_parser import ParseState
-from app.services.scholar_source import FetchResult
+from app.services.domains.scholars import application as scholar_service
+from app.services.domains.scholar.parser import ParseState
+from app.services.domains.scholar.source import FetchResult
 
 pytestmark = [pytest.mark.integration, pytest.mark.db]
 

tests/unit/test_scholar_source.py

@@ -1,4 +1,4 @@
-from app.services.scholar_source import _build_profile_url
+from app.services.domains.scholar.source import _build_profile_url
 
 
 def test_build_profile_url_includes_pagesize_for_initial_page() -> None:

tests/unit/test_user_settings.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import pytest
 
-from app.services.user_settings import (
+from app.services.domains.settings.application import (
     DEFAULT_NAV_VISIBLE_PAGES,
     HARD_MIN_REQUEST_DELAY_SECONDS,
     HARD_MIN_RUN_INTERVAL_MINUTES,