Intermediate commit

tests/unit/services/domains/arxiv/test_cache.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import asyncio
+import gc
+from datetime import datetime, timedelta, timezone
+
+import pytest
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import ArxivQueryCacheEntry
+from app.services.domains.arxiv.cache import (
+    build_query_fingerprint,
+    get_cached_feed,
+    run_with_inflight_dedupe,
+    set_cached_feed,
+)
+from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
+
+
+def _sample_feed(arxiv_id: str = "1234.5678") -> ArxivFeed:
+    return ArxivFeed(
+        entries=[
+            ArxivEntry(
+                entry_id_url=f"https://arxiv.org/abs/{arxiv_id}",
+                arxiv_id=arxiv_id,
+                title="Sample",
+                summary="Summary",
+                published=None,
+                updated=None,
+            )
+        ],
+        opensearch=ArxivOpenSearchMeta(total_results=1, start_index=0, items_per_page=1),
+    )
+
+
+def test_build_query_fingerprint_normalizes_search_and_id_params() -> None:
+    first = build_query_fingerprint(
+        params={
+            "search_query": '  TI:"Quantum   Fields"   AND AU:"Doe" ',
+            "start": 0,
+            "max_results": 3,
+        }
+    )
+    second = build_query_fingerprint(
+        params={
+            "search_query": 'ti:"quantum fields" and au:"doe"',
+            "start": 0,
+            "max_results": 3,
+        }
+    )
+    third = build_query_fingerprint(params={"id_list": " 2222.0002,1111.0001 "})
+    fourth = build_query_fingerprint(params={"id_list": "1111.0001, 2222.0002"})
+
+    assert first == second
+    assert third == fourth
+
+
+@pytest.mark.asyncio
+async def test_cache_entry_expires_and_is_deleted(db_session: AsyncSession) -> None:
+    query_fingerprint = build_query_fingerprint(params={"search_query": "ti:test", "start": 0})
+    now_utc = datetime(2026, 2, 26, 12, 0, tzinfo=timezone.utc)
+
+    await set_cached_feed(
+        query_fingerprint=query_fingerprint,
+        feed=_sample_feed(),
+        ttl_seconds=5.0,
+        max_entries=16,
+        now_utc=now_utc,
+    )
+
+    hit = await get_cached_feed(
+        query_fingerprint=query_fingerprint,
+        now_utc=now_utc + timedelta(seconds=2),
+    )
+    miss = await get_cached_feed(
+        query_fingerprint=query_fingerprint,
+        now_utc=now_utc + timedelta(seconds=8),
+    )
+
+    result = await db_session.execute(
+        select(ArxivQueryCacheEntry).where(
+            ArxivQueryCacheEntry.query_fingerprint == query_fingerprint
+        )
+    )
+    assert hit is not None
+    assert miss is None
+    assert result.scalar_one_or_none() is None
+
+
+@pytest.mark.asyncio
+async def test_inflight_dedupe_coalesces_identical_requests() -> None:
+    calls = {"count": 0}
+
+    async def _fetch_feed() -> ArxivFeed:
+        calls["count"] += 1
+        await asyncio.sleep(0.05)
+        return _sample_feed("9999.0001")
+
+    first, second = await asyncio.gather(
+        run_with_inflight_dedupe(query_fingerprint="same-key", fetch_feed=_fetch_feed),
+        run_with_inflight_dedupe(query_fingerprint="same-key", fetch_feed=_fetch_feed),
+    )
+
+    assert calls["count"] == 1
+    assert first.entries[0].arxiv_id == "9999.0001"
+    assert second.entries[0].arxiv_id == "9999.0001"
+
+
+@pytest.mark.asyncio
+async def test_inflight_owner_failure_without_joiner_has_no_unretrieved_exception() -> None:
+    loop = asyncio.get_running_loop()
+    messages: list[str] = []
+    previous_handler = loop.get_exception_handler()
+
+    def _capture_exception(_loop, context) -> None:
+        messages.append(str(context.get("message", "")))
+
+    loop.set_exception_handler(_capture_exception)
+    try:
+        async def _failing_fetch() -> ArxivFeed:
+            raise RuntimeError("owner_failed")
+
+        with pytest.raises(RuntimeError, match="owner_failed"):
+            await run_with_inflight_dedupe(
+                query_fingerprint="owner-failure-no-joiner",
+                fetch_feed=_failing_fetch,
+            )
+        gc.collect()
+        await asyncio.sleep(0)
+    finally:
+        loop.set_exception_handler(previous_handler)
+
+    assert "Future exception was never retrieved" not in " | ".join(messages)

tests/unit/services/domains/arxiv/test_client.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timezone
+import httpx
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.services.domains.arxiv import client as arxiv_client_module
+from app.services.domains.arxiv.client import ArxivClient
+from app.services.domains.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
+from app.services.domains.arxiv.rate_limit import ArxivCooldownStatus
+
+_CLIENT_FEED_XML = """<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom"
+      xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
+      xmlns:arxiv="http://arxiv.org/schemas/atom">
+  <opensearch:totalResults>1</opensearch:totalResults>
+  <opensearch:startIndex>0</opensearch:startIndex>
+  <opensearch:itemsPerPage>1</opensearch:itemsPerPage>
+  <entry>
+    <id>http://arxiv.org/abs/9999.0001</id>
+    <title>Client Entry</title>
+    <summary>Client summary</summary>
+  </entry>
+</feed>
+"""
+
+
+@pytest.mark.asyncio
+async def test_client_search_builds_query_and_sort_params() -> None:
+    captured: dict[str, object] = {}
+
+    async def _request_fn(*, params, request_email, timeout_seconds):
+        captured["params"] = params
+        captured["request_email"] = request_email
+        captured["timeout_seconds"] = timeout_seconds
+        return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
+
+    client = ArxivClient(request_fn=_request_fn)
+    feed = await client.search(
+        query='ti:"test"',
+        start=5,
+        max_results=7,
+        sort_by="submittedDate",
+        sort_order="ascending",
+        request_email="user@example.com",
+        timeout_seconds=3.5,
+    )
+
+    assert feed.opensearch.total_results == 1
+    assert captured["params"] == {
+        "search_query": 'ti:"test"',
+        "start": 5,
+        "max_results": 7,
+        "sortBy": "submittedDate",
+        "sortOrder": "ascending",
+    }
+    assert captured["request_email"] == "user@example.com"
+    assert captured["timeout_seconds"] == 3.5
+
+
+@pytest.mark.asyncio
+async def test_client_lookup_ids_builds_id_list_param() -> None:
+    captured: dict[str, object] = {}
+
+    async def _request_fn(*, params, request_email, timeout_seconds):
+        captured["params"] = params
+        return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
+
+    client = ArxivClient(request_fn=_request_fn)
+    await client.lookup_ids(id_list=["1234.5678", " 9999.0001 "], start=0, max_results=2)
+    assert captured["params"] == {"id_list": "1234.5678,9999.0001", "start": 0, "max_results": 2}
+
+
+@pytest.mark.asyncio
+async def test_client_search_rejects_invalid_sort_by() -> None:
+    async def _unused_request_fn(*, params, request_email, timeout_seconds):
+        return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
+
+    client = ArxivClient(request_fn=_unused_request_fn)
+    with pytest.raises(ArxivClientValidationError):
+        await client.search(query="x", sort_by="bad")
+
+
+@pytest.mark.asyncio
+async def test_client_lookup_ids_rejects_empty_list() -> None:
+    async def _unused_request_fn(*, params, request_email, timeout_seconds):
+        return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
+
+    client = ArxivClient(request_fn=_unused_request_fn)
+    with pytest.raises(ArxivClientValidationError):
+        await client.lookup_ids(id_list=[])
+
+
+@pytest.mark.asyncio
+async def test_client_search_rejects_negative_start() -> None:
+    async def _unused_request_fn(*, params, request_email, timeout_seconds):
+        return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
+
+    client = ArxivClient(request_fn=_unused_request_fn)
+    with pytest.raises(ArxivClientValidationError):
+        await client.search(query="ti:test", start=-1)
+
+
+@pytest.mark.asyncio
+async def test_client_propagates_http_status_error() -> None:
+    async def _request_fn(*, params, request_email, timeout_seconds):
+        request = httpx.Request("GET", "https://export.arxiv.org/api/query")
+        return httpx.Response(500, text="error", request=request)
+
+    client = ArxivClient(request_fn=_request_fn)
+    with pytest.raises(httpx.HTTPStatusError):
+        await client.search(query="ti:test")
+
+
+@pytest.mark.asyncio
+async def test_client_coalesces_concurrent_identical_search_requests() -> None:
+    calls = {"count": 0}
+
+    async def _request_fn(*, params, request_email, timeout_seconds):
+        calls["count"] += 1
+        await asyncio.sleep(0.05)
+        request = httpx.Request("GET", "https://export.arxiv.org/api/query")
+        return httpx.Response(200, text=_CLIENT_FEED_XML, request=request)
+
+    client = ArxivClient(request_fn=_request_fn)
+    await asyncio.gather(
+        client.search(query="ti:test"),
+        client.search(query="ti:test"),
+    )
+    assert calls["count"] == 1
+
+
+@pytest.mark.asyncio
+async def test_client_logs_cache_hit_and_miss(
+    db_session: AsyncSession,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    _ = db_session
+    calls = {"count": 0}
+    logged: list[dict[str, object]] = []
+
+    async def _request_fn(*, params, request_email, timeout_seconds):
+        calls["count"] += 1
+        request = httpx.Request("GET", "https://export.arxiv.org/api/query")
+        return httpx.Response(200, text=_CLIENT_FEED_XML, request=request)
+
+    def _capture_log(_msg: str, *args, **kwargs) -> None:
+        extra = kwargs.get("extra")
+        if isinstance(extra, dict):
+            logged.append(extra)
+
+    monkeypatch.setattr("app.services.domains.arxiv.client.logger.info", _capture_log)
+    client = ArxivClient(request_fn=_request_fn, cache_enabled=True)
+    await client.search(query="ti:test-cache")
+    await client.search(query="ti:test-cache")
+
+    events = [str(entry.get("event", "")) for entry in logged]
+    assert "arxiv.cache_miss" in events
+    assert "arxiv.cache_hit" in events
+    assert calls["count"] == 1
+
+
+@pytest.mark.asyncio
+async def test_request_feed_skips_live_call_when_global_cooldown_is_active(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    logged: list[dict[str, object]] = []
+    called = {"count": 0}
+
+    async def _cooldown_status(*, now_utc=None):
+        _ = now_utc
+        return ArxivCooldownStatus(
+            is_active=True,
+            remaining_seconds=61.0,
+            cooldown_until=datetime(2026, 2, 26, 12, 1, tzinfo=timezone.utc),
+        )
+
+    async def _unexpected_limit_call(*, fetch, source_path):  # pragma: no cover - defensive
+        _ = (fetch, source_path)
+        called["count"] += 1
+        return httpx.Response(200, text=_CLIENT_FEED_XML)
+
+    def _capture_warning(_msg: str, *args, **kwargs) -> None:
+        extra = kwargs.get("extra")
+        if isinstance(extra, dict):
+            logged.append(extra)
+
+    monkeypatch.setattr(arxiv_client_module, "get_arxiv_cooldown_status", _cooldown_status)
+    monkeypatch.setattr(arxiv_client_module, "run_with_global_arxiv_limit", _unexpected_limit_call)
+    monkeypatch.setattr("app.services.domains.arxiv.client.logger.warning", _capture_warning)
+
+    with pytest.raises(ArxivRateLimitError):
+        await arxiv_client_module._request_arxiv_feed(
+            params={"search_query": 'ti:"test"'},
+            request_email="user@example.com",
+            timeout_seconds=2.0,
+        )
+
+    assert called["count"] == 0
+    assert [entry.get("event") for entry in logged] == ["arxiv.request_skipped_cooldown"]

tests/unit/services/domains/arxiv/test_gateway.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from app.services.domains.arxiv import application as arxiv_application
+from app.services.domains.arxiv import gateway as arxiv_gateway
+from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
+from app.settings import settings
+
+
+def _item(*, title: str = "A Test Paper", scholar_label: str = "Ada Lovelace") -> SimpleNamespace:
+    return SimpleNamespace(title=title, scholar_label=scholar_label)
+
+
+def test_get_arxiv_gateway_returns_cached_instance() -> None:
+    previous = arxiv_gateway.set_arxiv_gateway(None)
+    try:
+        first = arxiv_gateway.get_arxiv_gateway()
+        second = arxiv_gateway.get_arxiv_gateway()
+        assert first is second
+    finally:
+        arxiv_gateway.set_arxiv_gateway(previous)
+
+
+@pytest.mark.asyncio
+async def test_application_discover_uses_gateway_override() -> None:
+    class FakeGateway:
+        def __init__(self) -> None:
+            self.calls: list[tuple[object, str | None, float | None]] = []
+
+        async def discover_arxiv_id_for_publication(
+            self,
+            *,
+            item,
+            request_email: str | None = None,
+            timeout_seconds: float | None = None,
+            max_results: int | None = None,
+        ) -> str | None:
+            self.calls.append((item, request_email, timeout_seconds))
+            return "1234.5678"
+
+    fake_gateway = FakeGateway()
+    previous = arxiv_gateway.set_arxiv_gateway(fake_gateway)
+    try:
+        result = await arxiv_application.discover_arxiv_id_for_publication(
+            item=_item(),
+            request_email="user@example.com",
+            timeout_seconds=7.0,
+        )
+        assert result == "1234.5678"
+        assert fake_gateway.calls
+        assert fake_gateway.calls[0][1] == "user@example.com"
+        assert fake_gateway.calls[0][2] == 7.0
+    finally:
+        arxiv_gateway.set_arxiv_gateway(previous)
+
+
+@pytest.mark.asyncio
+async def test_http_gateway_returns_none_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
+    class FakeClient:
+        def __init__(self) -> None:
+            self.calls = 0
+
+        async def search(self, **kwargs):
+            self.calls += 1
+            return ArxivFeed()
+
+    previous_enabled = bool(settings.arxiv_enabled)
+    fake_client = FakeClient()
+    object.__setattr__(settings, "arxiv_enabled", False)
+    try:
+        gateway = arxiv_gateway.HttpArxivGateway(client=fake_client)
+        result = await gateway.discover_arxiv_id_for_publication(item=_item())
+        assert result is None
+        assert fake_client.calls == 0
+    finally:
+        object.__setattr__(settings, "arxiv_enabled", previous_enabled)
+
+
+@pytest.mark.asyncio
+async def test_http_gateway_uses_client_search_for_discovery() -> None:
+    class FakeClient:
+        def __init__(self) -> None:
+            self.calls: list[dict] = []
+
+        async def search(self, **kwargs):
+            self.calls.append(kwargs)
+            return ArxivFeed(
+                entries=[
+                    ArxivEntry(
+                        entry_id_url="https://arxiv.org/abs/1234.5678v1",
+                        arxiv_id="1234.5678v1",
+                        title="Paper",
+                        summary="",
+                        published=None,
+                        updated=None,
+                    )
+                ],
+                opensearch=ArxivOpenSearchMeta(total_results=1, start_index=0, items_per_page=1),
+            )
+
+    fake_client = FakeClient()
+    gateway = arxiv_gateway.HttpArxivGateway(client=fake_client)
+    result = await gateway.discover_arxiv_id_for_publication(
+        item=_item(title="My Paper", scholar_label="Ada Lovelace"),
+        request_email="user@example.com",
+        timeout_seconds=2.0,
+        max_results=4,
+    )
+
+    assert result == "1234.5678v1"
+    assert fake_client.calls
+    first_call = fake_client.calls[0]
+    assert first_call["query"] == 'ti:"My Paper" AND au:"lovelace"'
+    assert first_call["request_email"] == "user@example.com"
+    assert first_call["timeout_seconds"] == 2.0
+    assert first_call["max_results"] == 4
+
+
+def test_build_arxiv_query_normalizes_noisy_mojibake_title() -> None:
+    noisy = "  Graph–Neural   Networks  Survey  "
+    clean = "Graph Neural Networks Survey"
+
+    assert arxiv_gateway.build_arxiv_query(noisy, None) == arxiv_gateway.build_arxiv_query(clean, None)

tests/unit/services/domains/arxiv/test_guards.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
+from app.services.domains.publication_identifiers.types import DisplayIdentifier
+from app.services.domains.publications.types import PublicationListItem
+
+
+def _item(
+    *,
+    title: str,
+    pub_url: str | None = None,
+    pdf_url: str | None = None,
+    display_identifier: DisplayIdentifier | None = None,
+) -> PublicationListItem:
+    return PublicationListItem(
+        publication_id=1,
+        scholar_profile_id=1,
+        scholar_label="Ada Lovelace",
+        title=title,
+        year=2024,
+        citation_count=0,
+        venue_text=None,
+        pub_url=pub_url,
+        pdf_url=pdf_url,
+        is_read=False,
+        first_seen_at=datetime.now(timezone.utc),
+        is_new_in_latest_run=True,
+        display_identifier=display_identifier,
+    )
+
+
+def test_arxiv_skip_reason_for_strong_doi_evidence() -> None:
+    item = _item(
+        title="A Robust and Reproducible Deep Learning Benchmark",
+        display_identifier=DisplayIdentifier(
+            kind="doi",
+            value="10.1000/example",
+            label="DOI: 10.1000/example",
+            url="https://doi.org/10.1000/example",
+            confidence_score=1.0,
+        ),
+    )
+    assert arxiv_skip_reason_for_item(item=item) == "strong_doi_present"
+
+
+def test_arxiv_skip_reason_for_existing_arxiv_link() -> None:
+    item = _item(
+        title="A Robust and Reproducible Deep Learning Benchmark",
+        pub_url="https://arxiv.org/abs/2501.00001",
+    )
+    assert arxiv_skip_reason_for_item(item=item) == "arxiv_identifier_present"
+
+
+def test_arxiv_skip_reason_for_low_quality_title() -> None:
+    item = _item(title="AI 2024")
+    assert arxiv_skip_reason_for_item(item=item) == "title_quality_below_threshold"
+
+
+def test_arxiv_skip_reason_none_for_eligible_item() -> None:
+    item = _item(title="Reliable Graph Neural Network Benchmarking Across Multiple Datasets")
+    assert arxiv_skip_reason_for_item(item=item) is None

tests/unit/services/domains/arxiv/test_parser.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import pytest
+
+from app.services.domains.arxiv.errors import ArxivParseError
+from app.services.domains.arxiv.parser import parse_arxiv_feed
+
+_VALID_FEED_XML = """<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom"
+      xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
+      xmlns:arxiv="http://arxiv.org/schemas/atom">
+  <opensearch:totalResults>2</opensearch:totalResults>
+  <opensearch:startIndex>0</opensearch:startIndex>
+  <opensearch:itemsPerPage>2</opensearch:itemsPerPage>
+  <entry>
+    <id>http://arxiv.org/abs/1234.5678v2</id>
+    <updated>2020-01-01T00:00:00Z</updated>
+    <published>2019-12-31T00:00:00Z</published>
+    <title>Test Entry</title>
+    <summary>Example summary</summary>
+    <author><name>Ada Lovelace</name></author>
+    <author><name>Grace Hopper</name></author>
+    <link href="http://arxiv.org/abs/1234.5678v2" />
+    <category term="cs.AI" />
+    <category term="stat.ML" />
+    <arxiv:primary_category term="cs.AI" />
+  </entry>
+</feed>
+"""
+
+
+def test_parse_arxiv_feed_extracts_entries_and_opensearch_meta() -> None:
+    feed = parse_arxiv_feed(_VALID_FEED_XML)
+    assert feed.opensearch.total_results == 2
+    assert feed.opensearch.start_index == 0
+    assert feed.opensearch.items_per_page == 2
+    assert len(feed.entries) == 1
+    entry = feed.entries[0]
+    assert entry.arxiv_id == "1234.5678v2"
+    assert entry.title == "Test Entry"
+    assert entry.summary == "Example summary"
+    assert entry.authors == ["Ada Lovelace", "Grace Hopper"]
+    assert entry.primary_category == "cs.AI"
+    assert entry.categories == ["cs.AI", "stat.ML"]
+
+
+def test_parse_arxiv_feed_raises_on_invalid_xml() -> None:
+    with pytest.raises(ArxivParseError):
+        parse_arxiv_feed("<feed><entry></feed>")
+
+
+def test_parse_arxiv_feed_raises_on_invalid_opensearch_integer() -> None:
+    payload = _VALID_FEED_XML.replace("<opensearch:totalResults>2</opensearch:totalResults>", "<opensearch:totalResults>x</opensearch:totalResults>")
+    with pytest.raises(ArxivParseError):
+        parse_arxiv_feed(payload)

tests/unit/services/domains/arxiv/test_rate_limit.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timedelta, timezone
+
+import httpx
+import pytest
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import ArxivRuntimeState
+from app.services.domains.arxiv.constants import ARXIV_RUNTIME_STATE_KEY
+from app.services.domains.arxiv.errors import ArxivRateLimitError
+from app.services.domains.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
+from app.settings import settings
+
+
+@pytest.mark.asyncio
+async def test_arxiv_rate_limit_respects_cooldown(db_session: AsyncSession) -> None:
+    db_session.add(
+        ArxivRuntimeState(
+            state_key=ARXIV_RUNTIME_STATE_KEY,
+            cooldown_until=datetime.now(timezone.utc) + timedelta(seconds=30),
+        )
+    )
+    await db_session.commit()
+
+    called = {"count": 0}
+
+    async def _fetch() -> httpx.Response:
+        called["count"] += 1
+        return httpx.Response(200, text="ok")
+
+    with pytest.raises(ArxivRateLimitError):
+        await run_with_global_arxiv_limit(fetch=_fetch)
+    assert called["count"] == 0
+
+
+@pytest.mark.asyncio
+async def test_arxiv_rate_limit_persists_cooldown_after_429(db_session: AsyncSession) -> None:
+    previous_interval = settings.arxiv_min_interval_seconds
+    previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
+    object.__setattr__(settings, "arxiv_min_interval_seconds", 0.0)
+    object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", 5.0)
+    try:
+        async def _fetch() -> httpx.Response:
+            return httpx.Response(429, text="rate limited")
+
+        with pytest.raises(ArxivRateLimitError):
+            await run_with_global_arxiv_limit(fetch=_fetch)
+    finally:
+        object.__setattr__(settings, "arxiv_min_interval_seconds", previous_interval)
+        object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", previous_cooldown)
+
+    result = await db_session.execute(
+        select(ArxivRuntimeState).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
+    )
+    state = result.scalar_one()
+    assert state.cooldown_until is not None
+    assert state.cooldown_until > datetime.now(timezone.utc)
+
+
+@pytest.mark.asyncio
+async def test_arxiv_rate_limit_serializes_concurrent_calls(db_session: AsyncSession) -> None:
+    previous_interval = settings.arxiv_min_interval_seconds
+    previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
+    object.__setattr__(settings, "arxiv_min_interval_seconds", 0.2)
+    object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", 5.0)
+    try:
+        call_times: list[float] = []
+
+        async def _fetch() -> httpx.Response:
+            call_times.append(asyncio.get_running_loop().time())
+            return httpx.Response(200, text="ok")
+
+        await asyncio.gather(
+            run_with_global_arxiv_limit(fetch=_fetch),
+            run_with_global_arxiv_limit(fetch=_fetch),
+        )
+    finally:
+        object.__setattr__(settings, "arxiv_min_interval_seconds", previous_interval)
+        object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", previous_cooldown)
+
+    assert len(call_times) == 2
+    assert call_times[1] - call_times[0] >= 0.18
+
+
+@pytest.mark.asyncio
+async def test_arxiv_rate_limit_logs_request_scheduled_and_completed(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    logged: list[dict[str, object]] = []
+
+    async def _fetch() -> httpx.Response:
+        return httpx.Response(200, text="ok")
+
+    def _capture_log(_msg: str, *args, **kwargs) -> None:
+        extra = kwargs.get("extra")
+        if isinstance(extra, dict):
+            logged.append(extra)
+
+    monkeypatch.setattr("app.services.domains.arxiv.rate_limit.logger.info", _capture_log)
+    await run_with_global_arxiv_limit(fetch=_fetch, source_path="search")
+
+    scheduled = [entry for entry in logged if entry.get("event") == "arxiv.request_scheduled"]
+    completed = [entry for entry in logged if entry.get("event") == "arxiv.request_completed"]
+
+    assert scheduled
+    assert completed
+    assert float(scheduled[0]["wait_seconds"]) >= 0.0
+    assert int(completed[0]["status_code"]) == 200
+    assert completed[0]["source_path"] == "search"
+
+
+@pytest.mark.asyncio
+async def test_arxiv_rate_limit_logs_cooldown_activation(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    logged_warning: list[dict[str, object]] = []
+    previous_interval = settings.arxiv_min_interval_seconds
+    previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
+    object.__setattr__(settings, "arxiv_min_interval_seconds", 0.0)
+    object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", 5.0)
+    try:
+        async def _fetch() -> httpx.Response:
+            return httpx.Response(429, text="rate limited")
+
+        def _capture_warning(_msg: str, *args, **kwargs) -> None:
+            extra = kwargs.get("extra")
+            if isinstance(extra, dict):
+                logged_warning.append(extra)
+
+        monkeypatch.setattr("app.services.domains.arxiv.rate_limit.logger.warning", _capture_warning)
+        with pytest.raises(ArxivRateLimitError):
+            await run_with_global_arxiv_limit(fetch=_fetch, source_path="lookup_ids")
+    finally:
+        object.__setattr__(settings, "arxiv_min_interval_seconds", previous_interval)
+        object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", previous_cooldown)
+
+    cooldown_events = [
+        entry for entry in logged_warning if entry.get("event") == "arxiv.cooldown_activated"
+    ]
+    assert cooldown_events
+    assert cooldown_events[0]["source_path"] == "lookup_ids"
+    assert float(cooldown_events[0]["cooldown_remaining_seconds"]) > 0.0
+
+
+@pytest.mark.asyncio
+async def test_get_arxiv_cooldown_status_reads_active_cooldown(db_session: AsyncSession) -> None:
+    now_utc = datetime(2026, 2, 26, 13, 0, tzinfo=timezone.utc)
+    existing = await db_session.get(ArxivRuntimeState, ARXIV_RUNTIME_STATE_KEY)
+    if existing is None:
+        db_session.add(
+            ArxivRuntimeState(
+                state_key=ARXIV_RUNTIME_STATE_KEY,
+                cooldown_until=now_utc + timedelta(seconds=45),
+            )
+        )
+    else:
+        existing.cooldown_until = now_utc + timedelta(seconds=45)
+    await db_session.commit()
+
+    status = await get_arxiv_cooldown_status(now_utc=now_utc)
+
+    assert status.is_active is True
+    assert status.cooldown_until is not None
+    assert int(status.remaining_seconds) == 45

tests/unit/services/domains/publications/test_dedup.py

@@ -1,29 +1,30 @@
-"""Unit tests for the identifier-based publication dedup sweep.
+"""Unit tests for publication dedup operations."""
 
-DB operations are mocked via AsyncMock so no database is required.
-"""
 from __future__ import annotations
 
+from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
 from app.db.models import ScholarPublication
+from app.services.domains.publications import dedup as dedup_service
 from app.services.domains.publications.dedup import (
+    NearDuplicateCluster,
+    NearDuplicateMember,
     find_identifier_duplicate_pairs,
+    find_near_duplicate_clusters,
     merge_duplicate_publication,
+    merge_near_duplicate_cluster,
     sweep_identifier_duplicates,
 )
 
 
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
 def _make_result(rows: list) -> MagicMock:
     mock_result = MagicMock()
     mock_result.scalars.return_value.all.return_value = rows
     mock_result.__iter__ = lambda self: iter(rows)
+    mock_result.all.return_value = rows
     return mock_result
 
 
@@ -35,10 +36,6 @@ def _session_with_execute_sequence(results: list) -> AsyncMock:
     return session
 
 
-# ---------------------------------------------------------------------------
-# find_identifier_duplicate_pairs
-# ---------------------------------------------------------------------------
-
 @pytest.mark.asyncio
 async def test_find_identifier_duplicate_pairs_returns_pairs() -> None:
     session = AsyncMock()
@@ -60,56 +57,104 @@ async def test_find_identifier_duplicate_pairs_returns_empty_when_no_duplicates(
     assert pairs == []
 
 
-# ---------------------------------------------------------------------------
-# merge_duplicate_publication
-# ---------------------------------------------------------------------------
+@pytest.mark.asyncio
+async def test_merge_duplicate_publication_migrates_links_and_identifiers() -> None:
+    session = AsyncMock()
+    winner = SimpleNamespace(
+        year=2014,
+        citation_count=10,
+        author_text=None,
+        venue_text=None,
+        pub_url=None,
+        pdf_url=None,
+        cluster_id=None,
+        canonical_title_hash=None,
+        title_raw="winner",
+        title_normalized="winner",
+    )
+    dup = SimpleNamespace(
+        year=2015,
+        citation_count=11,
+        author_text="a",
+        venue_text="v",
+        pub_url="https://example.org",
+        pdf_url="https://example.org/a.pdf",
+        cluster_id="x",
+        canonical_title_hash="hash",
+        title_raw="dup",
+        title_normalized="dup",
+    )
+
+    with (
+        patch(
+            "app.services.domains.publications.dedup._load_publication",
+            new=AsyncMock(side_effect=[winner, dup]),
+        ),
+        patch(
+            "app.services.domains.publications.dedup._migrate_scholar_links",
+            new=AsyncMock(),
+        ) as mock_links,
+        patch(
+            "app.services.domains.publications.dedup._migrate_identifiers",
+            new=AsyncMock(),
+        ) as mock_identifiers,
+    ):
+        await merge_duplicate_publication(session, winner_id=1, dup_id=2)
+
+    assert session.execute.await_count == 1
+    mock_links.assert_awaited_once_with(session, winner_id=1, dup_id=2)
+    mock_identifiers.assert_awaited_once_with(session, winner_id=1, dup_id=2)
+
 
 @pytest.mark.asyncio
-async def test_merge_duplicate_migrates_orphaned_scholar_links() -> None:
-    """Scholar link that only the dup has should be migrated to winner."""
+async def test_merge_duplicate_publication_rejects_missing_publications() -> None:
+    session = AsyncMock()
+
+    with patch(
+        "app.services.domains.publications.dedup._load_publication",
+        new=AsyncMock(side_effect=[None, None]),
+    ):
+        with pytest.raises(ValueError):
+            await merge_duplicate_publication(session, winner_id=1, dup_id=2)
+
+
+@pytest.mark.asyncio
+async def test_migrate_scholar_links_moves_orphans() -> None:
     dup_link = MagicMock(spec=ScholarPublication)
     dup_link.scholar_profile_id = 99
     dup_link.publication_id = 2
 
     session = _session_with_execute_sequence(
         results=[
-            [dup_link],  # dup links
-            [],          # winner profile ids (no conflict)
-            [],          # execute(delete(Publication)) result
+            [dup_link],
+            [],
         ]
     )
 
-    await merge_duplicate_publication(session, winner_id=1, dup_id=2)
+    await dedup_service._migrate_scholar_links(session, winner_id=1, dup_id=2)
 
     assert dup_link.publication_id == 1
-    session.delete.assert_not_awaited()  # not deleted; migrated instead
+    session.delete.assert_not_awaited()
 
 
 @pytest.mark.asyncio
-async def test_merge_duplicate_drops_conflicting_scholar_link() -> None:
-    """When winner already has a link for the same scholar, dup's link is deleted."""
+async def test_migrate_scholar_links_drops_conflicts() -> None:
     dup_link = MagicMock(spec=ScholarPublication)
     dup_link.scholar_profile_id = 88
     dup_link.publication_id = 2
 
     session = _session_with_execute_sequence(
         results=[
-            [dup_link],    # dup links
-            [(88,)],       # winner profiles (conflict: profile 88 already linked)
-            [],            # execute(delete(Publication)) result
+            [dup_link],
+            [(88,)],
         ]
     )
 
-    await merge_duplicate_publication(session, winner_id=1, dup_id=2)
+    await dedup_service._migrate_scholar_links(session, winner_id=1, dup_id=2)
 
     session.delete.assert_awaited_once_with(dup_link)
-    assert dup_link.publication_id == 2  # unchanged — link was deleted, not migrated
 
 
-# ---------------------------------------------------------------------------
-# sweep_identifier_duplicates
-# ---------------------------------------------------------------------------
-
 @pytest.mark.asyncio
 async def test_sweep_returns_zero_when_no_pairs() -> None:
     with patch(
@@ -145,7 +190,6 @@ async def test_sweep_returns_merge_count() -> None:
 
 @pytest.mark.asyncio
 async def test_sweep_merges_each_dup_only_once() -> None:
-    """A dup sharing two identifiers with the winner appears twice in pairs but merged once."""
     with (
         patch(
             "app.services.domains.publications.dedup.find_identifier_duplicate_pairs",
@@ -161,3 +205,80 @@ async def test_sweep_merges_each_dup_only_once() -> None:
 
     assert count == 1
     assert mock_merge.await_count == 1
+
+
+@pytest.mark.asyncio
+async def test_find_near_duplicate_clusters_groups_similar_titles() -> None:
+    first = dedup_service._candidate_from_row(
+        publication_id=10,
+        title="Adam: A method for stochastic optimization",
+        year=2014,
+        citation_count=100,
+    )
+    second = dedup_service._candidate_from_row(
+        publication_id=11,
+        title="†œAdam: A method for stochastic optimization, †3rd Int. Conf. Learn. Represent.",
+        year=2015,
+        citation_count=50,
+    )
+    assert first is not None
+    assert second is not None
+
+    with patch(
+        "app.services.domains.publications.dedup._load_near_duplicate_candidates",
+        new=AsyncMock(return_value=[first, second]),
+    ):
+        clusters = await find_near_duplicate_clusters(AsyncMock())
+
+    assert len(clusters) == 1
+    assert len(clusters[0].members) == 2
+    assert clusters[0].winner_publication_id == 10
+
+
+@pytest.mark.asyncio
+async def test_find_near_duplicate_clusters_skips_unrelated_titles() -> None:
+    first = dedup_service._candidate_from_row(
+        publication_id=21,
+        title="Adam optimizer",
+        year=2014,
+        citation_count=10,
+    )
+    second = dedup_service._candidate_from_row(
+        publication_id=22,
+        title="Diffusion models in vision",
+        year=2022,
+        citation_count=10,
+    )
+    assert first is not None
+    assert second is not None
+
+    with patch(
+        "app.services.domains.publications.dedup._load_near_duplicate_candidates",
+        new=AsyncMock(return_value=[first, second]),
+    ):
+        clusters = await find_near_duplicate_clusters(AsyncMock())
+
+    assert clusters == []
+
+
+@pytest.mark.asyncio
+async def test_merge_near_duplicate_cluster_merges_non_winner_members() -> None:
+    cluster = NearDuplicateCluster(
+        cluster_key="abc",
+        winner_publication_id=5,
+        similarity_score=1.0,
+        members=(
+            NearDuplicateMember(publication_id=5, title="Winner", year=2014, citation_count=10),
+            NearDuplicateMember(publication_id=6, title="Dup", year=2014, citation_count=3),
+            NearDuplicateMember(publication_id=7, title="Dup2", year=2014, citation_count=1),
+        ),
+    )
+
+    with patch(
+        "app.services.domains.publications.dedup.merge_duplicate_publication",
+        new=AsyncMock(),
+    ) as mock_merge:
+        merged = await merge_near_duplicate_cluster(AsyncMock(), cluster=cluster)
+
+    assert merged == 2
+    assert mock_merge.await_count == 2