justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

Intermediate commit

Browse source
This commit is contained in:
Justin Visser 2026-02-26 16:09:57 +01:00
parent 0e9e49df16
commit 3d4cfeff1a
65 changed files with 5507 additions and 333 deletions
Download patch file Download diff file Expand all files Collapse all files

134
tests/unit/services/domains/arxiv/test_cache.py Normal file
View file

@ -0,0 +1,134 @@
from __future__ import annotations
import asyncio
import gc
from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import ArxivQueryCacheEntry
from app.services.domains.arxiv.cache import (
build_query_fingerprint,
get_cached_feed,
run_with_inflight_dedupe,
set_cached_feed,
)
from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
def _sample_feed(arxiv_id: str = "1234.5678") -> ArxivFeed:
return ArxivFeed(
entries=[
ArxivEntry(
entry_id_url=f"https://arxiv.org/abs/{arxiv_id}",
arxiv_id=arxiv_id,
title="Sample",
summary="Summary",
published=None,
updated=None,
)
],
opensearch=ArxivOpenSearchMeta(total_results=1, start_index=0, items_per_page=1),
)
def test_build_query_fingerprint_normalizes_search_and_id_params() -> None:
first = build_query_fingerprint(
params={
"search_query": ' TI:"Quantum Fields" AND AU:"Doe" ',
"start": 0,
"max_results": 3,
}
)
second = build_query_fingerprint(
params={
"search_query": 'ti:"quantum fields" and au:"doe"',
"start": 0,
"max_results": 3,
}
)
third = build_query_fingerprint(params={"id_list": " 2222.0002,1111.0001 "})
fourth = build_query_fingerprint(params={"id_list": "1111.0001, 2222.0002"})
assert first == second
assert third == fourth
@pytest.mark.asyncio
async def test_cache_entry_expires_and_is_deleted(db_session: AsyncSession) -> None:
query_fingerprint = build_query_fingerprint(params={"search_query": "ti:test", "start": 0})
now_utc = datetime(2026, 2, 26, 12, 0, tzinfo=timezone.utc)
await set_cached_feed(
query_fingerprint=query_fingerprint,
feed=_sample_feed(),
ttl_seconds=5.0,
max_entries=16,
now_utc=now_utc,
)
hit = await get_cached_feed(
query_fingerprint=query_fingerprint,
now_utc=now_utc + timedelta(seconds=2),
)
miss = await get_cached_feed(
query_fingerprint=query_fingerprint,
now_utc=now_utc + timedelta(seconds=8),
)
result = await db_session.execute(
select(ArxivQueryCacheEntry).where(
ArxivQueryCacheEntry.query_fingerprint == query_fingerprint
)
)
assert hit is not None
assert miss is None
assert result.scalar_one_or_none() is None
@pytest.mark.asyncio
async def test_inflight_dedupe_coalesces_identical_requests() -> None:
calls = {"count": 0}
async def _fetch_feed() -> ArxivFeed:
calls["count"] += 1
await asyncio.sleep(0.05)
return _sample_feed("9999.0001")
first, second = await asyncio.gather(
run_with_inflight_dedupe(query_fingerprint="same-key", fetch_feed=_fetch_feed),
run_with_inflight_dedupe(query_fingerprint="same-key", fetch_feed=_fetch_feed),
)
assert calls["count"] == 1
assert first.entries[0].arxiv_id == "9999.0001"
assert second.entries[0].arxiv_id == "9999.0001"
@pytest.mark.asyncio
async def test_inflight_owner_failure_without_joiner_has_no_unretrieved_exception() -> None:
loop = asyncio.get_running_loop()
messages: list[str] = []
previous_handler = loop.get_exception_handler()
def _capture_exception(_loop, context) -> None:
messages.append(str(context.get("message", "")))
loop.set_exception_handler(_capture_exception)
try:
async def _failing_fetch() -> ArxivFeed:
raise RuntimeError("owner_failed")
with pytest.raises(RuntimeError, match="owner_failed"):
await run_with_inflight_dedupe(
query_fingerprint="owner-failure-no-joiner",
fetch_feed=_failing_fetch,
)
gc.collect()
await asyncio.sleep(0)
finally:
loop.set_exception_handler(previous_handler)
assert "Future exception was never retrieved" not in " | ".join(messages)

202
tests/unit/services/domains/arxiv/test_client.py Normal file
View file

@ -0,0 +1,202 @@
from __future__ import annotations
import asyncio
from datetime import datetime, timezone
import httpx
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from app.services.domains.arxiv import client as arxiv_client_module
from app.services.domains.arxiv.client import ArxivClient
from app.services.domains.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
from app.services.domains.arxiv.rate_limit import ArxivCooldownStatus
_CLIENT_FEED_XML = """<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
xmlns:arxiv="http://arxiv.org/schemas/atom">
<opensearch:totalResults>1</opensearch:totalResults>
<opensearch:startIndex>0</opensearch:startIndex>
<opensearch:itemsPerPage>1</opensearch:itemsPerPage>
<entry>
<id>http://arxiv.org/abs/9999.0001</id>
<title>Client Entry</title>
<summary>Client summary</summary>
</entry>
</feed>
"""
@pytest.mark.asyncio
async def test_client_search_builds_query_and_sort_params() -> None:
captured: dict[str, object] = {}
async def _request_fn(*, params, request_email, timeout_seconds):
captured["params"] = params
captured["request_email"] = request_email
captured["timeout_seconds"] = timeout_seconds
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
client = ArxivClient(request_fn=_request_fn)
feed = await client.search(
query='ti:"test"',
start=5,
max_results=7,
sort_by="submittedDate",
sort_order="ascending",
request_email="user@example.com",
timeout_seconds=3.5,
)
assert feed.opensearch.total_results == 1
assert captured["params"] == {
"search_query": 'ti:"test"',
"start": 5,
"max_results": 7,
"sortBy": "submittedDate",
"sortOrder": "ascending",
}
assert captured["request_email"] == "user@example.com"
assert captured["timeout_seconds"] == 3.5
@pytest.mark.asyncio
async def test_client_lookup_ids_builds_id_list_param() -> None:
captured: dict[str, object] = {}
async def _request_fn(*, params, request_email, timeout_seconds):
captured["params"] = params
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
client = ArxivClient(request_fn=_request_fn)
await client.lookup_ids(id_list=["1234.5678", " 9999.0001 "], start=0, max_results=2)
assert captured["params"] == {"id_list": "1234.5678,9999.0001", "start": 0, "max_results": 2}
@pytest.mark.asyncio
async def test_client_search_rejects_invalid_sort_by() -> None:
async def _unused_request_fn(*, params, request_email, timeout_seconds):
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
client = ArxivClient(request_fn=_unused_request_fn)
with pytest.raises(ArxivClientValidationError):
await client.search(query="x", sort_by="bad")
@pytest.mark.asyncio
async def test_client_lookup_ids_rejects_empty_list() -> None:
async def _unused_request_fn(*, params, request_email, timeout_seconds):
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
client = ArxivClient(request_fn=_unused_request_fn)
with pytest.raises(ArxivClientValidationError):
await client.lookup_ids(id_list=[])
@pytest.mark.asyncio
async def test_client_search_rejects_negative_start() -> None:
async def _unused_request_fn(*, params, request_email, timeout_seconds):
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
client = ArxivClient(request_fn=_unused_request_fn)
with pytest.raises(ArxivClientValidationError):
await client.search(query="ti:test", start=-1)
@pytest.mark.asyncio
async def test_client_propagates_http_status_error() -> None:
async def _request_fn(*, params, request_email, timeout_seconds):
request = httpx.Request("GET", "https://export.arxiv.org/api/query")
return httpx.Response(500, text="error", request=request)
client = ArxivClient(request_fn=_request_fn)
with pytest.raises(httpx.HTTPStatusError):
await client.search(query="ti:test")
@pytest.mark.asyncio
async def test_client_coalesces_concurrent_identical_search_requests() -> None:
calls = {"count": 0}
async def _request_fn(*, params, request_email, timeout_seconds):
calls["count"] += 1
await asyncio.sleep(0.05)
request = httpx.Request("GET", "https://export.arxiv.org/api/query")
return httpx.Response(200, text=_CLIENT_FEED_XML, request=request)
client = ArxivClient(request_fn=_request_fn)
await asyncio.gather(
client.search(query="ti:test"),
client.search(query="ti:test"),
)
assert calls["count"] == 1
@pytest.mark.asyncio
async def test_client_logs_cache_hit_and_miss(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_ = db_session
calls = {"count": 0}
logged: list[dict[str, object]] = []
async def _request_fn(*, params, request_email, timeout_seconds):
calls["count"] += 1
request = httpx.Request("GET", "https://export.arxiv.org/api/query")
return httpx.Response(200, text=_CLIENT_FEED_XML, request=request)
def _capture_log(_msg: str, *args, **kwargs) -> None:
extra = kwargs.get("extra")
if isinstance(extra, dict):
logged.append(extra)
monkeypatch.setattr("app.services.domains.arxiv.client.logger.info", _capture_log)
client = ArxivClient(request_fn=_request_fn, cache_enabled=True)
await client.search(query="ti:test-cache")
await client.search(query="ti:test-cache")
events = [str(entry.get("event", "")) for entry in logged]
assert "arxiv.cache_miss" in events
assert "arxiv.cache_hit" in events
assert calls["count"] == 1
@pytest.mark.asyncio
async def test_request_feed_skips_live_call_when_global_cooldown_is_active(
monkeypatch: pytest.MonkeyPatch,
) -> None:
logged: list[dict[str, object]] = []
called = {"count": 0}
async def _cooldown_status(*, now_utc=None):
_ = now_utc
return ArxivCooldownStatus(
is_active=True,
remaining_seconds=61.0,
cooldown_until=datetime(2026, 2, 26, 12, 1, tzinfo=timezone.utc),
)
async def _unexpected_limit_call(*, fetch, source_path): # pragma: no cover - defensive
_ = (fetch, source_path)
called["count"] += 1
return httpx.Response(200, text=_CLIENT_FEED_XML)
def _capture_warning(_msg: str, *args, **kwargs) -> None:
extra = kwargs.get("extra")
if isinstance(extra, dict):
logged.append(extra)
monkeypatch.setattr(arxiv_client_module, "get_arxiv_cooldown_status", _cooldown_status)
monkeypatch.setattr(arxiv_client_module, "run_with_global_arxiv_limit", _unexpected_limit_call)
monkeypatch.setattr("app.services.domains.arxiv.client.logger.warning", _capture_warning)
with pytest.raises(ArxivRateLimitError):
await arxiv_client_module._request_arxiv_feed(
params={"search_query": 'ti:"test"'},
request_email="user@example.com",
timeout_seconds=2.0,
)
assert called["count"] == 0
assert [entry.get("event") for entry in logged] == ["arxiv.request_skipped_cooldown"]

126
tests/unit/services/domains/arxiv/test_gateway.py Normal file
View file

@ -0,0 +1,126 @@
from __future__ import annotations
from types import SimpleNamespace
import pytest
from app.services.domains.arxiv import application as arxiv_application
from app.services.domains.arxiv import gateway as arxiv_gateway
from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
from app.settings import settings
def _item(*, title: str = "A Test Paper", scholar_label: str = "Ada Lovelace") -> SimpleNamespace:
return SimpleNamespace(title=title, scholar_label=scholar_label)
def test_get_arxiv_gateway_returns_cached_instance() -> None:
previous = arxiv_gateway.set_arxiv_gateway(None)
try:
first = arxiv_gateway.get_arxiv_gateway()
second = arxiv_gateway.get_arxiv_gateway()
assert first is second
finally:
arxiv_gateway.set_arxiv_gateway(previous)
@pytest.mark.asyncio
async def test_application_discover_uses_gateway_override() -> None:
class FakeGateway:
def __init__(self) -> None:
self.calls: list[tuple[object, str | None, float | None]] = []
async def discover_arxiv_id_for_publication(
self,
*,
item,
request_email: str | None = None,
timeout_seconds: float | None = None,
max_results: int | None = None,
) -> str | None:
self.calls.append((item, request_email, timeout_seconds))
return "1234.5678"
fake_gateway = FakeGateway()
previous = arxiv_gateway.set_arxiv_gateway(fake_gateway)
try:
result = await arxiv_application.discover_arxiv_id_for_publication(
item=_item(),
request_email="user@example.com",
timeout_seconds=7.0,
)
assert result == "1234.5678"
assert fake_gateway.calls
assert fake_gateway.calls[0][1] == "user@example.com"
assert fake_gateway.calls[0][2] == 7.0
finally:
arxiv_gateway.set_arxiv_gateway(previous)
@pytest.mark.asyncio
async def test_http_gateway_returns_none_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
class FakeClient:
def __init__(self) -> None:
self.calls = 0
async def search(self, **kwargs):
self.calls += 1
return ArxivFeed()
previous_enabled = bool(settings.arxiv_enabled)
fake_client = FakeClient()
object.__setattr__(settings, "arxiv_enabled", False)
try:
gateway = arxiv_gateway.HttpArxivGateway(client=fake_client)
result = await gateway.discover_arxiv_id_for_publication(item=_item())
assert result is None
assert fake_client.calls == 0
finally:
object.__setattr__(settings, "arxiv_enabled", previous_enabled)
@pytest.mark.asyncio
async def test_http_gateway_uses_client_search_for_discovery() -> None:
class FakeClient:
def __init__(self) -> None:
self.calls: list[dict] = []
async def search(self, **kwargs):
self.calls.append(kwargs)
return ArxivFeed(
entries=[
ArxivEntry(
entry_id_url="https://arxiv.org/abs/1234.5678v1",
arxiv_id="1234.5678v1",
title="Paper",
summary="",
published=None,
updated=None,
)
],
opensearch=ArxivOpenSearchMeta(total_results=1, start_index=0, items_per_page=1),
)
fake_client = FakeClient()
gateway = arxiv_gateway.HttpArxivGateway(client=fake_client)
result = await gateway.discover_arxiv_id_for_publication(
item=_item(title="My Paper", scholar_label="Ada Lovelace"),
request_email="user@example.com",
timeout_seconds=2.0,
max_results=4,
)
assert result == "1234.5678v1"
assert fake_client.calls
first_call = fake_client.calls[0]
assert first_call["query"] == 'ti:"My Paper" AND au:"lovelace"'
assert first_call["request_email"] == "user@example.com"
assert first_call["timeout_seconds"] == 2.0
assert first_call["max_results"] == 4
def test_build_arxiv_query_normalizes_noisy_mojibake_title() -> None:
noisy = " Graph–Neural Networks Survey "
clean = "Graph Neural Networks Survey"
assert arxiv_gateway.build_arxiv_query(noisy, None) == arxiv_gateway.build_arxiv_query(clean, None)

63
tests/unit/services/domains/arxiv/test_guards.py Normal file
View file

@ -0,0 +1,63 @@
from __future__ import annotations
from datetime import datetime, timezone
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.domains.publications.types import PublicationListItem
def _item(
*,
title: str,
pub_url: str | None = None,
pdf_url: str | None = None,
display_identifier: DisplayIdentifier | None = None,
) -> PublicationListItem:
return PublicationListItem(
publication_id=1,
scholar_profile_id=1,
scholar_label="Ada Lovelace",
title=title,
year=2024,
citation_count=0,
venue_text=None,
pub_url=pub_url,
pdf_url=pdf_url,
is_read=False,
first_seen_at=datetime.now(timezone.utc),
is_new_in_latest_run=True,
display_identifier=display_identifier,
)
def test_arxiv_skip_reason_for_strong_doi_evidence() -> None:
item = _item(
title="A Robust and Reproducible Deep Learning Benchmark",
display_identifier=DisplayIdentifier(
kind="doi",
value="10.1000/example",
label="DOI: 10.1000/example",
url="https://doi.org/10.1000/example",
confidence_score=1.0,
),
)
assert arxiv_skip_reason_for_item(item=item) == "strong_doi_present"
def test_arxiv_skip_reason_for_existing_arxiv_link() -> None:
item = _item(
title="A Robust and Reproducible Deep Learning Benchmark",
pub_url="https://arxiv.org/abs/2501.00001",
)
assert arxiv_skip_reason_for_item(item=item) == "arxiv_identifier_present"
def test_arxiv_skip_reason_for_low_quality_title() -> None:
item = _item(title="AI 2024")
assert arxiv_skip_reason_for_item(item=item) == "title_quality_below_threshold"
def test_arxiv_skip_reason_none_for_eligible_item() -> None:
item = _item(title="Reliable Graph Neural Network Benchmarking Across Multiple Datasets")
assert arxiv_skip_reason_for_item(item=item) is None

55
tests/unit/services/domains/arxiv/test_parser.py Normal file
View file

@ -0,0 +1,55 @@
from __future__ import annotations
import pytest
from app.services.domains.arxiv.errors import ArxivParseError
from app.services.domains.arxiv.parser import parse_arxiv_feed
_VALID_FEED_XML = """<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
xmlns:arxiv="http://arxiv.org/schemas/atom">
<opensearch:totalResults>2</opensearch:totalResults>
<opensearch:startIndex>0</opensearch:startIndex>
<opensearch:itemsPerPage>2</opensearch:itemsPerPage>
<entry>
<id>http://arxiv.org/abs/1234.5678v2</id>
<updated>2020-01-01T00:00:00Z</updated>
<published>2019-12-31T00:00:00Z</published>
<title>Test Entry</title>
<summary>Example summary</summary>
<author><name>Ada Lovelace</name></author>
<author><name>Grace Hopper</name></author>
<link href="http://arxiv.org/abs/1234.5678v2" />
<category term="cs.AI" />
<category term="stat.ML" />
<arxiv:primary_category term="cs.AI" />
</entry>
</feed>
"""
def test_parse_arxiv_feed_extracts_entries_and_opensearch_meta() -> None:
feed = parse_arxiv_feed(_VALID_FEED_XML)
assert feed.opensearch.total_results == 2
assert feed.opensearch.start_index == 0
assert feed.opensearch.items_per_page == 2
assert len(feed.entries) == 1
entry = feed.entries[0]
assert entry.arxiv_id == "1234.5678v2"
assert entry.title == "Test Entry"
assert entry.summary == "Example summary"
assert entry.authors == ["Ada Lovelace", "Grace Hopper"]
assert entry.primary_category == "cs.AI"
assert entry.categories == ["cs.AI", "stat.ML"]
def test_parse_arxiv_feed_raises_on_invalid_xml() -> None:
with pytest.raises(ArxivParseError):
parse_arxiv_feed("<feed><entry></feed>")
def test_parse_arxiv_feed_raises_on_invalid_opensearch_integer() -> None:
payload = _VALID_FEED_XML.replace("<opensearch:totalResults>2</opensearch:totalResults>", "<opensearch:totalResults>x</opensearch:totalResults>")
with pytest.raises(ArxivParseError):
parse_arxiv_feed(payload)

167
tests/unit/services/domains/arxiv/test_rate_limit.py Normal file
View file

@ -0,0 +1,167 @@
from __future__ import annotations
import asyncio
from datetime import datetime, timedelta, timezone
import httpx
import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import ArxivRuntimeState
from app.services.domains.arxiv.constants import ARXIV_RUNTIME_STATE_KEY
from app.services.domains.arxiv.errors import ArxivRateLimitError
from app.services.domains.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
from app.settings import settings
@pytest.mark.asyncio
async def test_arxiv_rate_limit_respects_cooldown(db_session: AsyncSession) -> None:
db_session.add(
ArxivRuntimeState(
state_key=ARXIV_RUNTIME_STATE_KEY,
cooldown_until=datetime.now(timezone.utc) + timedelta(seconds=30),
)
)
await db_session.commit()
called = {"count": 0}
async def _fetch() -> httpx.Response:
called["count"] += 1
return httpx.Response(200, text="ok")
with pytest.raises(ArxivRateLimitError):
await run_with_global_arxiv_limit(fetch=_fetch)
assert called["count"] == 0
@pytest.mark.asyncio
async def test_arxiv_rate_limit_persists_cooldown_after_429(db_session: AsyncSession) -> None:
previous_interval = settings.arxiv_min_interval_seconds
previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
object.__setattr__(settings, "arxiv_min_interval_seconds", 0.0)
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", 5.0)
try:
async def _fetch() -> httpx.Response:
return httpx.Response(429, text="rate limited")
with pytest.raises(ArxivRateLimitError):
await run_with_global_arxiv_limit(fetch=_fetch)
finally:
object.__setattr__(settings, "arxiv_min_interval_seconds", previous_interval)
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", previous_cooldown)
result = await db_session.execute(
select(ArxivRuntimeState).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
)
state = result.scalar_one()
assert state.cooldown_until is not None
assert state.cooldown_until > datetime.now(timezone.utc)
@pytest.mark.asyncio
async def test_arxiv_rate_limit_serializes_concurrent_calls(db_session: AsyncSession) -> None:
previous_interval = settings.arxiv_min_interval_seconds
previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
object.__setattr__(settings, "arxiv_min_interval_seconds", 0.2)
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", 5.0)
try:
call_times: list[float] = []
async def _fetch() -> httpx.Response:
call_times.append(asyncio.get_running_loop().time())
return httpx.Response(200, text="ok")
await asyncio.gather(
run_with_global_arxiv_limit(fetch=_fetch),
run_with_global_arxiv_limit(fetch=_fetch),
)
finally:
object.__setattr__(settings, "arxiv_min_interval_seconds", previous_interval)
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", previous_cooldown)
assert len(call_times) == 2
assert call_times[1] - call_times[0] >= 0.18
@pytest.mark.asyncio
async def test_arxiv_rate_limit_logs_request_scheduled_and_completed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
logged: list[dict[str, object]] = []
async def _fetch() -> httpx.Response:
return httpx.Response(200, text="ok")
def _capture_log(_msg: str, *args, **kwargs) -> None:
extra = kwargs.get("extra")
if isinstance(extra, dict):
logged.append(extra)
monkeypatch.setattr("app.services.domains.arxiv.rate_limit.logger.info", _capture_log)
await run_with_global_arxiv_limit(fetch=_fetch, source_path="search")
scheduled = [entry for entry in logged if entry.get("event") == "arxiv.request_scheduled"]
completed = [entry for entry in logged if entry.get("event") == "arxiv.request_completed"]
assert scheduled
assert completed
assert float(scheduled[0]["wait_seconds"]) >= 0.0
assert int(completed[0]["status_code"]) == 200
assert completed[0]["source_path"] == "search"
@pytest.mark.asyncio
async def test_arxiv_rate_limit_logs_cooldown_activation(
monkeypatch: pytest.MonkeyPatch,
) -> None:
logged_warning: list[dict[str, object]] = []
previous_interval = settings.arxiv_min_interval_seconds
previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
object.__setattr__(settings, "arxiv_min_interval_seconds", 0.0)
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", 5.0)
try:
async def _fetch() -> httpx.Response:
return httpx.Response(429, text="rate limited")
def _capture_warning(_msg: str, *args, **kwargs) -> None:
extra = kwargs.get("extra")
if isinstance(extra, dict):
logged_warning.append(extra)
monkeypatch.setattr("app.services.domains.arxiv.rate_limit.logger.warning", _capture_warning)
with pytest.raises(ArxivRateLimitError):
await run_with_global_arxiv_limit(fetch=_fetch, source_path="lookup_ids")
finally:
object.__setattr__(settings, "arxiv_min_interval_seconds", previous_interval)
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", previous_cooldown)
cooldown_events = [
entry for entry in logged_warning if entry.get("event") == "arxiv.cooldown_activated"
]
assert cooldown_events
assert cooldown_events[0]["source_path"] == "lookup_ids"
assert float(cooldown_events[0]["cooldown_remaining_seconds"]) > 0.0
@pytest.mark.asyncio
async def test_get_arxiv_cooldown_status_reads_active_cooldown(db_session: AsyncSession) -> None:
now_utc = datetime(2026, 2, 26, 13, 0, tzinfo=timezone.utc)
existing = await db_session.get(ArxivRuntimeState, ARXIV_RUNTIME_STATE_KEY)
if existing is None:
db_session.add(
ArxivRuntimeState(
state_key=ARXIV_RUNTIME_STATE_KEY,
cooldown_until=now_utc + timedelta(seconds=45),
)
)
else:
existing.cooldown_until = now_utc + timedelta(seconds=45)
await db_session.commit()
status = await get_arxiv_cooldown_status(now_utc=now_utc)
assert status.is_active is True
assert status.cooldown_until is not None
assert int(status.remaining_seconds) == 45

187
tests/unit/services/domains/publications/test_dedup.py
View file

@ -1,29 +1,30 @@
"""Unit tests for the identifier-based publication dedup sweep.
"""Unit tests for publication dedup operations."""
DB operations are mocked via AsyncMock so no database is required.
"""
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.db.models import ScholarPublication
from app.services.domains.publications import dedup as dedup_service
from app.services.domains.publications.dedup import (
NearDuplicateCluster,
NearDuplicateMember,
find_identifier_duplicate_pairs,
find_near_duplicate_clusters,
merge_duplicate_publication,
merge_near_duplicate_cluster,
sweep_identifier_duplicates,
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_result(rows: list) -> MagicMock:
mock_result = MagicMock()
mock_result.scalars.return_value.all.return_value = rows
mock_result.__iter__ = lambda self: iter(rows)
mock_result.all.return_value = rows
return mock_result
@ -35,10 +36,6 @@ def _session_with_execute_sequence(results: list) -> AsyncMock:
return session
# ---------------------------------------------------------------------------
# find_identifier_duplicate_pairs
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_find_identifier_duplicate_pairs_returns_pairs() -> None:
session = AsyncMock()
@ -60,56 +57,104 @@ async def test_find_identifier_duplicate_pairs_returns_empty_when_no_duplicates(
assert pairs == []
# ---------------------------------------------------------------------------
# merge_duplicate_publication
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_merge_duplicate_publication_migrates_links_and_identifiers() -> None:
session = AsyncMock()
winner = SimpleNamespace(
year=2014,
citation_count=10,
author_text=None,
venue_text=None,
pub_url=None,
pdf_url=None,
cluster_id=None,
canonical_title_hash=None,
title_raw="winner",
title_normalized="winner",
)
dup = SimpleNamespace(
year=2015,
citation_count=11,
author_text="a",
venue_text="v",
pub_url="https://example.org",
pdf_url="https://example.org/a.pdf",
cluster_id="x",
canonical_title_hash="hash",
title_raw="dup",
title_normalized="dup",
)
with (
patch(
"app.services.domains.publications.dedup._load_publication",
new=AsyncMock(side_effect=[winner, dup]),
),
patch(
"app.services.domains.publications.dedup._migrate_scholar_links",
new=AsyncMock(),
) as mock_links,
patch(
"app.services.domains.publications.dedup._migrate_identifiers",
new=AsyncMock(),
) as mock_identifiers,
):
await merge_duplicate_publication(session, winner_id=1, dup_id=2)
assert session.execute.await_count == 1
mock_links.assert_awaited_once_with(session, winner_id=1, dup_id=2)
mock_identifiers.assert_awaited_once_with(session, winner_id=1, dup_id=2)
@pytest.mark.asyncio
async def test_merge_duplicate_migrates_orphaned_scholar_links() -> None:
"""Scholar link that only the dup has should be migrated to winner."""
async def test_merge_duplicate_publication_rejects_missing_publications() -> None:
session = AsyncMock()
with patch(
"app.services.domains.publications.dedup._load_publication",
new=AsyncMock(side_effect=[None, None]),
):
with pytest.raises(ValueError):
await merge_duplicate_publication(session, winner_id=1, dup_id=2)
@pytest.mark.asyncio
async def test_migrate_scholar_links_moves_orphans() -> None:
dup_link = MagicMock(spec=ScholarPublication)
dup_link.scholar_profile_id = 99
dup_link.publication_id = 2
session = _session_with_execute_sequence(
results=[
[dup_link], # dup links
[], # winner profile ids (no conflict)
[], # execute(delete(Publication)) result
[dup_link],
[],
]
)
await merge_duplicate_publication(session, winner_id=1, dup_id=2)
await dedup_service._migrate_scholar_links(session, winner_id=1, dup_id=2)
assert dup_link.publication_id == 1
session.delete.assert_not_awaited() # not deleted; migrated instead
session.delete.assert_not_awaited()
@pytest.mark.asyncio
async def test_merge_duplicate_drops_conflicting_scholar_link() -> None:
"""When winner already has a link for the same scholar, dup's link is deleted."""
async def test_migrate_scholar_links_drops_conflicts() -> None:
dup_link = MagicMock(spec=ScholarPublication)
dup_link.scholar_profile_id = 88
dup_link.publication_id = 2
session = _session_with_execute_sequence(
results=[
[dup_link], # dup links
[(88,)], # winner profiles (conflict: profile 88 already linked)
[], # execute(delete(Publication)) result
[dup_link],
[(88,)],
]
)
await merge_duplicate_publication(session, winner_id=1, dup_id=2)
await dedup_service._migrate_scholar_links(session, winner_id=1, dup_id=2)
session.delete.assert_awaited_once_with(dup_link)
assert dup_link.publication_id == 2 # unchanged — link was deleted, not migrated
# ---------------------------------------------------------------------------
# sweep_identifier_duplicates
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_sweep_returns_zero_when_no_pairs() -> None:
with patch(
@ -145,7 +190,6 @@ async def test_sweep_returns_merge_count() -> None:
@pytest.mark.asyncio
async def test_sweep_merges_each_dup_only_once() -> None:
"""A dup sharing two identifiers with the winner appears twice in pairs but merged once."""
with (
patch(
"app.services.domains.publications.dedup.find_identifier_duplicate_pairs",
@ -161,3 +205,80 @@ async def test_sweep_merges_each_dup_only_once() -> None:
assert count == 1
assert mock_merge.await_count == 1
@pytest.mark.asyncio
async def test_find_near_duplicate_clusters_groups_similar_titles() -> None:
first = dedup_service._candidate_from_row(
publication_id=10,
title="Adam: A method for stochastic optimization",
year=2014,
citation_count=100,
)
second = dedup_service._candidate_from_row(
publication_id=11,
title="†œAdam: A method for stochastic optimization, †3rd Int. Conf. Learn. Represent.",
year=2015,
citation_count=50,
)
assert first is not None
assert second is not None
with patch(
"app.services.domains.publications.dedup._load_near_duplicate_candidates",
new=AsyncMock(return_value=[first, second]),
):
clusters = await find_near_duplicate_clusters(AsyncMock())
assert len(clusters) == 1
assert len(clusters[0].members) == 2
assert clusters[0].winner_publication_id == 10
@pytest.mark.asyncio
async def test_find_near_duplicate_clusters_skips_unrelated_titles() -> None:
first = dedup_service._candidate_from_row(
publication_id=21,
title="Adam optimizer",
year=2014,
citation_count=10,
)
second = dedup_service._candidate_from_row(
publication_id=22,
title="Diffusion models in vision",
year=2022,
citation_count=10,
)
assert first is not None
assert second is not None
with patch(
"app.services.domains.publications.dedup._load_near_duplicate_candidates",
new=AsyncMock(return_value=[first, second]),
):
clusters = await find_near_duplicate_clusters(AsyncMock())
assert clusters == []
@pytest.mark.asyncio
async def test_merge_near_duplicate_cluster_merges_non_winner_members() -> None:
cluster = NearDuplicateCluster(
cluster_key="abc",
winner_publication_id=5,
similarity_score=1.0,
members=(
NearDuplicateMember(publication_id=5, title="Winner", year=2014, citation_count=10),
NearDuplicateMember(publication_id=6, title="Dup", year=2014, citation_count=3),
NearDuplicateMember(publication_id=7, title="Dup2", year=2014, citation_count=1),
),
)
with patch(
"app.services.domains.publications.dedup.merge_duplicate_publication",
new=AsyncMock(),
) as mock_merge:
merged = await merge_near_duplicate_cluster(AsyncMock(), cluster=cluster)
assert merged == 2
assert mock_merge.await_count == 2

32
tests/unit/test_fingerprints.py
View file

@ -240,3 +240,35 @@ class TestCanonicalTitleForDedup:
]
canonicals = [canonical_title_for_dedup(v) for v in variants]
assert len(set(canonicals)) == 1, f"Expected one canonical, got: {canonicals}"
def test_strips_mojibake_conference_suffix(self) -> None:
noisy = (
"†œAdam: A method for stochastic optimization, "
"†3rd Int. Conf. Learn. Represent. ICLR 2015-Conf"
)
clean = "Adam: A method for stochastic optimization"
assert canonical_title_for_dedup(noisy) == normalize_title(clean)
def test_preserves_clean_subtitle_not_venue_metadata(self) -> None:
title = "Vision-Language Models - A Survey"
assert canonical_title_for_dedup(title) == normalize_title(title)
def test_strips_leading_author_fragment_before_core_title(self) -> None:
noisy = "and Ba.J.:Adam: a method for stochastic optimization"
clean = "Adam: a method for stochastic optimization"
assert canonical_title_for_dedup(noisy) == normalize_title(clean)
def test_strips_leading_date_prefix(self) -> None:
noisy = "January 7-9). Adam: A method for stochastic optimization"
clean = "Adam: A method for stochastic optimization"
assert canonical_title_for_dedup(noisy) == normalize_title(clean)
def test_strips_trailing_publication_type(self) -> None:
noisy = "Adam: A method for stochastic optimization. conference paper"
clean = "Adam: A method for stochastic optimization"
assert canonical_title_for_dedup(noisy) == normalize_title(clean)
def test_strips_trailing_month_year_parenthetical(self) -> None:
noisy = "Adam: A method for stochastic optimization (Jan 2017)"
clean = "Adam: A method for stochastic optimization"
assert canonical_title_for_dedup(noisy) == normalize_title(clean)

47
tests/unit/test_ingestion_arxiv_rate_limit.py Normal file
View file

@ -0,0 +1,47 @@
from __future__ import annotations
from types import SimpleNamespace
import pytest
from app.services.domains.arxiv.errors import ArxivRateLimitError
from app.services.domains.ingestion.application import ScholarIngestionService
from app.services.domains.publication_identifiers import application as identifier_service
@pytest.mark.asyncio
async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
service = ScholarIngestionService(source=object())
publication = SimpleNamespace(id=11, author_text="Ada Lovelace")
calls = {"sync": 0}
async def _raise_rate_limit(db_session, *, publication, scholar_label):
_ = (db_session, publication, scholar_label)
raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
async def _sync_fields(db_session, *, publication):
_ = (db_session, publication)
calls["sync"] += 1
monkeypatch.setattr(
identifier_service,
"discover_and_sync_identifiers_for_publication",
_raise_rate_limit,
)
monkeypatch.setattr(identifier_service, "sync_identifiers_for_publication_fields", _sync_fields)
async def _publish_noop(*args, **kwargs) -> None:
_ = (args, kwargs)
monkeypatch.setattr(service, "_publish_identifier_update_event", _publish_noop)
result = await service._discover_identifiers_for_enrichment(
object(),
publication=publication,
run_id=321,
allow_arxiv_lookup=True,
)
assert result is False
assert calls["sync"] == 1

61
tests/unit/test_ingestion_progress_reporting.py Normal file
View file

@ -0,0 +1,61 @@
from __future__ import annotations
import pytest
from app.services.domains.ingestion.application import ScholarIngestionService
from app.services.domains.ingestion.types import RunProgress, ScholarProcessingOutcome
def _outcome(*, scholar_profile_id: int, outcome_label: str) -> ScholarProcessingOutcome:
counters = {
"success": (1, 0, 0),
"partial": (1, 0, 1),
"failed": (0, 1, 0),
}
succeeded, failed, partial = counters[outcome_label]
return ScholarProcessingOutcome(
result_entry={
"scholar_profile_id": scholar_profile_id,
"outcome": outcome_label,
"state": "ok",
"state_reason": "publications_extracted",
"publication_count": 1,
},
succeeded_count_delta=succeeded,
failed_count_delta=failed,
partial_count_delta=partial,
discovered_publication_count=1,
)
def test_apply_outcome_to_progress_replaces_previous_scholar_outcome() -> None:
progress = RunProgress()
ScholarIngestionService._apply_outcome_to_progress(
progress=progress,
outcome=_outcome(scholar_profile_id=42, outcome_label="partial"),
)
ScholarIngestionService._apply_outcome_to_progress(
progress=progress,
outcome=_outcome(scholar_profile_id=42, outcome_label="success"),
)
assert len(progress.scholar_results) == 1
assert progress.scholar_results[0]["outcome"] == "success"
assert progress.succeeded_count == 1
assert progress.failed_count == 0
assert progress.partial_count == 0
def test_apply_outcome_to_progress_rejects_invalid_scholar_id() -> None:
progress = RunProgress()
invalid = ScholarProcessingOutcome(
result_entry={"scholar_profile_id": 0, "outcome": "success"},
succeeded_count_delta=1,
failed_count_delta=0,
partial_count_delta=0,
discovered_publication_count=0,
)
with pytest.raises(RuntimeError, match="missing valid scholar_profile_id"):
ScholarIngestionService._apply_outcome_to_progress(progress=progress, outcome=invalid)

148
tests/unit/test_publication_identifiers.py
View file

@ -1,6 +1,14 @@
from __future__ import annotations
import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Publication, PublicationIdentifier
from app.services.domains.arxiv import application as arxiv_service
from app.services.domains.publication_identifiers import application as identifier_service
from app.services.domains.publication_identifiers.types import IdentifierKind
from app.services.domains.publications.types import UnreadPublicationItem
def test_derive_display_identifier_prefers_doi_over_arxiv() -> None:
@ -40,7 +48,7 @@ def test_derive_display_identifier_uses_pmcid_when_present() -> None:
def test_normalize_arxiv_id_handles_urls() -> None:
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
assert normalize_arxiv_id("https://arxiv.org/abs/1504.08025") == "1504.08025"
assert normalize_arxiv_id("http://arxiv.org/pdf/1504.08025v2.pdf") == "1504.08025v2"
assert normalize_arxiv_id("https://arxiv.org/html/1504.08025v2") == "1504.08025v2"
@ -52,7 +60,8 @@ def test_normalize_arxiv_id_handles_urls() -> None:
def test_normalize_arxiv_id_handles_raw_text() -> None:
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
assert normalize_arxiv_id("1504.08025v1") == "1504.08025v1"
assert normalize_arxiv_id("arXiv:1504.08025") == "1504.08025"
assert normalize_arxiv_id("arxiv: 1504.08025v1") == "1504.08025v1"
assert normalize_arxiv_id("Preprint at arXiv:math/9901123v2") == "math/9901123v2"
@ -61,7 +70,7 @@ def test_normalize_arxiv_id_handles_raw_text() -> None:
def test_normalize_pmcid_handles_urls_and_text() -> None:
from app.services.domains.publication_identifiers.normalize import normalize_pmcid
assert normalize_pmcid("https://pmc.ncbi.nlm.nih.gov/articles/PMC2175868/") == "PMC2175868"
assert normalize_pmcid("http://www.ncbi.nlm.nih.gov/pmc/articles/PMC1234567") == "PMC1234567"
assert normalize_pmcid("PMCID: PMC1234567") == "PMC1234567"
@ -71,15 +80,140 @@ def test_normalize_pmcid_handles_urls_and_text() -> None:
def test_normalize_pmid_handles_urls() -> None:
from app.services.domains.publication_identifiers.normalize import normalize_pmid
assert normalize_pmid("https://pubmed.ncbi.nlm.nih.gov/12345678/") == "12345678"
assert normalize_pmid("http://pubmed.ncbi.nlm.nih.gov/12345678") == "12345678"
assert normalize_pmid("https://pubmed.ncbi.nlm.nih.gov/not-a-pmid/") is None
import pytest
from app.services.domains.arxiv import application as arxiv_service
from app.services.domains.publications.types import UnreadPublicationItem
def _publication(*, title: str, pub_url: str | None = None) -> Publication:
return Publication(
cluster_id=None,
fingerprint_sha256="f" * 64,
title_raw=title,
title_normalized=title.lower(),
year=2024,
citation_count=0,
author_text=None,
venue_text=None,
pub_url=pub_url,
pdf_url=None,
)
@pytest.mark.asyncio
async def test_discover_and_sync_skips_arxiv_when_crossref_finds_doi(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
publication = _publication(title="Reliable Paper Title for DOI discovery")
db_session.add(publication)
await db_session.flush()
async def _fake_crossref(*, item):
return "10.1000/strong-doi"
async def _fail_arxiv(*, item, request_email=None, timeout_seconds=None):
raise AssertionError("arXiv should be skipped after strong DOI evidence.")
monkeypatch.setattr(
"app.services.domains.crossref.application.discover_doi_for_publication",
_fake_crossref,
)
monkeypatch.setattr(
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication",
_fail_arxiv,
)
await identifier_service.discover_and_sync_identifiers_for_publication(
db_session,
publication=publication,
scholar_label="Ada Lovelace",
)
result = await db_session.execute(
select(PublicationIdentifier).where(
PublicationIdentifier.publication_id == int(publication.id),
PublicationIdentifier.kind == IdentifierKind.DOI.value,
)
)
assert result.scalar_one_or_none() is not None
@pytest.mark.asyncio
async def test_discover_and_sync_skips_arxiv_for_low_quality_title(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
publication = _publication(title="AI 2024")
db_session.add(publication)
await db_session.flush()
calls = {"count": 0}
async def _fake_crossref(*, item):
return None
async def _fake_arxiv(*, item, request_email=None, timeout_seconds=None):
calls["count"] += 1
return "1234.5678"
monkeypatch.setattr(
"app.services.domains.crossref.application.discover_doi_for_publication",
_fake_crossref,
)
monkeypatch.setattr(
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication",
_fake_arxiv,
)
await identifier_service.discover_and_sync_identifiers_for_publication(
db_session,
publication=publication,
scholar_label="Alan Turing",
)
assert calls["count"] == 0
@pytest.mark.asyncio
async def test_discover_and_sync_calls_arxiv_when_item_is_eligible(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
publication = _publication(title="Neural Representation Learning for Graph Signals")
db_session.add(publication)
await db_session.flush()
async def _fake_crossref(*, item):
return None
async def _fake_arxiv(*, item, request_email=None, timeout_seconds=None):
return "2501.00001v2"
monkeypatch.setattr(
"app.services.domains.crossref.application.discover_doi_for_publication",
_fake_crossref,
)
monkeypatch.setattr(
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication",
_fake_arxiv,
)
await identifier_service.discover_and_sync_identifiers_for_publication(
db_session,
publication=publication,
scholar_label="Grace Hopper",
)
result = await db_session.execute(
select(PublicationIdentifier).where(
PublicationIdentifier.publication_id == int(publication.id),
PublicationIdentifier.kind == IdentifierKind.ARXIV.value,
)
)
identifier = result.scalar_one_or_none()
assert identifier is not None
assert identifier.value_normalized == "2501.00001v2"
@pytest.mark.asyncio
async def test_discover_arxiv_id_returns_none_if_no_title() -> None:

78
tests/unit/test_publication_pdf_queue_policy.py
View file

@ -6,7 +6,6 @@ from types import SimpleNamespace
import pytest
from app.db.models import PublicationPdfJob
from app.services.domains.arxiv.application import ArxivRateLimitError
from app.services.domains.publications import pdf_queue
from app.services.domains.publications.pdf_resolution_pipeline import PipelineOutcome
from app.services.domains.unpaywall.application import OaResolutionOutcome
@ -134,8 +133,9 @@ def test_pdf_queue_manual_requeue_still_blocks_when_inflight() -> None:
@pytest.mark.asyncio
async def test_fetch_outcome_for_row_uses_pipeline_outcome(monkeypatch: pytest.MonkeyPatch) -> None:
async def _fake_pipeline(*, row, request_email=None, openalex_api_key=None):
async def _fake_pipeline(*, row, request_email=None, openalex_api_key=None, allow_arxiv_lookup=True):
assert request_email == "user@example.com"
assert allow_arxiv_lookup is True
return PipelineOutcome(
outcome=OaResolutionOutcome(
publication_id=row.publication_id,
@ -150,31 +150,40 @@ async def test_fetch_outcome_for_row_uses_pipeline_outcome(monkeypatch: pytest.M
monkeypatch.setattr(pdf_queue, "resolve_publication_pdf_outcome_for_row", _fake_pipeline)
outcome = await pdf_queue._fetch_outcome_for_row(row=_row(), request_email="user@example.com")
outcome, arxiv_rate_limited = await pdf_queue._fetch_outcome_for_row(
row=_row(),
request_email="user@example.com",
)
assert outcome.pdf_url == "https://arxiv.org/pdf/1703.06103"
assert outcome.source == "openalex"
assert outcome.used_crossref is False
assert arxiv_rate_limited is False
@pytest.mark.asyncio
async def test_fetch_outcome_for_row_returns_failed_outcome_when_pipeline_returns_none(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def _fake_pipeline(*, row, request_email=None, openalex_api_key=None):
async def _fake_pipeline(*, row, request_email=None, openalex_api_key=None, allow_arxiv_lookup=True):
assert request_email == "user@example.com"
return PipelineOutcome(outcome=None, scholar_candidates=None)
assert allow_arxiv_lookup is True
return PipelineOutcome(outcome=None, scholar_candidates=None, arxiv_rate_limited=True)
monkeypatch.setattr(pdf_queue, "resolve_publication_pdf_outcome_for_row", _fake_pipeline)
outcome = await pdf_queue._fetch_outcome_for_row(row=_row(), request_email="user@example.com")
outcome, arxiv_rate_limited = await pdf_queue._fetch_outcome_for_row(
row=_row(),
request_email="user@example.com",
)
assert outcome.pdf_url is None
assert outcome.failure_reason == pdf_queue.FAILURE_RESOLUTION_EXCEPTION
assert arxiv_rate_limited is True
@pytest.mark.asyncio
async def test_resolve_publication_row_persists_failed_outcome_before_reraising_arxiv_rate_limit(
async def test_resolve_publication_row_persists_outcome_and_returns_rate_limit_flag(
monkeypatch: pytest.MonkeyPatch,
) -> None:
captured: list[tuple[int, int, OaResolutionOutcome]] = []
@ -182,47 +191,64 @@ async def test_resolve_publication_row_persists_failed_outcome_before_reraising_
async def _noop_mark_attempt_started(*, publication_id: int, user_id: int) -> None:
return None
async def _raise_rate_limit(*, row, request_email=None, openalex_api_key=None):
raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
async def _fake_fetch(*, row, request_email=None, openalex_api_key=None, allow_arxiv_lookup=True):
assert allow_arxiv_lookup is True
return (
OaResolutionOutcome(
publication_id=row.publication_id,
doi=None,
pdf_url="https://fallback.example/test.pdf",
failure_reason=None,
source="unpaywall",
used_crossref=False,
),
True,
)
async def _capture_persist_outcome(*, publication_id: int, user_id: int, outcome: OaResolutionOutcome) -> None:
captured.append((publication_id, user_id, outcome))
monkeypatch.setattr(pdf_queue, "_mark_attempt_started", _noop_mark_attempt_started)
monkeypatch.setattr(pdf_queue, "_fetch_outcome_for_row", _raise_rate_limit)
monkeypatch.setattr(pdf_queue, "_fetch_outcome_for_row", _fake_fetch)
monkeypatch.setattr(pdf_queue, "_persist_outcome", _capture_persist_outcome)
with pytest.raises(ArxivRateLimitError):
await pdf_queue._resolve_publication_row(
user_id=42,
request_email="user@example.com",
row=_row(),
openalex_api_key="key",
)
rate_limited = await pdf_queue._resolve_publication_row(
user_id=42,
request_email="user@example.com",
row=_row(),
openalex_api_key="key",
)
assert len(captured) == 1
publication_id, user_id, outcome = captured[0]
assert publication_id == 1
assert user_id == 42
assert outcome.pdf_url is None
assert outcome.failure_reason == pdf_queue.FAILURE_RESOLUTION_EXCEPTION
assert outcome.pdf_url == "https://fallback.example/test.pdf"
assert rate_limited is True
@pytest.mark.asyncio
async def test_run_resolution_task_stops_batch_on_arxiv_rate_limit(
async def test_run_resolution_task_disables_arxiv_for_remaining_batch(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls: list[int] = []
calls: list[tuple[int, bool]] = []
first = _row()
second = SimpleNamespace(**{**first.__dict__, "publication_id": 2})
def _raise_session_factory_error():
raise RuntimeError("skip user settings lookup in test")
async def _fake_resolve_publication_row(*, user_id: int, request_email: str | None, row, openalex_api_key=None):
calls.append(int(row.publication_id))
if row.publication_id == 1:
raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
async def _fake_resolve_publication_row(
*,
user_id: int,
request_email: str | None,
row,
openalex_api_key=None,
allow_arxiv_lookup=True,
):
_ = (user_id, request_email, openalex_api_key)
calls.append((int(row.publication_id), bool(allow_arxiv_lookup)))
return row.publication_id == 1
monkeypatch.setattr(pdf_queue, "get_session_factory", _raise_session_factory_error)
monkeypatch.setattr(pdf_queue, "_resolve_publication_row", _fake_resolve_publication_row)
@ -233,4 +259,4 @@ async def test_run_resolution_task_stops_batch_on_arxiv_rate_limit(
rows=[first, second],
)
assert calls == [1]
assert calls == [(1, True), (2, False)]

100
tests/unit/test_publication_pdf_resolution_pipeline.py
View file

@ -5,6 +5,7 @@ from types import SimpleNamespace
import pytest
from app.services.domains.arxiv.errors import ArxivRateLimitError
from app.services.domains.publications import pdf_resolution_pipeline as pipeline
from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.domains.unpaywall.application import OaResolutionOutcome
@ -58,7 +59,8 @@ async def test_pipeline_prefers_openalex_before_arxiv(monkeypatch: pytest.Monkey
async def _fake_openalex(row, request_email: str | None = None, openalex_api_key: str | None = None):
return _api_outcome(pdf_url="https://oa.example.org/found.pdf", source="openalex")
async def _fail_arxiv(row):
async def _fail_arxiv(row, *, request_email: str | None = None, allow_lookup: bool = True):
_ = (row, request_email, allow_lookup)
raise AssertionError(f"arXiv should not run when OpenAlex candidate exists.")
monkeypatch.setattr(pipeline, "_openalex_outcome", _fake_openalex)
@ -76,7 +78,8 @@ async def test_pipeline_uses_arxiv_after_openalex_failure(monkeypatch: pytest.Mo
async def _fake_openalex(row, request_email: str | None = None, openalex_api_key: str | None = None):
return None
async def _fake_arxiv(row, request_email: str | None = None):
async def _fake_arxiv(row, *, request_email: str | None = None, allow_lookup: bool = True):
_ = allow_lookup
return _api_outcome(pdf_url="https://arxiv.org/pdf/1234.5678.pdf", source="arxiv")
async def _fail_oa(*, row, request_email):
@ -98,7 +101,8 @@ async def test_pipeline_uses_unpaywall_after_arxiv_failure(monkeypatch: pytest.M
async def _fake_openalex(row, request_email: str | None = None, openalex_api_key: str | None = None):
return None
async def _fake_arxiv(row, request_email: str | None = None):
async def _fake_arxiv(row, *, request_email: str | None = None, allow_lookup: bool = True):
_ = (request_email, allow_lookup)
return None
async def _fake_oa(*, row, request_email):
@ -114,3 +118,93 @@ async def test_pipeline_uses_unpaywall_after_arxiv_failure(monkeypatch: pytest.M
assert result.outcome is not None
assert result.outcome.pdf_url == "https://example.org/fallback.pdf"
assert result.outcome.source == "unpaywall"
@pytest.mark.asyncio
async def test_pipeline_falls_back_to_unpaywall_when_arxiv_is_rate_limited(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def _fake_openalex(row, request_email: str | None = None, openalex_api_key: str | None = None):
_ = (row, request_email, openalex_api_key)
return None
async def _raise_rate_limit(row, *, request_email: str | None = None, allow_lookup: bool = True):
_ = (row, request_email, allow_lookup)
raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
async def _fake_oa(*, row, request_email):
_ = (row, request_email)
return _oa_fallback_outcome(pdf_url="https://example.org/fallback.pdf", source="unpaywall")
monkeypatch.setattr(pipeline, "_openalex_outcome", _fake_openalex)
monkeypatch.setattr(pipeline, "_arxiv_outcome", _raise_rate_limit)
monkeypatch.setattr(pipeline, "_oa_outcome", _fake_oa)
result = await pipeline.resolve_publication_pdf_outcome_for_row(row=_row(), request_email="user@example.com")
assert result.outcome is not None
assert result.outcome.source == "unpaywall"
assert result.arxiv_rate_limited is True
@pytest.mark.asyncio
async def test_arxiv_outcome_skips_when_strong_doi_identifier(
monkeypatch: pytest.MonkeyPatch,
) -> None:
row = _row(
display_identifier=DisplayIdentifier(
kind="doi",
value="10.1000/example",
label="DOI",
url="https://doi.org/10.1000/example",
confidence_score=1.0,
)
)
async def _fail_discover(*, item, request_email: str | None = None, timeout_seconds: float | None = None):
raise AssertionError("arXiv lookup should be skipped when DOI evidence is strong.")
monkeypatch.setattr(
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication",
_fail_discover,
)
outcome = await pipeline._arxiv_outcome(row, request_email="user@example.com")
assert outcome is None
@pytest.mark.asyncio
async def test_arxiv_outcome_skips_when_title_quality_is_low(
monkeypatch: pytest.MonkeyPatch,
) -> None:
row = _row()
row.title = "AI 2024"
async def _fail_discover(*, item, request_email: str | None = None, timeout_seconds: float | None = None):
raise AssertionError("arXiv lookup should be skipped for low-quality titles.")
monkeypatch.setattr(
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication",
_fail_discover,
)
outcome = await pipeline._arxiv_outcome(row, request_email="user@example.com")
assert outcome is None
@pytest.mark.asyncio
async def test_arxiv_outcome_calls_arxiv_when_eligible(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def _fake_discover(*, item, request_email: str | None = None, timeout_seconds: float | None = None):
return "1234.5678"
monkeypatch.setattr(
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication",
_fake_discover,
)
row = _row()
row.title = "Reliable Graph Neural Network Benchmark across Multiple Datasets"
outcome = await pipeline._arxiv_outcome(row, request_email="user@example.com")
assert outcome is not None
assert outcome.source == "arxiv"
assert outcome.pdf_url == "https://arxiv.org/pdf/1234.5678.pdf"

39
tests/unit/test_scholar_source.py
View file

@ -1,4 +1,7 @@
from app.services.domains.scholar.source import _build_profile_url
import pytest
from app.services.domains.scholar import rate_limit as scholar_rate_limit
from app.services.domains.scholar.source import FetchResult, LiveScholarSource, _build_profile_url
def test_build_profile_url_includes_pagesize_for_initial_page() -> None:
@ -11,3 +14,37 @@ def test_build_profile_url_includes_pagesize_for_initial_page() -> None:
assert "user=abcDEF123456" in url
assert "pagesize=100" in url
assert "cstart=" not in url
@pytest.mark.asyncio
async def test_live_scholar_source_applies_global_throttle(monkeypatch: pytest.MonkeyPatch) -> None:
captured_interval: list[float] = []
async def _fake_wait_for_scholar_slot(*, min_interval_seconds: float) -> None:
captured_interval.append(min_interval_seconds)
monkeypatch.setattr(
scholar_rate_limit,
"wait_for_scholar_slot",
_fake_wait_for_scholar_slot,
)
expected_result = FetchResult(
requested_url="https://example.test/scholar",
status_code=200,
final_url="https://example.test/scholar",
body="ok",
error=None,
)
source = LiveScholarSource(min_interval_seconds=7.0)
monkeypatch.setattr(source, "_fetch_sync", lambda _url: expected_result)
result = await source.fetch_profile_page_html(
"abcDEF123456",
cstart=0,
pagesize=100,
)
assert result == expected_result
assert captured_interval == [7.0]

144
tests/unit/test_scholars_create_hydration.py Normal file
View file

@ -0,0 +1,144 @@
from types import SimpleNamespace
import pytest
from app.api.routers import scholars as scholars_router
class _FakeSession:
def __init__(self) -> None:
self.commits = 0
self.rollbacks = 0
async def commit(self) -> None:
self.commits += 1
async def rollback(self) -> None:
self.rollbacks += 1
@pytest.mark.asyncio
async def test_create_hydration_skips_when_global_throttle_active(
monkeypatch: pytest.MonkeyPatch,
) -> None:
profile = SimpleNamespace(
id=42,
profile_image_url=None,
display_name="",
)
async def _fail_hydration(*_args, **_kwargs):
raise AssertionError("hydrate_profile_metadata should not run when throttle is active")
monkeypatch.setattr(
scholars_router.scholar_rate_limit,
"remaining_scholar_slot_seconds",
lambda **_kwargs: 3.5,
)
monkeypatch.setattr(
scholars_router.scholar_service,
"hydrate_profile_metadata",
_fail_hydration,
)
result = await scholars_router._hydrate_scholar_metadata_if_needed(
db_session=None,
profile=profile,
source=object(),
user_id=7,
)
assert result is profile
@pytest.mark.asyncio
async def test_create_hydration_runs_when_throttle_is_clear(
monkeypatch: pytest.MonkeyPatch,
) -> None:
profile = SimpleNamespace(
id=84,
profile_image_url=None,
display_name="",
)
async def _hydrate_profile_metadata(*_args, **_kwargs):
profile.display_name = "Ada Lovelace"
return profile
monkeypatch.setattr(
scholars_router.scholar_rate_limit,
"remaining_scholar_slot_seconds",
lambda **_kwargs: 0.0,
)
monkeypatch.setattr(
scholars_router.scholar_service,
"hydrate_profile_metadata",
_hydrate_profile_metadata,
)
result = await scholars_router._hydrate_scholar_metadata_if_needed(
db_session=None,
profile=profile,
source=object(),
user_id=8,
)
assert result.display_name == "Ada Lovelace"
@pytest.mark.asyncio
async def test_initial_scrape_job_enqueued_on_create(
monkeypatch: pytest.MonkeyPatch,
) -> None:
profile = SimpleNamespace(id=101)
session = _FakeSession()
upsert_calls: list[dict[str, object]] = []
async def _fake_upsert_job(*_args, **kwargs):
upsert_calls.append(kwargs)
monkeypatch.setattr(
scholars_router,
"_auto_enqueue_new_scholar_enabled",
lambda: True,
)
monkeypatch.setattr(
scholars_router.ingestion_queue_service,
"upsert_job",
_fake_upsert_job,
)
queued = await scholars_router._enqueue_initial_scrape_job_for_scholar(
session,
profile=profile,
user_id=9,
)
assert queued is True
assert session.commits == 1
assert session.rollbacks == 0
assert upsert_calls[0]["reason"] == scholars_router.INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON
@pytest.mark.asyncio
async def test_initial_scrape_job_not_enqueued_when_disabled(
monkeypatch: pytest.MonkeyPatch,
) -> None:
profile = SimpleNamespace(id=202)
session = _FakeSession()
monkeypatch.setattr(
scholars_router,
"_auto_enqueue_new_scholar_enabled",
lambda: False,
)
queued = await scholars_router._enqueue_initial_scrape_job_for_scholar(
session,
profile=profile,
user_id=11,
)
assert queued is False
assert session.commits == 0
assert session.rollbacks == 0