Intermediate commit
This commit is contained in:
parent
0e9e49df16
commit
3d4cfeff1a
65 changed files with 5507 additions and 333 deletions
134
tests/unit/services/domains/arxiv/test_cache.py
Normal file
134
tests/unit/services/domains/arxiv/test_cache.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import ArxivQueryCacheEntry
|
||||
from app.services.domains.arxiv.cache import (
|
||||
build_query_fingerprint,
|
||||
get_cached_feed,
|
||||
run_with_inflight_dedupe,
|
||||
set_cached_feed,
|
||||
)
|
||||
from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
||||
|
||||
|
||||
def _sample_feed(arxiv_id: str = "1234.5678") -> ArxivFeed:
|
||||
return ArxivFeed(
|
||||
entries=[
|
||||
ArxivEntry(
|
||||
entry_id_url=f"https://arxiv.org/abs/{arxiv_id}",
|
||||
arxiv_id=arxiv_id,
|
||||
title="Sample",
|
||||
summary="Summary",
|
||||
published=None,
|
||||
updated=None,
|
||||
)
|
||||
],
|
||||
opensearch=ArxivOpenSearchMeta(total_results=1, start_index=0, items_per_page=1),
|
||||
)
|
||||
|
||||
|
||||
def test_build_query_fingerprint_normalizes_search_and_id_params() -> None:
|
||||
first = build_query_fingerprint(
|
||||
params={
|
||||
"search_query": ' TI:"Quantum Fields" AND AU:"Doe" ',
|
||||
"start": 0,
|
||||
"max_results": 3,
|
||||
}
|
||||
)
|
||||
second = build_query_fingerprint(
|
||||
params={
|
||||
"search_query": 'ti:"quantum fields" and au:"doe"',
|
||||
"start": 0,
|
||||
"max_results": 3,
|
||||
}
|
||||
)
|
||||
third = build_query_fingerprint(params={"id_list": " 2222.0002,1111.0001 "})
|
||||
fourth = build_query_fingerprint(params={"id_list": "1111.0001, 2222.0002"})
|
||||
|
||||
assert first == second
|
||||
assert third == fourth
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_entry_expires_and_is_deleted(db_session: AsyncSession) -> None:
|
||||
query_fingerprint = build_query_fingerprint(params={"search_query": "ti:test", "start": 0})
|
||||
now_utc = datetime(2026, 2, 26, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
await set_cached_feed(
|
||||
query_fingerprint=query_fingerprint,
|
||||
feed=_sample_feed(),
|
||||
ttl_seconds=5.0,
|
||||
max_entries=16,
|
||||
now_utc=now_utc,
|
||||
)
|
||||
|
||||
hit = await get_cached_feed(
|
||||
query_fingerprint=query_fingerprint,
|
||||
now_utc=now_utc + timedelta(seconds=2),
|
||||
)
|
||||
miss = await get_cached_feed(
|
||||
query_fingerprint=query_fingerprint,
|
||||
now_utc=now_utc + timedelta(seconds=8),
|
||||
)
|
||||
|
||||
result = await db_session.execute(
|
||||
select(ArxivQueryCacheEntry).where(
|
||||
ArxivQueryCacheEntry.query_fingerprint == query_fingerprint
|
||||
)
|
||||
)
|
||||
assert hit is not None
|
||||
assert miss is None
|
||||
assert result.scalar_one_or_none() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inflight_dedupe_coalesces_identical_requests() -> None:
|
||||
calls = {"count": 0}
|
||||
|
||||
async def _fetch_feed() -> ArxivFeed:
|
||||
calls["count"] += 1
|
||||
await asyncio.sleep(0.05)
|
||||
return _sample_feed("9999.0001")
|
||||
|
||||
first, second = await asyncio.gather(
|
||||
run_with_inflight_dedupe(query_fingerprint="same-key", fetch_feed=_fetch_feed),
|
||||
run_with_inflight_dedupe(query_fingerprint="same-key", fetch_feed=_fetch_feed),
|
||||
)
|
||||
|
||||
assert calls["count"] == 1
|
||||
assert first.entries[0].arxiv_id == "9999.0001"
|
||||
assert second.entries[0].arxiv_id == "9999.0001"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inflight_owner_failure_without_joiner_has_no_unretrieved_exception() -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
messages: list[str] = []
|
||||
previous_handler = loop.get_exception_handler()
|
||||
|
||||
def _capture_exception(_loop, context) -> None:
|
||||
messages.append(str(context.get("message", "")))
|
||||
|
||||
loop.set_exception_handler(_capture_exception)
|
||||
try:
|
||||
async def _failing_fetch() -> ArxivFeed:
|
||||
raise RuntimeError("owner_failed")
|
||||
|
||||
with pytest.raises(RuntimeError, match="owner_failed"):
|
||||
await run_with_inflight_dedupe(
|
||||
query_fingerprint="owner-failure-no-joiner",
|
||||
fetch_feed=_failing_fetch,
|
||||
)
|
||||
gc.collect()
|
||||
await asyncio.sleep(0)
|
||||
finally:
|
||||
loop.set_exception_handler(previous_handler)
|
||||
|
||||
assert "Future exception was never retrieved" not in " | ".join(messages)
|
||||
202
tests/unit/services/domains/arxiv/test_client.py
Normal file
202
tests/unit/services/domains/arxiv/test_client.py
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.services.domains.arxiv import client as arxiv_client_module
|
||||
from app.services.domains.arxiv.client import ArxivClient
|
||||
from app.services.domains.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
|
||||
from app.services.domains.arxiv.rate_limit import ArxivCooldownStatus
|
||||
|
||||
_CLIENT_FEED_XML = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom"
|
||||
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
|
||||
xmlns:arxiv="http://arxiv.org/schemas/atom">
|
||||
<opensearch:totalResults>1</opensearch:totalResults>
|
||||
<opensearch:startIndex>0</opensearch:startIndex>
|
||||
<opensearch:itemsPerPage>1</opensearch:itemsPerPage>
|
||||
<entry>
|
||||
<id>http://arxiv.org/abs/9999.0001</id>
|
||||
<title>Client Entry</title>
|
||||
<summary>Client summary</summary>
|
||||
</entry>
|
||||
</feed>
|
||||
"""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_search_builds_query_and_sort_params() -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def _request_fn(*, params, request_email, timeout_seconds):
|
||||
captured["params"] = params
|
||||
captured["request_email"] = request_email
|
||||
captured["timeout_seconds"] = timeout_seconds
|
||||
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
|
||||
|
||||
client = ArxivClient(request_fn=_request_fn)
|
||||
feed = await client.search(
|
||||
query='ti:"test"',
|
||||
start=5,
|
||||
max_results=7,
|
||||
sort_by="submittedDate",
|
||||
sort_order="ascending",
|
||||
request_email="user@example.com",
|
||||
timeout_seconds=3.5,
|
||||
)
|
||||
|
||||
assert feed.opensearch.total_results == 1
|
||||
assert captured["params"] == {
|
||||
"search_query": 'ti:"test"',
|
||||
"start": 5,
|
||||
"max_results": 7,
|
||||
"sortBy": "submittedDate",
|
||||
"sortOrder": "ascending",
|
||||
}
|
||||
assert captured["request_email"] == "user@example.com"
|
||||
assert captured["timeout_seconds"] == 3.5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_lookup_ids_builds_id_list_param() -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def _request_fn(*, params, request_email, timeout_seconds):
|
||||
captured["params"] = params
|
||||
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
|
||||
|
||||
client = ArxivClient(request_fn=_request_fn)
|
||||
await client.lookup_ids(id_list=["1234.5678", " 9999.0001 "], start=0, max_results=2)
|
||||
assert captured["params"] == {"id_list": "1234.5678,9999.0001", "start": 0, "max_results": 2}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_search_rejects_invalid_sort_by() -> None:
|
||||
async def _unused_request_fn(*, params, request_email, timeout_seconds):
|
||||
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
|
||||
|
||||
client = ArxivClient(request_fn=_unused_request_fn)
|
||||
with pytest.raises(ArxivClientValidationError):
|
||||
await client.search(query="x", sort_by="bad")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_lookup_ids_rejects_empty_list() -> None:
|
||||
async def _unused_request_fn(*, params, request_email, timeout_seconds):
|
||||
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
|
||||
|
||||
client = ArxivClient(request_fn=_unused_request_fn)
|
||||
with pytest.raises(ArxivClientValidationError):
|
||||
await client.lookup_ids(id_list=[])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_search_rejects_negative_start() -> None:
|
||||
async def _unused_request_fn(*, params, request_email, timeout_seconds):
|
||||
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
|
||||
|
||||
client = ArxivClient(request_fn=_unused_request_fn)
|
||||
with pytest.raises(ArxivClientValidationError):
|
||||
await client.search(query="ti:test", start=-1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_propagates_http_status_error() -> None:
|
||||
async def _request_fn(*, params, request_email, timeout_seconds):
|
||||
request = httpx.Request("GET", "https://export.arxiv.org/api/query")
|
||||
return httpx.Response(500, text="error", request=request)
|
||||
|
||||
client = ArxivClient(request_fn=_request_fn)
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
await client.search(query="ti:test")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_coalesces_concurrent_identical_search_requests() -> None:
|
||||
calls = {"count": 0}
|
||||
|
||||
async def _request_fn(*, params, request_email, timeout_seconds):
|
||||
calls["count"] += 1
|
||||
await asyncio.sleep(0.05)
|
||||
request = httpx.Request("GET", "https://export.arxiv.org/api/query")
|
||||
return httpx.Response(200, text=_CLIENT_FEED_XML, request=request)
|
||||
|
||||
client = ArxivClient(request_fn=_request_fn)
|
||||
await asyncio.gather(
|
||||
client.search(query="ti:test"),
|
||||
client.search(query="ti:test"),
|
||||
)
|
||||
assert calls["count"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_logs_cache_hit_and_miss(
|
||||
db_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_ = db_session
|
||||
calls = {"count": 0}
|
||||
logged: list[dict[str, object]] = []
|
||||
|
||||
async def _request_fn(*, params, request_email, timeout_seconds):
|
||||
calls["count"] += 1
|
||||
request = httpx.Request("GET", "https://export.arxiv.org/api/query")
|
||||
return httpx.Response(200, text=_CLIENT_FEED_XML, request=request)
|
||||
|
||||
def _capture_log(_msg: str, *args, **kwargs) -> None:
|
||||
extra = kwargs.get("extra")
|
||||
if isinstance(extra, dict):
|
||||
logged.append(extra)
|
||||
|
||||
monkeypatch.setattr("app.services.domains.arxiv.client.logger.info", _capture_log)
|
||||
client = ArxivClient(request_fn=_request_fn, cache_enabled=True)
|
||||
await client.search(query="ti:test-cache")
|
||||
await client.search(query="ti:test-cache")
|
||||
|
||||
events = [str(entry.get("event", "")) for entry in logged]
|
||||
assert "arxiv.cache_miss" in events
|
||||
assert "arxiv.cache_hit" in events
|
||||
assert calls["count"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_feed_skips_live_call_when_global_cooldown_is_active(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
logged: list[dict[str, object]] = []
|
||||
called = {"count": 0}
|
||||
|
||||
async def _cooldown_status(*, now_utc=None):
|
||||
_ = now_utc
|
||||
return ArxivCooldownStatus(
|
||||
is_active=True,
|
||||
remaining_seconds=61.0,
|
||||
cooldown_until=datetime(2026, 2, 26, 12, 1, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
async def _unexpected_limit_call(*, fetch, source_path): # pragma: no cover - defensive
|
||||
_ = (fetch, source_path)
|
||||
called["count"] += 1
|
||||
return httpx.Response(200, text=_CLIENT_FEED_XML)
|
||||
|
||||
def _capture_warning(_msg: str, *args, **kwargs) -> None:
|
||||
extra = kwargs.get("extra")
|
||||
if isinstance(extra, dict):
|
||||
logged.append(extra)
|
||||
|
||||
monkeypatch.setattr(arxiv_client_module, "get_arxiv_cooldown_status", _cooldown_status)
|
||||
monkeypatch.setattr(arxiv_client_module, "run_with_global_arxiv_limit", _unexpected_limit_call)
|
||||
monkeypatch.setattr("app.services.domains.arxiv.client.logger.warning", _capture_warning)
|
||||
|
||||
with pytest.raises(ArxivRateLimitError):
|
||||
await arxiv_client_module._request_arxiv_feed(
|
||||
params={"search_query": 'ti:"test"'},
|
||||
request_email="user@example.com",
|
||||
timeout_seconds=2.0,
|
||||
)
|
||||
|
||||
assert called["count"] == 0
|
||||
assert [entry.get("event") for entry in logged] == ["arxiv.request_skipped_cooldown"]
|
||||
126
tests/unit/services/domains/arxiv/test_gateway.py
Normal file
126
tests/unit/services/domains/arxiv/test_gateway.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.domains.arxiv import application as arxiv_application
|
||||
from app.services.domains.arxiv import gateway as arxiv_gateway
|
||||
from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
||||
from app.settings import settings
|
||||
|
||||
|
||||
def _item(*, title: str = "A Test Paper", scholar_label: str = "Ada Lovelace") -> SimpleNamespace:
|
||||
return SimpleNamespace(title=title, scholar_label=scholar_label)
|
||||
|
||||
|
||||
def test_get_arxiv_gateway_returns_cached_instance() -> None:
|
||||
previous = arxiv_gateway.set_arxiv_gateway(None)
|
||||
try:
|
||||
first = arxiv_gateway.get_arxiv_gateway()
|
||||
second = arxiv_gateway.get_arxiv_gateway()
|
||||
assert first is second
|
||||
finally:
|
||||
arxiv_gateway.set_arxiv_gateway(previous)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_application_discover_uses_gateway_override() -> None:
|
||||
class FakeGateway:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[object, str | None, float | None]] = []
|
||||
|
||||
async def discover_arxiv_id_for_publication(
|
||||
self,
|
||||
*,
|
||||
item,
|
||||
request_email: str | None = None,
|
||||
timeout_seconds: float | None = None,
|
||||
max_results: int | None = None,
|
||||
) -> str | None:
|
||||
self.calls.append((item, request_email, timeout_seconds))
|
||||
return "1234.5678"
|
||||
|
||||
fake_gateway = FakeGateway()
|
||||
previous = arxiv_gateway.set_arxiv_gateway(fake_gateway)
|
||||
try:
|
||||
result = await arxiv_application.discover_arxiv_id_for_publication(
|
||||
item=_item(),
|
||||
request_email="user@example.com",
|
||||
timeout_seconds=7.0,
|
||||
)
|
||||
assert result == "1234.5678"
|
||||
assert fake_gateway.calls
|
||||
assert fake_gateway.calls[0][1] == "user@example.com"
|
||||
assert fake_gateway.calls[0][2] == 7.0
|
||||
finally:
|
||||
arxiv_gateway.set_arxiv_gateway(previous)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_gateway_returns_none_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
class FakeClient:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
async def search(self, **kwargs):
|
||||
self.calls += 1
|
||||
return ArxivFeed()
|
||||
|
||||
previous_enabled = bool(settings.arxiv_enabled)
|
||||
fake_client = FakeClient()
|
||||
object.__setattr__(settings, "arxiv_enabled", False)
|
||||
try:
|
||||
gateway = arxiv_gateway.HttpArxivGateway(client=fake_client)
|
||||
result = await gateway.discover_arxiv_id_for_publication(item=_item())
|
||||
assert result is None
|
||||
assert fake_client.calls == 0
|
||||
finally:
|
||||
object.__setattr__(settings, "arxiv_enabled", previous_enabled)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_gateway_uses_client_search_for_discovery() -> None:
|
||||
class FakeClient:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[dict] = []
|
||||
|
||||
async def search(self, **kwargs):
|
||||
self.calls.append(kwargs)
|
||||
return ArxivFeed(
|
||||
entries=[
|
||||
ArxivEntry(
|
||||
entry_id_url="https://arxiv.org/abs/1234.5678v1",
|
||||
arxiv_id="1234.5678v1",
|
||||
title="Paper",
|
||||
summary="",
|
||||
published=None,
|
||||
updated=None,
|
||||
)
|
||||
],
|
||||
opensearch=ArxivOpenSearchMeta(total_results=1, start_index=0, items_per_page=1),
|
||||
)
|
||||
|
||||
fake_client = FakeClient()
|
||||
gateway = arxiv_gateway.HttpArxivGateway(client=fake_client)
|
||||
result = await gateway.discover_arxiv_id_for_publication(
|
||||
item=_item(title="My Paper", scholar_label="Ada Lovelace"),
|
||||
request_email="user@example.com",
|
||||
timeout_seconds=2.0,
|
||||
max_results=4,
|
||||
)
|
||||
|
||||
assert result == "1234.5678v1"
|
||||
assert fake_client.calls
|
||||
first_call = fake_client.calls[0]
|
||||
assert first_call["query"] == 'ti:"My Paper" AND au:"lovelace"'
|
||||
assert first_call["request_email"] == "user@example.com"
|
||||
assert first_call["timeout_seconds"] == 2.0
|
||||
assert first_call["max_results"] == 4
|
||||
|
||||
|
||||
def test_build_arxiv_query_normalizes_noisy_mojibake_title() -> None:
|
||||
noisy = " Graph–Neural Networks Survey "
|
||||
clean = "Graph Neural Networks Survey"
|
||||
|
||||
assert arxiv_gateway.build_arxiv_query(noisy, None) == arxiv_gateway.build_arxiv_query(clean, None)
|
||||
63
tests/unit/services/domains/arxiv/test_guards.py
Normal file
63
tests/unit/services/domains/arxiv/test_guards.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
|
||||
from app.services.domains.publication_identifiers.types import DisplayIdentifier
|
||||
from app.services.domains.publications.types import PublicationListItem
|
||||
|
||||
|
||||
def _item(
|
||||
*,
|
||||
title: str,
|
||||
pub_url: str | None = None,
|
||||
pdf_url: str | None = None,
|
||||
display_identifier: DisplayIdentifier | None = None,
|
||||
) -> PublicationListItem:
|
||||
return PublicationListItem(
|
||||
publication_id=1,
|
||||
scholar_profile_id=1,
|
||||
scholar_label="Ada Lovelace",
|
||||
title=title,
|
||||
year=2024,
|
||||
citation_count=0,
|
||||
venue_text=None,
|
||||
pub_url=pub_url,
|
||||
pdf_url=pdf_url,
|
||||
is_read=False,
|
||||
first_seen_at=datetime.now(timezone.utc),
|
||||
is_new_in_latest_run=True,
|
||||
display_identifier=display_identifier,
|
||||
)
|
||||
|
||||
|
||||
def test_arxiv_skip_reason_for_strong_doi_evidence() -> None:
|
||||
item = _item(
|
||||
title="A Robust and Reproducible Deep Learning Benchmark",
|
||||
display_identifier=DisplayIdentifier(
|
||||
kind="doi",
|
||||
value="10.1000/example",
|
||||
label="DOI: 10.1000/example",
|
||||
url="https://doi.org/10.1000/example",
|
||||
confidence_score=1.0,
|
||||
),
|
||||
)
|
||||
assert arxiv_skip_reason_for_item(item=item) == "strong_doi_present"
|
||||
|
||||
|
||||
def test_arxiv_skip_reason_for_existing_arxiv_link() -> None:
|
||||
item = _item(
|
||||
title="A Robust and Reproducible Deep Learning Benchmark",
|
||||
pub_url="https://arxiv.org/abs/2501.00001",
|
||||
)
|
||||
assert arxiv_skip_reason_for_item(item=item) == "arxiv_identifier_present"
|
||||
|
||||
|
||||
def test_arxiv_skip_reason_for_low_quality_title() -> None:
|
||||
item = _item(title="AI 2024")
|
||||
assert arxiv_skip_reason_for_item(item=item) == "title_quality_below_threshold"
|
||||
|
||||
|
||||
def test_arxiv_skip_reason_none_for_eligible_item() -> None:
|
||||
item = _item(title="Reliable Graph Neural Network Benchmarking Across Multiple Datasets")
|
||||
assert arxiv_skip_reason_for_item(item=item) is None
|
||||
55
tests/unit/services/domains/arxiv/test_parser.py
Normal file
55
tests/unit/services/domains/arxiv/test_parser.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.domains.arxiv.errors import ArxivParseError
|
||||
from app.services.domains.arxiv.parser import parse_arxiv_feed
|
||||
|
||||
_VALID_FEED_XML = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom"
|
||||
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
|
||||
xmlns:arxiv="http://arxiv.org/schemas/atom">
|
||||
<opensearch:totalResults>2</opensearch:totalResults>
|
||||
<opensearch:startIndex>0</opensearch:startIndex>
|
||||
<opensearch:itemsPerPage>2</opensearch:itemsPerPage>
|
||||
<entry>
|
||||
<id>http://arxiv.org/abs/1234.5678v2</id>
|
||||
<updated>2020-01-01T00:00:00Z</updated>
|
||||
<published>2019-12-31T00:00:00Z</published>
|
||||
<title>Test Entry</title>
|
||||
<summary>Example summary</summary>
|
||||
<author><name>Ada Lovelace</name></author>
|
||||
<author><name>Grace Hopper</name></author>
|
||||
<link href="http://arxiv.org/abs/1234.5678v2" />
|
||||
<category term="cs.AI" />
|
||||
<category term="stat.ML" />
|
||||
<arxiv:primary_category term="cs.AI" />
|
||||
</entry>
|
||||
</feed>
|
||||
"""
|
||||
|
||||
|
||||
def test_parse_arxiv_feed_extracts_entries_and_opensearch_meta() -> None:
|
||||
feed = parse_arxiv_feed(_VALID_FEED_XML)
|
||||
assert feed.opensearch.total_results == 2
|
||||
assert feed.opensearch.start_index == 0
|
||||
assert feed.opensearch.items_per_page == 2
|
||||
assert len(feed.entries) == 1
|
||||
entry = feed.entries[0]
|
||||
assert entry.arxiv_id == "1234.5678v2"
|
||||
assert entry.title == "Test Entry"
|
||||
assert entry.summary == "Example summary"
|
||||
assert entry.authors == ["Ada Lovelace", "Grace Hopper"]
|
||||
assert entry.primary_category == "cs.AI"
|
||||
assert entry.categories == ["cs.AI", "stat.ML"]
|
||||
|
||||
|
||||
def test_parse_arxiv_feed_raises_on_invalid_xml() -> None:
|
||||
with pytest.raises(ArxivParseError):
|
||||
parse_arxiv_feed("<feed><entry></feed>")
|
||||
|
||||
|
||||
def test_parse_arxiv_feed_raises_on_invalid_opensearch_integer() -> None:
|
||||
payload = _VALID_FEED_XML.replace("<opensearch:totalResults>2</opensearch:totalResults>", "<opensearch:totalResults>x</opensearch:totalResults>")
|
||||
with pytest.raises(ArxivParseError):
|
||||
parse_arxiv_feed(payload)
|
||||
167
tests/unit/services/domains/arxiv/test_rate_limit.py
Normal file
167
tests/unit/services/domains/arxiv/test_rate_limit.py
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import ArxivRuntimeState
|
||||
from app.services.domains.arxiv.constants import ARXIV_RUNTIME_STATE_KEY
|
||||
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
||||
from app.services.domains.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
|
||||
from app.settings import settings
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arxiv_rate_limit_respects_cooldown(db_session: AsyncSession) -> None:
|
||||
db_session.add(
|
||||
ArxivRuntimeState(
|
||||
state_key=ARXIV_RUNTIME_STATE_KEY,
|
||||
cooldown_until=datetime.now(timezone.utc) + timedelta(seconds=30),
|
||||
)
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
called = {"count": 0}
|
||||
|
||||
async def _fetch() -> httpx.Response:
|
||||
called["count"] += 1
|
||||
return httpx.Response(200, text="ok")
|
||||
|
||||
with pytest.raises(ArxivRateLimitError):
|
||||
await run_with_global_arxiv_limit(fetch=_fetch)
|
||||
assert called["count"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arxiv_rate_limit_persists_cooldown_after_429(db_session: AsyncSession) -> None:
|
||||
previous_interval = settings.arxiv_min_interval_seconds
|
||||
previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
|
||||
object.__setattr__(settings, "arxiv_min_interval_seconds", 0.0)
|
||||
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", 5.0)
|
||||
try:
|
||||
async def _fetch() -> httpx.Response:
|
||||
return httpx.Response(429, text="rate limited")
|
||||
|
||||
with pytest.raises(ArxivRateLimitError):
|
||||
await run_with_global_arxiv_limit(fetch=_fetch)
|
||||
finally:
|
||||
object.__setattr__(settings, "arxiv_min_interval_seconds", previous_interval)
|
||||
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", previous_cooldown)
|
||||
|
||||
result = await db_session.execute(
|
||||
select(ArxivRuntimeState).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
|
||||
)
|
||||
state = result.scalar_one()
|
||||
assert state.cooldown_until is not None
|
||||
assert state.cooldown_until > datetime.now(timezone.utc)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arxiv_rate_limit_serializes_concurrent_calls(db_session: AsyncSession) -> None:
|
||||
previous_interval = settings.arxiv_min_interval_seconds
|
||||
previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
|
||||
object.__setattr__(settings, "arxiv_min_interval_seconds", 0.2)
|
||||
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", 5.0)
|
||||
try:
|
||||
call_times: list[float] = []
|
||||
|
||||
async def _fetch() -> httpx.Response:
|
||||
call_times.append(asyncio.get_running_loop().time())
|
||||
return httpx.Response(200, text="ok")
|
||||
|
||||
await asyncio.gather(
|
||||
run_with_global_arxiv_limit(fetch=_fetch),
|
||||
run_with_global_arxiv_limit(fetch=_fetch),
|
||||
)
|
||||
finally:
|
||||
object.__setattr__(settings, "arxiv_min_interval_seconds", previous_interval)
|
||||
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", previous_cooldown)
|
||||
|
||||
assert len(call_times) == 2
|
||||
assert call_times[1] - call_times[0] >= 0.18
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arxiv_rate_limit_logs_request_scheduled_and_completed(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
logged: list[dict[str, object]] = []
|
||||
|
||||
async def _fetch() -> httpx.Response:
|
||||
return httpx.Response(200, text="ok")
|
||||
|
||||
def _capture_log(_msg: str, *args, **kwargs) -> None:
|
||||
extra = kwargs.get("extra")
|
||||
if isinstance(extra, dict):
|
||||
logged.append(extra)
|
||||
|
||||
monkeypatch.setattr("app.services.domains.arxiv.rate_limit.logger.info", _capture_log)
|
||||
await run_with_global_arxiv_limit(fetch=_fetch, source_path="search")
|
||||
|
||||
scheduled = [entry for entry in logged if entry.get("event") == "arxiv.request_scheduled"]
|
||||
completed = [entry for entry in logged if entry.get("event") == "arxiv.request_completed"]
|
||||
|
||||
assert scheduled
|
||||
assert completed
|
||||
assert float(scheduled[0]["wait_seconds"]) >= 0.0
|
||||
assert int(completed[0]["status_code"]) == 200
|
||||
assert completed[0]["source_path"] == "search"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arxiv_rate_limit_logs_cooldown_activation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
logged_warning: list[dict[str, object]] = []
|
||||
previous_interval = settings.arxiv_min_interval_seconds
|
||||
previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
|
||||
object.__setattr__(settings, "arxiv_min_interval_seconds", 0.0)
|
||||
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", 5.0)
|
||||
try:
|
||||
async def _fetch() -> httpx.Response:
|
||||
return httpx.Response(429, text="rate limited")
|
||||
|
||||
def _capture_warning(_msg: str, *args, **kwargs) -> None:
|
||||
extra = kwargs.get("extra")
|
||||
if isinstance(extra, dict):
|
||||
logged_warning.append(extra)
|
||||
|
||||
monkeypatch.setattr("app.services.domains.arxiv.rate_limit.logger.warning", _capture_warning)
|
||||
with pytest.raises(ArxivRateLimitError):
|
||||
await run_with_global_arxiv_limit(fetch=_fetch, source_path="lookup_ids")
|
||||
finally:
|
||||
object.__setattr__(settings, "arxiv_min_interval_seconds", previous_interval)
|
||||
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", previous_cooldown)
|
||||
|
||||
cooldown_events = [
|
||||
entry for entry in logged_warning if entry.get("event") == "arxiv.cooldown_activated"
|
||||
]
|
||||
assert cooldown_events
|
||||
assert cooldown_events[0]["source_path"] == "lookup_ids"
|
||||
assert float(cooldown_events[0]["cooldown_remaining_seconds"]) > 0.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_arxiv_cooldown_status_reads_active_cooldown(db_session: AsyncSession) -> None:
|
||||
now_utc = datetime(2026, 2, 26, 13, 0, tzinfo=timezone.utc)
|
||||
existing = await db_session.get(ArxivRuntimeState, ARXIV_RUNTIME_STATE_KEY)
|
||||
if existing is None:
|
||||
db_session.add(
|
||||
ArxivRuntimeState(
|
||||
state_key=ARXIV_RUNTIME_STATE_KEY,
|
||||
cooldown_until=now_utc + timedelta(seconds=45),
|
||||
)
|
||||
)
|
||||
else:
|
||||
existing.cooldown_until = now_utc + timedelta(seconds=45)
|
||||
await db_session.commit()
|
||||
|
||||
status = await get_arxiv_cooldown_status(now_utc=now_utc)
|
||||
|
||||
assert status.is_active is True
|
||||
assert status.cooldown_until is not None
|
||||
assert int(status.remaining_seconds) == 45
|
||||
Loading…
Add table
Add a link
Reference in a new issue