Big changes
This commit is contained in:
parent
4240ad38e2
commit
e3f0d63fec
99 changed files with 8804 additions and 1731 deletions
109
tests/unit/test_publication_pdf_queue_policy.py
Normal file
109
tests/unit/test_publication_pdf_queue_policy.py
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.db.models import PublicationPdfJob
|
||||
from app.services.domains.publications import pdf_queue
|
||||
|
||||
|
||||
def _job(
|
||||
*,
|
||||
status: str,
|
||||
attempt_count: int,
|
||||
last_attempt_at: datetime | None,
|
||||
) -> PublicationPdfJob:
|
||||
return PublicationPdfJob(
|
||||
publication_id=1,
|
||||
status=status,
|
||||
attempt_count=attempt_count,
|
||||
last_attempt_at=last_attempt_at,
|
||||
)
|
||||
|
||||
|
||||
def test_pdf_queue_auto_enqueue_blocks_recent_attempt(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
|
||||
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
|
||||
job = _job(
|
||||
status=pdf_queue.PDF_STATUS_FAILED,
|
||||
attempt_count=1,
|
||||
last_attempt_at=now - timedelta(hours=2),
|
||||
)
|
||||
assert pdf_queue._can_enqueue_job(job, force_retry=False) is True
|
||||
|
||||
|
||||
def test_pdf_queue_auto_enqueue_blocks_recent_first_retry(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
|
||||
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
|
||||
job = _job(
|
||||
status=pdf_queue.PDF_STATUS_FAILED,
|
||||
attempt_count=1,
|
||||
last_attempt_at=now - timedelta(minutes=20),
|
||||
)
|
||||
assert pdf_queue._can_enqueue_job(job, force_retry=False) is False
|
||||
|
||||
|
||||
def test_pdf_queue_auto_enqueue_blocks_after_max_attempts(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
|
||||
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
|
||||
job = _job(
|
||||
status=pdf_queue.PDF_STATUS_FAILED,
|
||||
attempt_count=3,
|
||||
last_attempt_at=now - timedelta(days=2),
|
||||
)
|
||||
assert pdf_queue._can_enqueue_job(job, force_retry=False) is False
|
||||
|
||||
|
||||
def test_pdf_queue_auto_enqueue_blocks_second_retry_within_day(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
|
||||
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
|
||||
job = _job(
|
||||
status=pdf_queue.PDF_STATUS_FAILED,
|
||||
attempt_count=2,
|
||||
last_attempt_at=now - timedelta(hours=2),
|
||||
)
|
||||
assert pdf_queue._can_enqueue_job(job, force_retry=False) is False
|
||||
|
||||
|
||||
def test_pdf_queue_manual_requeue_bypasses_cooldown_and_max_attempts(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
|
||||
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
|
||||
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
|
||||
job = _job(
|
||||
status=pdf_queue.PDF_STATUS_FAILED,
|
||||
attempt_count=5,
|
||||
last_attempt_at=now - timedelta(minutes=10),
|
||||
)
|
||||
assert pdf_queue._can_enqueue_job(job, force_retry=True) is True
|
||||
|
||||
|
||||
def test_pdf_queue_manual_requeue_still_blocks_when_inflight() -> None:
|
||||
running = _job(
|
||||
status=pdf_queue.PDF_STATUS_RUNNING,
|
||||
attempt_count=1,
|
||||
last_attempt_at=None,
|
||||
)
|
||||
queued = _job(
|
||||
status=pdf_queue.PDF_STATUS_QUEUED,
|
||||
attempt_count=1,
|
||||
last_attempt_at=None,
|
||||
)
|
||||
assert pdf_queue._can_enqueue_job(running, force_retry=True) is False
|
||||
assert pdf_queue._can_enqueue_job(queued, force_retry=True) is False
|
||||
45
tests/unit/test_request_delay_policy.py
Normal file
45
tests/unit/test_request_delay_policy.py
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.services.domains.ingestion.application import ScholarIngestionService
|
||||
from app.services.domains.ingestion import scheduler as scheduler_module
|
||||
from app.settings import settings
|
||||
|
||||
|
||||
def _set_policy_minimum(value: int) -> int:
|
||||
previous = int(settings.ingestion_min_request_delay_seconds)
|
||||
object.__setattr__(settings, "ingestion_min_request_delay_seconds", int(value))
|
||||
return previous
|
||||
|
||||
|
||||
def test_ingestion_effective_request_delay_respects_policy_minimum() -> None:
|
||||
previous = _set_policy_minimum(8)
|
||||
try:
|
||||
assert ScholarIngestionService._effective_request_delay_seconds(7) == 8
|
||||
assert ScholarIngestionService._effective_request_delay_seconds(12) == 12
|
||||
finally:
|
||||
object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous)
|
||||
|
||||
|
||||
def test_scheduler_effective_request_delay_respects_policy_minimum() -> None:
|
||||
previous = _set_policy_minimum(9)
|
||||
try:
|
||||
assert scheduler_module._effective_request_delay_seconds(None) == 9
|
||||
assert scheduler_module._effective_request_delay_seconds(6) == 9
|
||||
assert scheduler_module._effective_request_delay_seconds(11) == 11
|
||||
finally:
|
||||
object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous)
|
||||
|
||||
|
||||
def test_scheduler_candidate_row_clamps_request_delay() -> None:
|
||||
previous = _set_policy_minimum(6)
|
||||
try:
|
||||
candidate = scheduler_module.SchedulerService._candidate_from_row(
|
||||
(1, 15, 1, None, None),
|
||||
now_utc=datetime(2026, 2, 21, tzinfo=timezone.utc),
|
||||
)
|
||||
assert candidate is not None
|
||||
assert candidate.request_delay_seconds == 6
|
||||
finally:
|
||||
object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous)
|
||||
94
tests/unit/test_unpaywall_pdf_discovery.py
Normal file
94
tests/unit/test_unpaywall_pdf_discovery.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.domains.unpaywall import pdf_discovery
|
||||
|
||||
|
||||
def test_looks_like_pdf_url_detects_path_and_query_variants() -> None:
|
||||
assert pdf_discovery.looks_like_pdf_url("https://example.org/file.pdf")
|
||||
assert pdf_discovery.looks_like_pdf_url("https://example.org/download?target=paper.pdf")
|
||||
assert not pdf_discovery.looks_like_pdf_url("https://example.org/landing")
|
||||
assert not pdf_discovery.looks_like_pdf_url(None)
|
||||
|
||||
|
||||
def test_normalized_candidate_urls_extracts_and_resolves_relative_links() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<head>
|
||||
<base href="https://publisher.example.org/articles/42/" />
|
||||
<meta name="citation_pdf_url" content="/pdfs/42-main.pdf" />
|
||||
<link rel="alternate" type="application/pdf" href="https://cdn.example.org/42-supp.pdf" />
|
||||
</head>
|
||||
<body>
|
||||
<a href="appendix.pdf">Appendix</a>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
candidates = pdf_discovery._normalized_candidate_urls(
|
||||
page_url="https://publisher.example.org/articles/42",
|
||||
html=html,
|
||||
)
|
||||
assert len(candidates) == 3
|
||||
assert "https://cdn.example.org/42-supp.pdf" in candidates
|
||||
assert "https://publisher.example.org/pdfs/42-main.pdf" in candidates
|
||||
assert "https://publisher.example.org/articles/42/appendix.pdf" in candidates
|
||||
|
||||
|
||||
def test_normalized_candidate_urls_extracts_plain_text_urls() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<body>
|
||||
Footnote URL: http://www.mext.go.jp/component/english/file.pdf
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
candidates = pdf_discovery._normalized_candidate_urls(
|
||||
page_url="https://www.science.org/doi/10.1126/science.1239057",
|
||||
html=html,
|
||||
)
|
||||
assert "http://www.mext.go.jp/component/english/file.pdf" in candidates
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, *, status_code: int, content_type: str, text: str) -> None:
|
||||
self.status_code = status_code
|
||||
self.headers = {"content-type": content_type}
|
||||
self.text = text
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, pages: dict[str, _FakeResponse]) -> None:
|
||||
self._pages = pages
|
||||
|
||||
async def get(self, url: str, follow_redirects: bool = True): # noqa: ARG002
|
||||
return self._pages[url]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_pdf_from_landing_page_follows_one_hop_html_candidate(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
async def _skip_wait(*args, **kwargs): # noqa: ARG001, ANN002
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(pdf_discovery, "wait_for_unpaywall_slot", _skip_wait)
|
||||
landing_url = "https://example.org/article"
|
||||
hop_url = "https://example.org/doi/full/abc"
|
||||
pdf_url = "https://downloads.example.org/archive/paper.pdf"
|
||||
client = _FakeClient(
|
||||
{
|
||||
landing_url: _FakeResponse(
|
||||
status_code=200,
|
||||
content_type="text/html",
|
||||
text=f"<html><body><a href=\"{hop_url}\">View article</a></body></html>",
|
||||
),
|
||||
hop_url: _FakeResponse(
|
||||
status_code=200,
|
||||
content_type="text/html",
|
||||
text=f"<html><body>Download here: {pdf_url}</body></html>",
|
||||
),
|
||||
}
|
||||
)
|
||||
resolved = await pdf_discovery.resolve_pdf_from_landing_page(client, page_url=landing_url)
|
||||
assert resolved == pdf_url
|
||||
89
tests/unit/test_unpaywall_resolution.py
Normal file
89
tests/unit/test_unpaywall_resolution.py
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.domains.publications.types import PublicationListItem
|
||||
from app.services.domains.unpaywall import application as unpaywall_app
|
||||
|
||||
|
||||
class _DummyAsyncClient:
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
|
||||
def _item(publication_id: int) -> PublicationListItem:
|
||||
return PublicationListItem(
|
||||
publication_id=publication_id,
|
||||
scholar_profile_id=1,
|
||||
scholar_label="Shinya Yamanaka",
|
||||
title="Induction of pluripotent stem cells",
|
||||
year=2007,
|
||||
citation_count=1000,
|
||||
venue_text="Cell",
|
||||
pub_url="https://doi.org/10.1016/j.cell.2007.11.019",
|
||||
doi=None,
|
||||
pdf_url=None,
|
||||
is_read=False,
|
||||
first_seen_at=datetime.now(timezone.utc),
|
||||
is_new_in_latest_run=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unpaywall_resolve_prefers_direct_pdf_without_landing_crawl(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
payload = {
|
||||
"doi": "10.1016/j.cell.2007.11.019",
|
||||
"best_oa_location": {
|
||||
"url_for_pdf": "https://oa.example.org/article.pdf",
|
||||
"url": "https://oa.example.org/landing",
|
||||
},
|
||||
}
|
||||
|
||||
async def _fake_resolve_item_payload(**_kwargs):
|
||||
return payload, False
|
||||
|
||||
async def _fail_crawl(_client, *, page_url: str):
|
||||
raise AssertionError(f"unexpected landing crawl: {page_url}")
|
||||
|
||||
monkeypatch.setattr(unpaywall_app, "_resolve_item_payload", _fake_resolve_item_payload)
|
||||
monkeypatch.setattr(unpaywall_app, "resolve_pdf_from_landing_page", _fail_crawl)
|
||||
monkeypatch.setattr("httpx.AsyncClient", _DummyAsyncClient)
|
||||
resolved = await unpaywall_app.resolve_publication_oa_metadata([_item(1)], request_email="user@example.com")
|
||||
assert resolved == {1: ("10.1016/j.cell.2007.11.019", "https://oa.example.org/article.pdf")}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unpaywall_resolve_crawls_landing_page_when_pdf_url_is_not_direct(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
payload = {
|
||||
"doi": "10.1016/j.cell.2007.11.019",
|
||||
"best_oa_location": {
|
||||
"url_for_pdf": "https://oa.example.org/view?paper=42",
|
||||
"url": "https://oa.example.org/landing/42",
|
||||
},
|
||||
}
|
||||
crawled_pages: list[str] = []
|
||||
|
||||
async def _fake_resolve_item_payload(**_kwargs):
|
||||
return payload, False
|
||||
|
||||
async def _fake_crawl(_client, *, page_url: str):
|
||||
crawled_pages.append(page_url)
|
||||
return "https://oa.example.org/files/paper-42.pdf"
|
||||
|
||||
monkeypatch.setattr(unpaywall_app, "_resolve_item_payload", _fake_resolve_item_payload)
|
||||
monkeypatch.setattr(unpaywall_app, "resolve_pdf_from_landing_page", _fake_crawl)
|
||||
monkeypatch.setattr("httpx.AsyncClient", _DummyAsyncClient)
|
||||
resolved = await unpaywall_app.resolve_publication_oa_metadata([_item(2)], request_email="user@example.com")
|
||||
assert resolved == {2: ("10.1016/j.cell.2007.11.019", "https://oa.example.org/files/paper-42.pdf")}
|
||||
assert "https://oa.example.org/landing/42" in crawled_pages
|
||||
Loading…
Add table
Add a link
Reference in a new issue