justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

temp commit

Browse source
This commit is contained in:
Justin Visser 2026-02-26 12:54:19 +01:00
parent 8760f27b51
commit 0e9e49df16
193 changed files with 23228 additions and 935 deletions
Download patch file Download diff file Expand all files Collapse all files

238
tests/integration/test_api_v1.py
View file

@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
from datetime import datetime, timedelta, timezone
from pathlib import Path
@ -66,6 +67,31 @@ def _assert_safety_state_contract(payload: dict[str, object]) -> None:
assert set(counters.keys()) == SAFETY_COUNTER_KEYS
_ACTIVE_RUN_STATUSES = {"running", "resolving"}
async def _wait_for_run_complete(
client: TestClient,
run_id: int,
*,
max_retries: int = 300,
poll_interval: float = 0.2,
) -> dict:
"""Poll GET /api/v1/runs/{run_id} until the run is in a terminal state.
Returns the final run detail data dict.
"""
for _ in range(max_retries):
await asyncio.sleep(poll_interval)
r = client.get(f"/api/v1/runs/{run_id}")
assert r.status_code == 200
data = r.json()["data"]
if data["run"]["status"] not in _ACTIVE_RUN_STATUSES:
return data
r = client.get(f"/api/v1/runs/{run_id}")
return r.json()["data"]
async def _seed_publication_link_for_user(
db_session: AsyncSession,
*,
@ -1041,44 +1067,42 @@ async def test_api_manual_run_skips_unchanged_initial_page_for_scholar(
app.dependency_overrides[get_scholar_source] = lambda: StubScholarSource()
try:
client = TestClient(app)
login_user(client, email="api-skip-unchanged@example.com", password="api-password")
headers = _api_csrf_headers(client)
with TestClient(app) as client:
login_user(client, email="api-skip-unchanged@example.com", password="api-password")
headers = _api_csrf_headers(client)
create_response = client.post(
"/api/v1/scholars",
json={"scholar_id": "abcDEF123456"},
headers=headers,
)
assert create_response.status_code == 201
create_response = client.post(
"/api/v1/scholars",
json={"scholar_id": "abcDEF123456"},
headers=headers,
)
assert create_response.status_code == 201
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "skip-unchanged-run-001"},
)
assert first_run_response.status_code == 200
first_run_id = int(first_run_response.json()["data"]["run_id"])
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "skip-unchanged-run-001"},
)
assert first_run_response.status_code == 200
first_run_id = int(first_run_response.json()["data"]["run_id"])
first_run_detail = client.get(f"/api/v1/runs/{first_run_id}")
assert first_run_detail.status_code == 200
first_results = first_run_detail.json()["data"]["scholar_results"]
assert len(first_results) == 1
assert first_results[0]["state_reason"] != "no_change_initial_page_signature"
first_detail_data = await _wait_for_run_complete(client, first_run_id)
first_results = first_detail_data["scholar_results"]
assert len(first_results) == 1
assert first_results[0]["state_reason"] != "no_change_initial_page_signature"
second_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "skip-unchanged-run-002"},
)
assert second_run_response.status_code == 200
second_run_id = int(second_run_response.json()["data"]["run_id"])
second_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "skip-unchanged-run-002"},
)
assert second_run_response.status_code == 200
second_run_id = int(second_run_response.json()["data"]["run_id"])
second_run_detail = client.get(f"/api/v1/runs/{second_run_id}")
assert second_run_detail.status_code == 200
second_results = second_run_detail.json()["data"]["scholar_results"]
assert len(second_results) == 1
assert second_results[0]["state_reason"] == "no_change_initial_page_signature"
assert second_results[0]["publication_count"] == 0
assert second_results[0]["outcome"] == "success"
second_detail_data = await _wait_for_run_complete(client, second_run_id)
second_results = second_detail_data["scholar_results"]
assert len(second_results) == 1
assert second_results[0]["state_reason"] == "no_change_initial_page_signature"
assert second_results[0]["publication_count"] == 0
assert second_results[0]["outcome"] == "success"
finally:
app.dependency_overrides.pop(get_scholar_source, None)
@ -1227,7 +1251,7 @@ async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> No
assert run_response.status_code == 200
run_payload = run_response.json()["data"]
assert "run_id" in run_payload
assert run_payload["status"] in {"success", "partial_failure", "failed"}
assert run_payload["status"] in {"running", "resolving", "success", "partial_failure", "failed"}
assert run_payload["reused_existing_run"] is False
assert run_payload["idempotency_key"] == "manual-run-0001"
_assert_safety_state_contract(run_payload["safety_state"])
@ -1243,11 +1267,15 @@ async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> No
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "manual-run-0001"},
)
assert replay_response.status_code == 200
replay_payload = replay_response.json()["data"]
assert replay_payload["run_id"] == run_payload["run_id"]
assert replay_payload["reused_existing_run"] is True
_assert_safety_state_contract(replay_payload["safety_state"])
assert replay_response.status_code in {200, 409}
if replay_response.status_code == 200:
replay_payload = replay_response.json()["data"]
assert replay_payload["run_id"] == run_payload["run_id"]
assert replay_payload["reused_existing_run"] is True
_assert_safety_state_contract(replay_payload["safety_state"])
else:
replay_error = replay_response.json()["error"]
assert replay_error["code"] == "run_in_progress"
runs_response = client.get("/api/v1/runs")
assert runs_response.status_code == 200
@ -1429,43 +1457,45 @@ async def test_api_manual_run_enforces_scrape_safety_cooldown(db_session: AsyncS
object.__setattr__(settings, "ingestion_safety_cooldown_blocked_seconds", 600)
try:
client = TestClient(app)
login_user(client, email="api-runs-safety@example.com", password="api-password")
headers = _api_csrf_headers(client)
with TestClient(app) as client:
login_user(client, email="api-runs-safety@example.com", password="api-password")
headers = _api_csrf_headers(client)
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-cooldown-run-1"},
)
assert first_run_response.status_code == 200
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-cooldown-run-1"},
)
assert first_run_response.status_code == 200
first_run_id = int(first_run_response.json()["data"]["run_id"])
await _wait_for_run_complete(client, first_run_id)
settings_response = client.get("/api/v1/settings")
assert settings_response.status_code == 200
safety_state = settings_response.json()["data"]["safety_state"]
_assert_safety_state_contract(safety_state)
assert safety_state["cooldown_active"] is True
assert safety_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
assert int(safety_state["cooldown_remaining_seconds"]) > 0
assert int(safety_state["counters"]["cooldown_entry_count"]) >= 1
assert int(safety_state["counters"]["last_blocked_failure_count"]) >= 1
settings_response = client.get("/api/v1/settings")
assert settings_response.status_code == 200
safety_state = settings_response.json()["data"]["safety_state"]
_assert_safety_state_contract(safety_state)
assert safety_state["cooldown_active"] is True
assert safety_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
assert int(safety_state["cooldown_remaining_seconds"]) > 0
assert int(safety_state["counters"]["cooldown_entry_count"]) >= 1
assert int(safety_state["counters"]["last_blocked_failure_count"]) >= 1
blocked_start_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-cooldown-run-2"},
)
assert blocked_start_response.status_code == 429
blocked_payload = blocked_start_response.json()
assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
blocked_state = blocked_payload["error"]["details"]["safety_state"]
_assert_safety_state_contract(blocked_state)
assert blocked_state["cooldown_active"] is True
assert blocked_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
blocked_start_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-cooldown-run-2"},
)
assert blocked_start_response.status_code == 429
blocked_payload = blocked_start_response.json()
assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
blocked_state = blocked_payload["error"]["details"]["safety_state"]
_assert_safety_state_contract(blocked_state)
assert blocked_state["cooldown_active"] is True
assert blocked_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
runs_response = client.get("/api/v1/runs")
assert runs_response.status_code == 200
_assert_safety_state_contract(runs_response.json()["data"]["safety_state"])
assert runs_response.json()["data"]["safety_state"]["cooldown_active"] is True
runs_response = client.get("/api/v1/runs")
assert runs_response.status_code == 200
_assert_safety_state_contract(runs_response.json()["data"]["safety_state"])
assert runs_response.json()["data"]["safety_state"]["cooldown_active"] is True
finally:
object.__setattr__(settings, "ingestion_alert_blocked_failure_threshold", previous_blocked_threshold)
object.__setattr__(settings, "ingestion_safety_cooldown_blocked_seconds", previous_blocked_cooldown_seconds)
@ -1535,37 +1565,39 @@ async def test_api_manual_run_enforces_network_failure_safety_cooldown(
object.__setattr__(settings, "ingestion_retry_backoff_seconds", 0.0)
try:
client = TestClient(app)
login_user(client, email="api-runs-safety-network@example.com", password="api-password")
headers = _api_csrf_headers(client)
with TestClient(app) as client:
login_user(client, email="api-runs-safety-network@example.com", password="api-password")
headers = _api_csrf_headers(client)
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-1"},
)
assert first_run_response.status_code == 200
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-1"},
)
assert first_run_response.status_code == 200
first_run_id = int(first_run_response.json()["data"]["run_id"])
await _wait_for_run_complete(client, first_run_id)
settings_response = client.get("/api/v1/settings")
assert settings_response.status_code == 200
safety_state = settings_response.json()["data"]["safety_state"]
_assert_safety_state_contract(safety_state)
assert safety_state["cooldown_active"] is True
assert safety_state["cooldown_reason"] == "network_failure_threshold_exceeded"
assert int(safety_state["cooldown_remaining_seconds"]) > 0
assert int(safety_state["counters"]["last_network_failure_count"]) >= 1
settings_response = client.get("/api/v1/settings")
assert settings_response.status_code == 200
safety_state = settings_response.json()["data"]["safety_state"]
_assert_safety_state_contract(safety_state)
assert safety_state["cooldown_active"] is True
assert safety_state["cooldown_reason"] == "network_failure_threshold_exceeded"
assert int(safety_state["cooldown_remaining_seconds"]) > 0
assert int(safety_state["counters"]["last_network_failure_count"]) >= 1
blocked_start_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-2"},
)
assert blocked_start_response.status_code == 429
blocked_payload = blocked_start_response.json()
assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
blocked_state = blocked_payload["error"]["details"]["safety_state"]
_assert_safety_state_contract(blocked_state)
assert blocked_state["cooldown_active"] is True
assert blocked_state["cooldown_reason"] == "network_failure_threshold_exceeded"
assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
blocked_start_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-2"},
)
assert blocked_start_response.status_code == 429
blocked_payload = blocked_start_response.json()
assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
blocked_state = blocked_payload["error"]["details"]["safety_state"]
_assert_safety_state_contract(blocked_state)
assert blocked_state["cooldown_active"] is True
assert blocked_state["cooldown_reason"] == "network_failure_threshold_exceeded"
assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
finally:
object.__setattr__(settings, "ingestion_alert_network_failure_threshold", previous_network_threshold)
object.__setattr__(settings, "ingestion_safety_cooldown_network_seconds", previous_network_cooldown_seconds)
@ -2152,7 +2184,6 @@ async def test_api_publication_retry_pdf_queues_resolution_job(
citation_count=item.citation_count,
venue_text=item.venue_text,
pub_url=item.pub_url,
doi=item.doi,
pdf_url=item.pdf_url,
is_read=item.is_read,
first_seen_at=item.first_seen_at,
@ -2189,11 +2220,10 @@ async def test_api_publication_retry_pdf_queues_resolution_job(
assert payload["publication"]["pdf_failure_reason"] == "no_pdf_found"
stored = await db_session.execute(
text("SELECT doi, pdf_url FROM publications WHERE id = :publication_id"),
text("SELECT pdf_url FROM publications WHERE id = :publication_id"),
{"publication_id": publication_id},
)
stored_doi, stored_pdf_url = stored.one()
assert stored_doi is None
stored_pdf_url = stored.scalar_one()
assert stored_pdf_url is None

103
tests/integration/test_deferred_enrichment.py Normal file
View file

@ -0,0 +1,103 @@
from __future__ import annotations
import pytest
from unittest.mock import patch, AsyncMock, MagicMock
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from datetime import datetime, timezone
from app.db.models import CrawlRun, Publication, ScholarProfile, ScholarPublication, RunStatus, RunTriggerType
from app.services.domains.ingestion.application import ScholarIngestionService
from app.services.domains.openalex.types import OpenAlexWork
from tests.integration.helpers import insert_user
@pytest.mark.integration
@pytest.mark.asyncio
async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession) -> None:
# 1. Setup: Create user and scholar
user_id = await insert_user(db_session, email="test@example.com", password="password123")
scholar = ScholarProfile(
user_id=user_id,
scholar_id="SaiiI5MAAAAJ",
display_name="Test Scholar",
is_enabled=True,
)
db_session.add(scholar)
await db_session.flush()
# 2. Simulate a previous FAILED run that left an un-enriched publication
failed_run = CrawlRun(
user_id=user_id,
status=RunStatus.FAILED,
trigger_type=RunTriggerType.MANUAL,
start_dt=datetime.now(timezone.utc),
)
db_session.add(failed_run)
await db_session.flush()
pub = Publication(
title_raw="A fast quantum mechanical algorithm for database search",
title_normalized="a fast quantum mechanical algorithm for database search",
fingerprint_sha256="dummy_fingerprint_for_test",
author_text="LK Grover",
openalex_enriched=False,
)
db_session.add(pub)
await db_session.flush()
link = ScholarPublication(
scholar_profile_id=scholar.id,
publication_id=pub.id,
first_seen_run_id=failed_run.id,
)
db_session.add(link)
await db_session.commit()
# 3. Create a NEW run
new_run = CrawlRun(
user_id=user_id,
status=RunStatus.RUNNING,
trigger_type=RunTriggerType.MANUAL,
start_dt=datetime.now(timezone.utc),
)
db_session.add(new_run)
await db_session.commit()
# 4. Mock OpenAlex client to return enrichment data
mock_work = OpenAlexWork(
openalex_id="W1234567",
doi="10.1145/237814.237866",
pmid=None,
pmcid=None,
title="A fast quantum mechanical algorithm for database search",
publication_year=1996,
cited_by_count=1000,
is_oa=True,
oa_url="http://example.com/grover.pdf",
)
mock_source = MagicMock()
service = ScholarIngestionService(source=mock_source)
# We patch the client at its source, and also mock arXiv to avoid real HTTP calls
with patch("app.services.domains.openalex.client.OpenAlexClient") as MockClient, \
patch("app.services.domains.arxiv.application.discover_arxiv_id_for_publication", new=AsyncMock(return_value=None)):
mock_instance = MockClient.return_value
mock_instance.get_works_by_filter = AsyncMock(return_value=[mock_work])
# 5. Execute the enrichment pass for the NEW run
await service._enrich_pending_publications(db_session, run_id=new_run.id)
await db_session.commit()
# 6. Verification: The publication from the FAILED run should now be enriched
await db_session.refresh(pub)
assert pub.openalex_enriched is True
assert pub.citation_count == 1000
assert pub.pdf_url == "http://example.com/grover.pdf"
# Double check it was indeed processed
stmt = select(Publication).where(Publication.id == pub.id)
result = await db_session.execute(stmt)
enriched_pub = result.scalar_one()
assert enriched_pub.openalex_enriched is True

4
tests/integration/test_fixture_probe_runs.py
View file

@ -144,7 +144,9 @@ async def test_fixture_probe_run_emits_failure_and_retry_summary_metrics(
request_delay_seconds=0,
network_error_retries=1,
retry_backoff_seconds=0.0,
max_pages_per_scholar=1,
rate_limit_retries=0,
rate_limit_backoff_seconds=0.0,
max_pages_per_scholar=10,
page_size=100,
auto_queue_continuations=False,
alert_blocked_failure_threshold=1,

24
tests/integration/test_migrations.py
View file

@ -19,7 +19,7 @@ EXPECTED_TABLES = {
}
EXPECTED_ENUMS = {"run_status", "run_trigger_type"}
EXPECTED_REVISION = "20260221_0015"
EXPECTED_REVISION = "20260225_0022"
@pytest.mark.integration
@ -137,6 +137,28 @@ async def test_crawl_runs_has_manual_idempotency_unique_index(db_session: AsyncS
assert "WHERE" in indexdef
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations
@pytest.mark.asyncio
async def test_crawl_runs_has_single_active_run_unique_index(db_session: AsyncSession) -> None:
result = await db_session.execute(
text(
"""
SELECT indexdef
FROM pg_indexes
WHERE schemaname = 'public'
AND tablename = 'crawl_runs'
AND indexname = 'uq_crawl_runs_user_active'
"""
)
)
indexdef = result.scalar_one()
assert "UNIQUE INDEX" in indexdef
assert "(user_id)" in indexdef
assert "WHERE" in indexdef
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations

514
tests/integration/test_run_lifecycle_consistency.py Normal file
View file

@ -0,0 +1,514 @@
from __future__ import annotations
import asyncio
from typing import Any
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from app.api.runtime_deps import get_ingestion_service
from app.db.models import CrawlRun, Publication, RunStatus, RunTriggerType
from app.main import app
from app.services.domains.ingestion.application import ScholarIngestionService
from tests.integration.helpers import insert_user, login_user
def _csrf_headers(client: TestClient) -> dict[str, str]:
response = client.get("/api/v1/auth/me")
assert response.status_code == 200
csrf_token = response.json()["data"]["csrf_token"]
assert isinstance(csrf_token, str) and csrf_token
return {"X-CSRF-Token": csrf_token}
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_crawl_runs_enforce_single_active_run_per_user(db_session: AsyncSession) -> None:
user_id = await insert_user(
db_session,
email="api-single-active@example.com",
password="api-password",
)
await db_session.execute(
text(
"""
INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count)
VALUES (:user_id, 'manual', 'running', 1, 0)
"""
),
{"user_id": user_id},
)
await db_session.commit()
with pytest.raises(IntegrityError):
await db_session.execute(
text(
"""
INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count)
VALUES (:user_id, 'scheduled', 'resolving', 1, 0)
"""
),
{"user_id": user_id},
)
await db_session.commit()
await db_session.rollback()
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_manual_run_conflicts_when_an_active_run_exists(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
user_id = await insert_user(
db_session,
email="api-run-conflict@example.com",
password="api-password",
)
await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, :scholar_id, :display_name, true)
"""
),
{"user_id": user_id, "scholar_id": "runConflict001", "display_name": "Run Conflict"},
)
await db_session.commit()
service = ScholarIngestionService(source=object())
async def _stall_execute_run(**_kwargs: Any) -> None:
await asyncio.sleep(0.4)
monkeypatch.setattr(service, "execute_run", _stall_execute_run)
app.dependency_overrides[get_ingestion_service] = lambda: service
try:
client = TestClient(app)
login_user(client, email="api-run-conflict@example.com", password="api-password")
headers = _csrf_headers(client)
first_response = client.post("/api/v1/runs/manual", headers=headers)
assert first_response.status_code == 200
assert first_response.json()["data"]["status"] == "running"
second_response = client.post("/api/v1/runs/manual", headers=headers)
assert second_response.status_code == 409
payload = second_response.json()
assert payload["error"]["code"] == "run_in_progress"
await asyncio.sleep(0.45)
finally:
app.dependency_overrides.pop(get_ingestion_service, None)
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_cancel_allows_resolving_and_rejects_terminal_run(db_session: AsyncSession) -> None:
user_id = await insert_user(
db_session,
email="api-cancel-resolving@example.com",
password="api-password",
)
run_result = await db_session.execute(
text(
"""
INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count)
VALUES (:user_id, 'manual', 'resolving', 1, 2)
RETURNING id
"""
),
{"user_id": user_id},
)
run_id = int(run_result.scalar_one())
await db_session.commit()
client = TestClient(app)
login_user(client, email="api-cancel-resolving@example.com", password="api-password")
headers = _csrf_headers(client)
cancel_response = client.post(f"/api/v1/runs/{run_id}/cancel", headers=headers)
assert cancel_response.status_code == 200
assert cancel_response.json()["data"]["run"]["status"] == "canceled"
cancel_again_response = client.post(f"/api/v1/runs/{run_id}/cancel", headers=headers)
assert cancel_again_response.status_code == 409
assert cancel_again_response.json()["error"]["code"] == "run_not_cancelable"
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_background_enrichment_preserves_canceled_status(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
user_id = await insert_user(
db_session,
email="api-cancel-enrichment@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, 'cancelResolve001', 'Cancel Resolve', true)
RETURNING id
"""
),
{"user_id": user_id},
)
scholar_profile_id = int(scholar_result.scalar_one())
run = CrawlRun(
user_id=user_id,
trigger_type=RunTriggerType.MANUAL,
status=RunStatus.RESOLVING,
scholar_count=1,
)
publication = Publication(
fingerprint_sha256=f"{(user_id + 9000):064x}",
title_raw="Cancel During Resolving",
title_normalized="cancel during resolving",
citation_count=0,
openalex_enriched=False,
)
db_session.add_all([run, publication])
await db_session.flush()
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (
scholar_profile_id,
publication_id,
first_seen_run_id,
is_read
)
VALUES (:scholar_profile_id, :publication_id, :run_id, false)
"""
),
{
"scholar_profile_id": scholar_profile_id,
"publication_id": int(publication.id),
"run_id": int(run.id),
},
)
await db_session.commit()
session_factory = async_sessionmaker(db_session.bind, expire_on_commit=False)
class _OpenAlexClientStub:
def __init__(self, *args: Any, **kwargs: Any) -> None:
_ = (args, kwargs)
async def get_works_by_filter(self, *_args: Any, **_kwargs: Any) -> list[Any]:
async with session_factory() as cancel_session:
run_to_cancel = await cancel_session.get(CrawlRun, int(run.id))
assert run_to_cancel is not None
run_to_cancel.status = RunStatus.CANCELED
await cancel_session.commit()
return []
monkeypatch.setattr(
"app.services.domains.openalex.client.OpenAlexClient",
_OpenAlexClientStub,
)
service = ScholarIngestionService(source=object())
await service._background_enrich(
session_factory,
run_id=int(run.id),
intended_final_status=RunStatus.SUCCESS,
)
await db_session.refresh(run)
await db_session.refresh(publication)
assert run.status == RunStatus.CANCELED
assert publication.openalex_enriched is False
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_publications_list_endpoint_does_not_raise_name_error(
db_session: AsyncSession,
) -> None:
user_id = await insert_user(
db_session,
email="api-publications-nameerror@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, 'pubNameError001', 'Pub NameError', true)
RETURNING id
"""
),
{"user_id": user_id},
)
scholar_profile_id = int(scholar_result.scalar_one())
publication_result = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 1)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 2222):064x}",
"title_raw": "NameError Regression",
"title_normalized": "nameerror regression",
},
)
publication_id = int(publication_result.scalar_one())
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
VALUES (:scholar_profile_id, :publication_id, false, false)
"""
),
{"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
)
await db_session.commit()
client = TestClient(app)
login_user(client, email="api-publications-nameerror@example.com", password="api-password")
response = client.get("/api/v1/publications?mode=all&limit=10&offset=0")
assert response.status_code == 200
assert len(response.json()["data"]["publications"]) == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_partial_discovery_exception_keeps_new_pub_count_consistent(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
user_id = await insert_user(
db_session,
email="api-pub-count-consistency@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, 'countConsistency001', 'Count Consistency', true)
RETURNING id
"""
),
{"user_id": user_id},
)
scholar_profile_id = int(scholar_result.scalar_one())
run = CrawlRun(
user_id=user_id,
trigger_type=RunTriggerType.MANUAL,
status=RunStatus.RUNNING,
scholar_count=1,
new_pub_count=0,
)
publication = Publication(
fingerprint_sha256=f"{(user_id + 777):064x}",
title_raw="Persisted Discovery",
title_normalized="persisted discovery",
citation_count=0,
)
db_session.add_all([run, publication])
await db_session.commit()
service = ScholarIngestionService(source=object())
call_count = 0
async def _resolve_publication_stub(*_args: Any, **_kwargs: Any) -> Publication:
nonlocal call_count
call_count += 1
if call_count == 1:
return publication
raise RuntimeError("mid_page_failure")
monkeypatch.setattr(service, "_resolve_publication", _resolve_publication_stub)
from app.services.domains.scholar.parser_types import PublicationCandidate
from app.db.models import ScholarProfile
scholar = await db_session.get(ScholarProfile, scholar_profile_id)
assert scholar is not None
publications = [
PublicationCandidate(
title="Persisted Discovery",
title_url=None,
cluster_id=None,
year=None,
citation_count=0,
authors_text=None,
venue_text=None,
pdf_url=None,
),
PublicationCandidate(
title="Will Fail",
title_url=None,
cluster_id=None,
year=None,
citation_count=0,
authors_text=None,
venue_text=None,
pdf_url=None,
),
]
with pytest.raises(RuntimeError, match="mid_page_failure"):
await service._upsert_profile_publications(
db_session,
run=run,
scholar=scholar,
publications=publications,
)
refreshed_run = await db_session.get(CrawlRun, int(run.id))
assert refreshed_run is not None
assert int(refreshed_run.new_pub_count) == 1
link_count_result = await db_session.execute(
text(
"""
SELECT count(*)
FROM scholar_publications
WHERE scholar_profile_id = :scholar_profile_id
AND first_seen_run_id = :run_id
"""
),
{"scholar_profile_id": scholar_profile_id, "run_id": int(run.id)},
)
assert int(link_count_result.scalar_one()) == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_publications_pagination_snapshot_stays_stable_across_inserts(
db_session: AsyncSession,
) -> None:
user_id = await insert_user(
db_session,
email="api-publications-snapshot@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, 'pagingSnapshot001', 'Paging Snapshot', true)
RETURNING id
"""
),
{"user_id": user_id},
)
scholar_profile_id = int(scholar_result.scalar_one())
publication_ids: list[int] = []
for index in range(4):
created = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 1)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 1000 + index):064x}",
"title_raw": f"Stable Page {index}",
"title_normalized": f"stable page {index}",
},
)
publication_ids.append(int(created.scalar_one()))
for publication_id in publication_ids:
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
VALUES (:scholar_profile_id, :publication_id, false, false)
"""
),
{"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
)
await db_session.commit()
client = TestClient(app)
login_user(client, email="api-publications-snapshot@example.com", password="api-password")
first_page_response = client.get("/api/v1/publications?mode=all&page=1&page_size=2")
assert first_page_response.status_code == 200
first_page = first_page_response.json()["data"]
first_page_ids = [int(item["publication_id"]) for item in first_page["publications"]]
snapshot = first_page["snapshot"]
assert isinstance(snapshot, str) and snapshot
inserted = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 1)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 5000):064x}",
"title_raw": "Inserted After Snapshot",
"title_normalized": "inserted after snapshot",
},
)
inserted_publication_id = int(inserted.scalar_one())
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
VALUES (:scholar_profile_id, :publication_id, false, false)
"""
),
{"scholar_profile_id": scholar_profile_id, "publication_id": inserted_publication_id},
)
await db_session.commit()
second_page_response = client.get(
"/api/v1/publications",
params={
"mode": "all",
"page": 2,
"page_size": 2,
"snapshot": snapshot,
},
)
assert second_page_response.status_code == 200
second_page = second_page_response.json()["data"]
second_page_ids = [int(item["publication_id"]) for item in second_page["publications"]]
first_page_again_response = client.get(
"/api/v1/publications",
params={
"mode": "all",
"page": 1,
"page_size": 2,
"snapshot": snapshot,
},
)
assert first_page_again_response.status_code == 200
first_page_again = first_page_again_response.json()["data"]
first_page_again_ids = [
int(item["publication_id"]) for item in first_page_again["publications"]
]
assert first_page_again_ids == first_page_ids
assert not (set(first_page_ids) & set(second_page_ids))
assert inserted_publication_id not in set(first_page_ids + second_page_ids)

64
tests/unit/services/domains/openalex/test_client.py Normal file
View file

@ -0,0 +1,64 @@
import pytest
from app.services.domains.openalex.types import OpenAlexWork
def test_parse_openalex_work_from_api_dict() -> None:
raw_api_response = {
"id": "https://openalex.org/W2741809807",
"doi": "https://doi.org/10.1038/s41586-020-0315-z",
"title": "Machine learning and the physical sciences",
"publication_year": 2019,
"cited_by_count": 1420,
"ids": {
"openalex": "https://openalex.org/W2741809807",
"doi": "https://doi.org/10.1038/s41586-020-0315-z",
"mag": "2741809807",
"pmid": "https://pubmed.ncbi.nlm.nih.gov/32040050",
"pmcid": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7325852"
},
"open_access": {
"is_oa": True,
"oa_url": "https://example.com/pdf"
},
"authorships": [
{
"author_position": "first",
"author": {
"id": "https://openalex.org/A1969205032",
"display_name": "Giuseppe Carleo"
}
},
{
"author_position": "middle",
"author": {
"id": "https://openalex.org/A4356881717",
"display_name": "Ignacio Cirac"
}
}
]
}
work = OpenAlexWork.from_api_dict(raw_api_response)
assert work.openalex_id == "https://openalex.org/W2741809807"
assert work.title == "Machine learning and the physical sciences"
assert work.doi == "10.1038/s41586-020-0315-z"
assert work.pmid == "32040050"
assert work.pmcid == "PMC7325852"
assert work.publication_year == 2019
assert work.cited_by_count == 1420
assert work.is_oa is True
assert work.oa_url == "https://example.com/pdf"
assert len(work.authors) == 2
assert work.authors[0].display_name == "Giuseppe Carleo"
assert work.authors[1].display_name == "Ignacio Cirac"
def test_parse_openalex_work_empty() -> None:
work = OpenAlexWork.from_api_dict({"id": "W123"})
assert work.openalex_id == "W123"
assert work.doi is None
assert work.pmid is None
assert work.title is None
assert work.publication_year is None
assert work.cited_by_count == 0
assert work.is_oa is False
assert len(work.authors) == 0

55
tests/unit/services/domains/openalex/test_matching.py Normal file
View file

@ -0,0 +1,55 @@
import pytest
from app.services.domains.openalex.types import OpenAlexWork
from app.services.domains.openalex.matching import find_best_match
def test_find_best_match_exact_title():
cand1 = OpenAlexWork.from_api_dict({"id": "W1", "title": "Exact Title of the Paper"})
cand2 = OpenAlexWork.from_api_dict({"id": "W2", "title": "Totally Different Paper"})
match = find_best_match("Exact Title of the Paper", 2020, "Author A", [cand1, cand2])
assert match is not None
assert match.openalex_id == "W1"
def test_find_best_match_fuzzy_title():
# Only differences are punctuation or minor phrasing (e.g., matching a preprint title vs published)
cand1 = OpenAlexWork.from_api_dict({"id": "W1", "title": "Fuzzier Title: A Study on OpenAlex"})
cand2 = OpenAlexWork.from_api_dict({"id": "W2", "title": "Some completely unrelated work"})
match = find_best_match("Fuzzier Title A Study on OpenAlex", 2021, "Author B", [cand1, cand2])
assert match is not None
assert match.openalex_id == "W1"
def test_find_best_match_rejects_low_score():
cand1 = OpenAlexWork.from_api_dict({"id": "W1", "title": "Cats in hats"})
match = find_best_match("Dogs with logs", 2020, "Author A", [cand1])
assert match is None
def test_find_best_match_year_tiebreaker():
# Both titles are very similar, one has exact year.
cand1 = OpenAlexWork.from_api_dict({"id": "W1", "title": "The exact same title", "publication_year": 2018})
cand2 = OpenAlexWork.from_api_dict({"id": "W2", "title": "The exact same title", "publication_year": 2020})
match = find_best_match("The exact same title", 2020, "Author A", [cand1, cand2])
assert match is not None
assert match.openalex_id == "W2"
def test_find_best_match_author_tiebreaker():
# Titles and years match exactly. Author overlap decides it.
cand1 = OpenAlexWork.from_api_dict({
"id": "W1",
"title": "A popular title",
"publication_year": 2020,
"authorships": [{"author": {"display_name": "Smith, J"}}]
})
cand2 = OpenAlexWork.from_api_dict({
"id": "W2",
"title": "A popular title",
"publication_year": 2020,
"authorships": [{"author": {"display_name": "Doe, J"}}]
})
# Target authors contains "Doe"
match = find_best_match("A popular title", 2020, "A Einstein, J Doe", [cand1, cand2])
assert match is not None
assert match.openalex_id == "W2"

163
tests/unit/services/domains/publications/test_dedup.py Normal file
View file

@ -0,0 +1,163 @@
"""Unit tests for the identifier-based publication dedup sweep.
DB operations are mocked via AsyncMock so no database is required.
"""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.db.models import ScholarPublication
from app.services.domains.publications.dedup import (
find_identifier_duplicate_pairs,
merge_duplicate_publication,
sweep_identifier_duplicates,
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_result(rows: list) -> MagicMock:
mock_result = MagicMock()
mock_result.scalars.return_value.all.return_value = rows
mock_result.__iter__ = lambda self: iter(rows)
return mock_result
def _session_with_execute_sequence(results: list) -> AsyncMock:
session = AsyncMock()
session.execute = AsyncMock(side_effect=[_make_result(r) for r in results])
session.delete = AsyncMock()
session.flush = AsyncMock()
return session
# ---------------------------------------------------------------------------
# find_identifier_duplicate_pairs
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_find_identifier_duplicate_pairs_returns_pairs() -> None:
session = AsyncMock()
session.execute = AsyncMock(return_value=_make_result([(1, 2)]))
pairs = await find_identifier_duplicate_pairs(session)
assert pairs == [(1, 2)]
session.execute.assert_awaited_once()
@pytest.mark.asyncio
async def test_find_identifier_duplicate_pairs_returns_empty_when_no_duplicates() -> None:
session = AsyncMock()
session.execute = AsyncMock(return_value=_make_result([]))
pairs = await find_identifier_duplicate_pairs(session)
assert pairs == []
# ---------------------------------------------------------------------------
# merge_duplicate_publication
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_merge_duplicate_migrates_orphaned_scholar_links() -> None:
"""Scholar link that only the dup has should be migrated to winner."""
dup_link = MagicMock(spec=ScholarPublication)
dup_link.scholar_profile_id = 99
dup_link.publication_id = 2
session = _session_with_execute_sequence(
results=[
[dup_link], # dup links
[], # winner profile ids (no conflict)
[], # execute(delete(Publication)) result
]
)
await merge_duplicate_publication(session, winner_id=1, dup_id=2)
assert dup_link.publication_id == 1
session.delete.assert_not_awaited() # not deleted; migrated instead
@pytest.mark.asyncio
async def test_merge_duplicate_drops_conflicting_scholar_link() -> None:
"""When winner already has a link for the same scholar, dup's link is deleted."""
dup_link = MagicMock(spec=ScholarPublication)
dup_link.scholar_profile_id = 88
dup_link.publication_id = 2
session = _session_with_execute_sequence(
results=[
[dup_link], # dup links
[(88,)], # winner profiles (conflict: profile 88 already linked)
[], # execute(delete(Publication)) result
]
)
await merge_duplicate_publication(session, winner_id=1, dup_id=2)
session.delete.assert_awaited_once_with(dup_link)
assert dup_link.publication_id == 2 # unchanged — link was deleted, not migrated
# ---------------------------------------------------------------------------
# sweep_identifier_duplicates
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_sweep_returns_zero_when_no_pairs() -> None:
with patch(
"app.services.domains.publications.dedup.find_identifier_duplicate_pairs",
new=AsyncMock(return_value=[]),
):
session = AsyncMock()
count = await sweep_identifier_duplicates(session)
assert count == 0
session.flush.assert_not_awaited()
@pytest.mark.asyncio
async def test_sweep_returns_merge_count() -> None:
with (
patch(
"app.services.domains.publications.dedup.find_identifier_duplicate_pairs",
new=AsyncMock(return_value=[(1, 2), (3, 4)]),
),
patch(
"app.services.domains.publications.dedup.merge_duplicate_publication",
new=AsyncMock(),
) as mock_merge,
):
session = AsyncMock()
count = await sweep_identifier_duplicates(session)
assert count == 2
assert mock_merge.await_count == 2
session.flush.assert_awaited_once()
@pytest.mark.asyncio
async def test_sweep_merges_each_dup_only_once() -> None:
"""A dup sharing two identifiers with the winner appears twice in pairs but merged once."""
with (
patch(
"app.services.domains.publications.dedup.find_identifier_duplicate_pairs",
new=AsyncMock(return_value=[(1, 2), (1, 2)]),
),
patch(
"app.services.domains.publications.dedup.merge_duplicate_publication",
new=AsyncMock(),
) as mock_merge,
):
session = AsyncMock()
count = await sweep_identifier_duplicates(session)
assert count == 1
assert mock_merge.await_count == 1

242
tests/unit/test_fingerprints.py Normal file
View file

@ -0,0 +1,242 @@
from __future__ import annotations
import pytest
from app.services.domains.ingestion.fingerprints import (
_dedupe_publication_candidates,
canonical_title_for_dedup,
fuzzy_titles_match,
normalize_title,
)
from app.services.domains.scholar.parser import PublicationCandidate
def _candidate(
title: str,
*,
cluster_id: str | None = None,
year: int | None = 2024,
authors_text: str | None = "Smith, J",
venue_text: str | None = "ICML",
) -> PublicationCandidate:
return PublicationCandidate(
title=title,
title_url=None,
cluster_id=cluster_id,
year=year,
citation_count=None,
authors_text=authors_text,
venue_text=venue_text,
pdf_url=None,
)
class TestFuzzyTitlesMatch:
def test_identical_titles(self) -> None:
assert fuzzy_titles_match("Deep Learning for NLP", "deep learning for nlp") is True
def test_minor_word_difference(self) -> None:
assert fuzzy_titles_match(
"A Survey on Deep Learning Methods for NLP",
"Survey on Deep Learning Methods for NLP",
) is True
def test_punctuation_difference(self) -> None:
assert fuzzy_titles_match(
"Attention Is All You Need",
"Attention Is All You Need.",
) is True
def test_colon_vs_dash_subtitle(self) -> None:
assert fuzzy_titles_match(
"Deep Learning: A Comprehensive Survey",
"Deep Learning - A Comprehensive Survey",
) is True
def test_completely_different_titles(self) -> None:
assert fuzzy_titles_match(
"Deep Learning for NLP",
"Climate Change Impact on Agriculture",
) is False
def test_short_title_no_false_positive(self) -> None:
assert fuzzy_titles_match("On Trees", "On Forests") is False
def test_empty_title(self) -> None:
assert fuzzy_titles_match("", "Deep Learning") is False
def test_custom_threshold(self) -> None:
# Lower threshold catches more distant matches
assert fuzzy_titles_match(
"A Survey on Deep Learning",
"Survey on Machine Learning Approaches",
threshold=0.3,
) is True
# Default threshold rejects them
assert fuzzy_titles_match(
"A Survey on Deep Learning",
"Survey on Machine Learning Approaches",
) is False
class TestDedupePublicationCandidates:
def test_exact_duplicates_by_cluster_id(self) -> None:
pubs = [
_candidate("Title A", cluster_id="c1"),
_candidate("Title A Copy", cluster_id="c1"),
]
result = _dedupe_publication_candidates(pubs)
assert len(result) == 1
assert result[0].title == "Title A"
def test_fuzzy_duplicates_without_cluster_id(self) -> None:
pubs = [
_candidate("Attention Is All You Need"),
_candidate("Attention Is All You Need."),
]
result = _dedupe_publication_candidates(pubs)
assert len(result) == 1
def test_distinct_titles_preserved(self) -> None:
pubs = [
_candidate("Deep Learning for NLP"),
_candidate("Reinforcement Learning for Robotics"),
]
result = _dedupe_publication_candidates(pubs)
assert len(result) == 2
def test_fallback_aligned_with_db_fingerprint(self) -> None:
"""Same title/year/first_author/first_venue should deduplicate even with
different full authors_text or venue_text."""
pubs = [
_candidate(
"My Paper",
authors_text="Smith, J; Jones, A",
venue_text="International Conference on ML",
),
_candidate(
"My Paper",
authors_text="Smith, J; Baker, B",
venue_text="International Conference for ML",
),
]
result = _dedupe_publication_candidates(pubs)
# Both share first_author_last_name="smith" and first_venue_word="international"
assert len(result) == 1
def test_mixed_cluster_and_fuzzy(self) -> None:
pubs = [
_candidate("A Comprehensive Survey on Deep Learning Methods", cluster_id="c1"),
_candidate("Comprehensive Survey on Deep Learning Methods"), # fuzzy match (subtitle stripped)
_candidate("Completely Different Study"),
]
result = _dedupe_publication_candidates(pubs)
assert len(result) == 2
titles = [p.title for p in result]
assert "A Comprehensive Survey on Deep Learning Methods" in titles
assert "Completely Different Study" in titles
def test_scholar_noise_variants_collapse_to_one(self) -> None:
"""The motivating case: three Scholar display variants of the Adam paper."""
pubs = [
_candidate(
"Adam: A method for stochastic optimization, preprint (2014)",
year=2014,
venue_text="",
),
_candidate(
"Adam: A Method for Stochastic Optimization. arXiv, Jan 29, 2017. doi: 10.48550/arxiv.1412.6980",
year=2017,
venue_text="arXiv",
),
_candidate(
"Adam a method for stochastic optimization. Comput. Sci",
year=2015,
venue_text="Comput. Sci",
),
]
result = _dedupe_publication_candidates(pubs)
assert len(result) == 1
assert result[0].title == pubs[0].title
def test_distinct_papers_not_merged(self) -> None:
"""Papers with different core titles must not be collapsed."""
pubs = [
_candidate("Adam: A Method for Stochastic Optimization"),
_candidate("SGD: Stochastic Gradient Descent Revisited"),
_candidate("Attention Is All You Need"),
]
result = _dedupe_publication_candidates(pubs)
assert len(result) == 3
def test_cross_page_dedup_via_seen_canonical(self) -> None:
"""seen_canonical threads state across two separate calls (simulating two pages)."""
seen: set[str] = set()
page1 = [_candidate("Adam: A Method for Stochastic Optimization")]
page2 = [
_candidate(
"Adam: A method for stochastic optimization, preprint (2014)",
year=2014,
),
_candidate("An Entirely Different Paper"),
]
result1 = _dedupe_publication_candidates(page1, seen_canonical=seen)
result2 = _dedupe_publication_candidates(page2, seen_canonical=seen)
assert len(result1) == 1
# Noisy Adam variant from page 2 is suppressed; distinct paper survives
assert len(result2) == 1
assert result2[0].title == "An Entirely Different Paper"
def test_first_seen_wins_in_noise_collapse(self) -> None:
"""First occurrence in page order is the kept candidate."""
pubs = [
_candidate("Adam: A Method for Stochastic Optimization", year=2015),
_candidate("Adam: A method for stochastic optimization, preprint (2014)", year=2014),
]
result = _dedupe_publication_candidates(pubs)
assert len(result) == 1
assert result[0].year == 2015 # first wins
class TestCanonicalTitleForDedup:
def test_strips_doi_suffix(self) -> None:
title = "Adam: A Method for Stochastic Optimization. doi: 10.48550/arxiv.1412.6980"
assert canonical_title_for_dedup(title) == normalize_title(
"Adam: A Method for Stochastic Optimization"
)
def test_strips_arxiv_metadata_suffix(self) -> None:
title = "Adam: A Method for Stochastic Optimization. arXiv, Jan 29, 2017"
assert canonical_title_for_dedup(title) == normalize_title(
"Adam: A Method for Stochastic Optimization"
)
def test_strips_preprint_parenthetical(self) -> None:
title = "Adam: A method for stochastic optimization, preprint (2014)"
assert canonical_title_for_dedup(title) == normalize_title(
"Adam: A method for stochastic optimization"
)
def test_strips_venue_sentence_suffix(self) -> None:
title = "Adam a method for stochastic optimization. Comput. Sci"
assert canonical_title_for_dedup(title) == normalize_title(
"Adam a method for stochastic optimization"
)
def test_strips_trailing_year_in_parens(self) -> None:
assert canonical_title_for_dedup("Deep Learning (2018)") == normalize_title("Deep Learning")
def test_preserves_clean_title(self) -> None:
title = "Attention Is All You Need"
assert canonical_title_for_dedup(title) == normalize_title(title)
def test_adam_variants_produce_identical_canonical(self) -> None:
variants = [
"Adam: A method for stochastic optimization, preprint (2014)",
"Adam: A Method for Stochastic Optimization. arXiv, Jan 29, 2017. doi: 10.48550/arxiv.1412.6980",
"Adam a method for stochastic optimization. Comput. Sci",
]
canonicals = [canonical_title_for_dedup(v) for v in variants]
assert len(set(canonicals)) == 1, f"Expected one canonical, got: {canonicals}"

69
tests/unit/test_publication_identifiers.py
View file

@ -36,3 +36,72 @@ def test_derive_display_identifier_uses_pmcid_when_present() -> None:
assert display is not None
assert display.kind == "pmcid"
assert display.value == "PMC2175868"
def test_normalize_arxiv_id_handles_urls() -> None:
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
assert normalize_arxiv_id("https://arxiv.org/abs/1504.08025") == "1504.08025"
assert normalize_arxiv_id("http://arxiv.org/pdf/1504.08025v2.pdf") == "1504.08025v2"
assert normalize_arxiv_id("https://arxiv.org/html/1504.08025v2") == "1504.08025v2"
# Modern arxiv format
assert normalize_arxiv_id("https://arxiv.org/abs/2012.00001") == "2012.00001"
# Old arxiv format
assert normalize_arxiv_id("https://arxiv.org/abs/math/9901123") == "math/9901123"
def test_normalize_arxiv_id_handles_raw_text() -> None:
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
assert normalize_arxiv_id("arXiv:1504.08025") == "1504.08025"
assert normalize_arxiv_id("arxiv: 1504.08025v1") == "1504.08025v1"
assert normalize_arxiv_id("Preprint at arXiv:math/9901123v2") == "math/9901123v2"
assert normalize_arxiv_id("Not an arxiv: 123") is None
def test_normalize_pmcid_handles_urls_and_text() -> None:
from app.services.domains.publication_identifiers.normalize import normalize_pmcid
assert normalize_pmcid("https://pmc.ncbi.nlm.nih.gov/articles/PMC2175868/") == "PMC2175868"
assert normalize_pmcid("http://www.ncbi.nlm.nih.gov/pmc/articles/PMC1234567") == "PMC1234567"
assert normalize_pmcid("PMCID: PMC1234567") == "PMC1234567"
assert normalize_pmcid("pmc1234567") == "PMC1234567"
assert normalize_pmcid("Not a PMCID 1234567") is None
def test_normalize_pmid_handles_urls() -> None:
from app.services.domains.publication_identifiers.normalize import normalize_pmid
assert normalize_pmid("https://pubmed.ncbi.nlm.nih.gov/12345678/") == "12345678"
assert normalize_pmid("http://pubmed.ncbi.nlm.nih.gov/12345678") == "12345678"
assert normalize_pmid("https://pubmed.ncbi.nlm.nih.gov/not-a-pmid/") is None
import pytest
from app.services.domains.arxiv import application as arxiv_service
from app.services.domains.publications.types import UnreadPublicationItem
@pytest.mark.asyncio
async def test_discover_arxiv_id_returns_none_if_no_title() -> None:
item = UnreadPublicationItem(
publication_id=1,
scholar_profile_id=1,
scholar_label="First Last",
title="",
year=2023,
citation_count=0,
venue_text=None,
pub_url=None,
pdf_url=None,
)
result = await arxiv_service.discover_arxiv_id_for_publication(item=item)
assert result is None
@pytest.mark.asyncio
async def test_build_arxiv_query() -> None:
query = arxiv_service._build_arxiv_query("Super AI Model", "Smith")
assert query == 'ti:"Super AI Model" AND au:"Smith"'
query2 = arxiv_service._build_arxiv_query("Only Title Here", None)
assert query2 == 'ti:"Only Title Here"'

72
tests/unit/test_publication_pdf_queue_policy.py
View file

@ -6,6 +6,7 @@ from types import SimpleNamespace
import pytest
from app.db.models import PublicationPdfJob
from app.services.domains.arxiv.application import ArxivRateLimitError
from app.services.domains.publications import pdf_queue
from app.services.domains.publications.pdf_resolution_pipeline import PipelineOutcome
from app.services.domains.unpaywall.application import OaResolutionOutcome
@ -133,7 +134,7 @@ def test_pdf_queue_manual_requeue_still_blocks_when_inflight() -> None:
@pytest.mark.asyncio
async def test_fetch_outcome_for_row_uses_pipeline_outcome(monkeypatch: pytest.MonkeyPatch) -> None:
async def _fake_pipeline(*, row, request_email=None):
async def _fake_pipeline(*, row, request_email=None, openalex_api_key=None):
assert request_email == "user@example.com"
return PipelineOutcome(
outcome=OaResolutionOutcome(
@ -141,7 +142,7 @@ async def test_fetch_outcome_for_row_uses_pipeline_outcome(monkeypatch: pytest.M
doi=None,
pdf_url="https://arxiv.org/pdf/1703.06103",
failure_reason=None,
source=pdf_queue.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE,
source="openalex",
used_crossref=False,
),
scholar_candidates=None,
@ -152,7 +153,7 @@ async def test_fetch_outcome_for_row_uses_pipeline_outcome(monkeypatch: pytest.M
outcome = await pdf_queue._fetch_outcome_for_row(row=_row(), request_email="user@example.com")
assert outcome.pdf_url == "https://arxiv.org/pdf/1703.06103"
assert outcome.source == pdf_queue.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE
assert outcome.source == "openalex"
assert outcome.used_crossref is False
@ -160,7 +161,7 @@ async def test_fetch_outcome_for_row_uses_pipeline_outcome(monkeypatch: pytest.M
async def test_fetch_outcome_for_row_returns_failed_outcome_when_pipeline_returns_none(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def _fake_pipeline(*, row, request_email=None):
async def _fake_pipeline(*, row, request_email=None, openalex_api_key=None):
assert request_email == "user@example.com"
return PipelineOutcome(outcome=None, scholar_candidates=None)
@ -170,3 +171,66 @@ async def test_fetch_outcome_for_row_returns_failed_outcome_when_pipeline_return
assert outcome.pdf_url is None
assert outcome.failure_reason == pdf_queue.FAILURE_RESOLUTION_EXCEPTION
@pytest.mark.asyncio
async def test_resolve_publication_row_persists_failed_outcome_before_reraising_arxiv_rate_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
captured: list[tuple[int, int, OaResolutionOutcome]] = []
async def _noop_mark_attempt_started(*, publication_id: int, user_id: int) -> None:
return None
async def _raise_rate_limit(*, row, request_email=None, openalex_api_key=None):
raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
async def _capture_persist_outcome(*, publication_id: int, user_id: int, outcome: OaResolutionOutcome) -> None:
captured.append((publication_id, user_id, outcome))
monkeypatch.setattr(pdf_queue, "_mark_attempt_started", _noop_mark_attempt_started)
monkeypatch.setattr(pdf_queue, "_fetch_outcome_for_row", _raise_rate_limit)
monkeypatch.setattr(pdf_queue, "_persist_outcome", _capture_persist_outcome)
with pytest.raises(ArxivRateLimitError):
await pdf_queue._resolve_publication_row(
user_id=42,
request_email="user@example.com",
row=_row(),
openalex_api_key="key",
)
assert len(captured) == 1
publication_id, user_id, outcome = captured[0]
assert publication_id == 1
assert user_id == 42
assert outcome.pdf_url is None
assert outcome.failure_reason == pdf_queue.FAILURE_RESOLUTION_EXCEPTION
@pytest.mark.asyncio
async def test_run_resolution_task_stops_batch_on_arxiv_rate_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls: list[int] = []
first = _row()
second = SimpleNamespace(**{**first.__dict__, "publication_id": 2})
def _raise_session_factory_error():
raise RuntimeError("skip user settings lookup in test")
async def _fake_resolve_publication_row(*, user_id: int, request_email: str | None, row, openalex_api_key=None):
calls.append(int(row.publication_id))
if row.publication_id == 1:
raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
monkeypatch.setattr(pdf_queue, "get_session_factory", _raise_session_factory_error)
monkeypatch.setattr(pdf_queue, "_resolve_publication_row", _fake_resolve_publication_row)
await pdf_queue._run_resolution_task(
user_id=42,
request_email="user@example.com",
rows=[first, second],
)
assert calls == [1]

149
tests/unit/test_publication_pdf_resolution_pipeline.py
View file

@ -1,32 +1,16 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
from types import SimpleNamespace
import pytest
from app.services.domains.publications import pdf_resolution_pipeline as pipeline
from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.domains.unpaywall.application import OaResolutionOutcome
@dataclass(frozen=True)
class _Candidate:
url: str
confidence_score: float
label_present: bool
reason: str
@dataclass(frozen=True)
class _Candidates:
container_seen: bool
labeled_candidate: _Candidate | None
fallback_candidate: _Candidate | None
warnings: tuple[str, ...] = ()
def _row(*, doi: str | None = None) -> SimpleNamespace:
def _row(*, display_identifier: DisplayIdentifier | None = None) -> SimpleNamespace:
return SimpleNamespace(
publication_id=1,
scholar_profile_id=1,
@ -36,7 +20,7 @@ def _row(*, doi: str | None = None) -> SimpleNamespace:
citation_count=0,
venue_text=None,
pub_url="https://scholar.google.com/citations?view_op=view_citation&citation_for_view=abc:xyz",
doi=doi,
display_identifier=display_identifier,
pdf_url=None,
is_read=False,
is_favorite=False,
@ -45,7 +29,20 @@ def _row(*, doi: str | None = None) -> SimpleNamespace:
)
def _oa_outcome(*, pdf_url: str | None, source: str = "unpaywall") -> OaResolutionOutcome:
def _api_outcome(*, pdf_url: str | None, source: str = "unpaywall") -> OaResolutionOutcome | None:
if not pdf_url:
return None
return OaResolutionOutcome(
publication_id=1,
doi="10.1000/example",
pdf_url=pdf_url,
failure_reason=None if pdf_url else "no_pdf_found",
source=source,
used_crossref=False,
)
def _oa_fallback_outcome(*, pdf_url: str | None, source: str = "unpaywall") -> OaResolutionOutcome:
return OaResolutionOutcome(
publication_id=1,
doi="10.1000/example",
@ -57,97 +54,63 @@ def _oa_outcome(*, pdf_url: str | None, source: str = "unpaywall") -> OaResoluti
@pytest.mark.asyncio
async def test_pipeline_prefers_labeled_scholar_candidate_before_oa(monkeypatch: pytest.MonkeyPatch) -> None:
async def _fake_candidates(_url):
return _Candidates(
container_seen=True,
labeled_candidate=_Candidate(
url="https://arxiv.org/pdf/1703.06103",
confidence_score=0.98,
label_present=True,
reason="scholar_link_labeled_pdf",
),
fallback_candidate=None,
)
async def test_pipeline_prefers_openalex_before_arxiv(monkeypatch: pytest.MonkeyPatch) -> None:
async def _fake_openalex(row, request_email: str | None = None, openalex_api_key: str | None = None):
return _api_outcome(pdf_url="https://oa.example.org/found.pdf", source="openalex")
async def _fail_oa(*, row, request_email):
raise AssertionError(f"OA should not run when labeled Scholar candidate exists: {row.publication_id} {request_email}")
async def _fail_arxiv(row):
raise AssertionError(f"arXiv should not run when OpenAlex candidate exists.")
monkeypatch.setattr(pipeline, "fetch_link_candidates_from_scholar_publication_page", _fake_candidates)
monkeypatch.setattr(pipeline, "_oa_outcome", _fail_oa)
result = await pipeline.resolve_publication_pdf_outcome_for_row(row=_row(), request_email="user@example.com")
assert result.outcome is not None
assert result.outcome.pdf_url == "https://arxiv.org/pdf/1703.06103"
assert result.outcome.source == pipeline.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE
@pytest.mark.asyncio
async def test_pipeline_uses_oa_result_before_unlabeled_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
async def _fake_candidates(_url):
return _Candidates(
container_seen=True,
labeled_candidate=None,
fallback_candidate=_Candidate(
url="https://example.org/download/42",
confidence_score=0.2,
label_present=False,
reason="scholar_link_unlabeled_fallback",
),
)
async def _fake_oa(*, row, request_email):
assert request_email == "user@example.com"
return _oa_outcome(pdf_url="https://oa.example.org/found.pdf")
async def _fail_fallback(*, row, candidate):
raise AssertionError(f"Unlabeled fallback should not run when OA returns PDF: {row.publication_id} {candidate.url}")
monkeypatch.setattr(pipeline, "fetch_link_candidates_from_scholar_publication_page", _fake_candidates)
monkeypatch.setattr(pipeline, "_oa_outcome", _fake_oa)
monkeypatch.setattr(pipeline, "_unlabeled_fallback_outcome", _fail_fallback)
monkeypatch.setattr(pipeline, "_openalex_outcome", _fake_openalex)
monkeypatch.setattr(pipeline, "_arxiv_outcome", _fail_arxiv)
result = await pipeline.resolve_publication_pdf_outcome_for_row(row=_row(), request_email="user@example.com")
assert result.outcome is not None
assert result.outcome.pdf_url == "https://oa.example.org/found.pdf"
assert result.outcome.source == "unpaywall"
assert result.outcome.source == "openalex"
@pytest.mark.asyncio
async def test_pipeline_uses_unlabeled_fallback_after_oa_failure(monkeypatch: pytest.MonkeyPatch) -> None:
fallback_candidate = _Candidate(
url="https://example.org/download/42",
confidence_score=0.2,
label_present=False,
reason="scholar_link_unlabeled_fallback",
)
async def test_pipeline_uses_arxiv_after_openalex_failure(monkeypatch: pytest.MonkeyPatch) -> None:
async def _fake_openalex(row, request_email: str | None = None, openalex_api_key: str | None = None):
return None
async def _fake_candidates(_url):
return _Candidates(container_seen=True, labeled_candidate=None, fallback_candidate=fallback_candidate)
async def _fake_arxiv(row, request_email: str | None = None):
return _api_outcome(pdf_url="https://arxiv.org/pdf/1234.5678.pdf", source="arxiv")
async def _fail_oa(*, row, request_email):
raise AssertionError("Unpaywall should not run when arXiv returns PDF.")
monkeypatch.setattr(pipeline, "_openalex_outcome", _fake_openalex)
monkeypatch.setattr(pipeline, "_arxiv_outcome", _fake_arxiv)
monkeypatch.setattr(pipeline, "_oa_outcome", _fail_oa)
result = await pipeline.resolve_publication_pdf_outcome_for_row(row=_row(), request_email="user@example.com")
assert result.outcome is not None
assert result.outcome.pdf_url == "https://arxiv.org/pdf/1234.5678.pdf"
assert result.outcome.source == "arxiv"
@pytest.mark.asyncio
async def test_pipeline_uses_unpaywall_after_arxiv_failure(monkeypatch: pytest.MonkeyPatch) -> None:
async def _fake_openalex(row, request_email: str | None = None, openalex_api_key: str | None = None):
return None
async def _fake_arxiv(row, request_email: str | None = None):
return None
async def _fake_oa(*, row, request_email):
assert request_email == "user@example.com"
return _oa_outcome(pdf_url=None)
return _oa_fallback_outcome(pdf_url="https://example.org/fallback.pdf", source="unpaywall")
async def _fake_fallback(*, row, candidate):
assert candidate == fallback_candidate
return OaResolutionOutcome(
publication_id=row.publication_id,
doi=row.doi,
pdf_url="https://example.org/fallback.pdf",
failure_reason=None,
source=pipeline.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED,
used_crossref=False,
)
monkeypatch.setattr(pipeline, "fetch_link_candidates_from_scholar_publication_page", _fake_candidates)
monkeypatch.setattr(pipeline, "_openalex_outcome", _fake_openalex)
monkeypatch.setattr(pipeline, "_arxiv_outcome", _fake_arxiv)
monkeypatch.setattr(pipeline, "_oa_outcome", _fake_oa)
monkeypatch.setattr(pipeline, "_unlabeled_fallback_outcome", _fake_fallback)
result = await pipeline.resolve_publication_pdf_outcome_for_row(row=_row(), request_email="user@example.com")
assert result.outcome is not None
assert result.outcome.pdf_url == "https://example.org/fallback.pdf"
assert result.outcome.source == pipeline.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED
assert result.outcome.source == "unpaywall"

76
tests/unit/test_scholar_publication_pdf.py
View file

@ -1,76 +0,0 @@
from __future__ import annotations
import pytest
from app.services.domains.scholar.parser_types import ScholarDomInvariantError
from app.services.domains.scholar.publication_pdf import (
extract_link_candidates_from_publication_detail_html,
is_scholar_publication_detail_url,
)
def test_extract_link_candidates_from_publication_detail_html_reads_gsc_oci_pdf_link() -> None:
html = """
<html><body>
<div id="gsc_oci_title_gg">
<div class="gsc_oci_title_ggi">
<a href="https://arxiv.org/pdf/1703.06103" data-clk="x">
<span class="gsc_vcd_title_ggt">[PDF]</span> from arxiv.org
</a>
</div>
</div>
</body></html>
"""
candidates = extract_link_candidates_from_publication_detail_html(html)
assert candidates.labeled_candidate is not None
assert candidates.labeled_candidate.url == "https://arxiv.org/pdf/1703.06103"
def test_extract_link_candidates_from_publication_detail_html_returns_no_candidates_when_container_missing() -> None:
html = "<html><body><div id='gsc_oci_title'>No PDF section</div></body></html>"
candidates = extract_link_candidates_from_publication_detail_html(html)
assert candidates.container_seen is False
assert candidates.labeled_candidate is None
assert candidates.fallback_candidate is None
def test_extract_pdf_url_from_publication_detail_html_fails_fast_on_malformed_pdf_container() -> None:
html = """
<html><body>
<div id="gsc_oci_title_gg">
<div class="gsc_oci_title_ggi">
<a data-clk="x"><span class="gsc_vcd_title_ggt">[PDF]</span> from example.org</a>
</div>
</div>
</body></html>
"""
with pytest.raises(ScholarDomInvariantError) as exc:
extract_link_candidates_from_publication_detail_html(html)
assert exc.value.code == "layout_publication_link_missing_href"
def test_extract_link_candidates_from_publication_detail_html_keeps_unlabeled_fallback() -> None:
html = """
<html><body>
<div id="gsc_oci_title_gg">
<div class="gsc_oci_title_ggi">
<a href="https://example.org/download?id=42">from example.org</a>
</div>
</div>
</body></html>
"""
candidates = extract_link_candidates_from_publication_detail_html(html)
assert candidates.container_seen is True
assert candidates.labeled_candidate is None
assert candidates.fallback_candidate is not None
assert candidates.fallback_candidate.url == "https://example.org/download?id=42"
assert candidates.fallback_candidate.label_present is False
assert "scholar_publication_link_unlabeled_only" in candidates.warnings
assert candidates.labeled_candidate is None
def test_is_scholar_publication_detail_url_matches_view_citation_links() -> None:
assert is_scholar_publication_detail_url(
"https://scholar.google.com/citations?view_op=view_citation&hl=en&user=8200InoAAAAJ&citation_for_view=8200InoAAAAJ:gsN89kCJA0AC"
) is True
assert is_scholar_publication_detail_url("https://example.org/paper") is False

19
tests/unit/test_unpaywall_resolution.py
View file

@ -6,6 +6,7 @@ from datetime import datetime, timezone
import pytest
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.domains.unpaywall import application as unpaywall_app
@ -31,7 +32,7 @@ def _item(publication_id: int) -> PublicationListItem:
citation_count=1000,
venue_text="Cell",
pub_url="https://doi.org/10.1016/j.cell.2007.11.019",
doi=None,
display_identifier=None,
pdf_url=None,
is_read=False,
first_seen_at=datetime.now(timezone.utc),
@ -44,7 +45,13 @@ def test_publication_doi_uses_stored_value_when_metadata_has_no_doi() -> None:
_item(99),
pub_url="https://scholar.google.com/citations?view_op=view_citation&citation_for_view=abc:123",
venue_text="Cell 130 (5), 2007",
doi="10.1016/j.cell.2007.11.019",
display_identifier=DisplayIdentifier(
kind="doi",
value="10.1016/j.cell.2007.11.019",
label="DOI",
url=None,
confidence_score=1.0,
),
)
assert unpaywall_app._publication_doi(item) == "10.1016/j.cell.2007.11.019"
@ -54,7 +61,13 @@ def test_publication_doi_prefers_explicit_metadata_doi_over_stored_value() -> No
_item(100),
pub_url="https://doi.org/10.2000/fresh-value",
venue_text="Cell",
doi="10.1000/stale-value",
display_identifier=DisplayIdentifier(
kind="doi",
value="10.1000/stale-value",
label="DOI",
url=None,
confidence_score=1.0,
),
)
assert unpaywall_app._publication_doi(item) == "10.2000/fresh-value"